Add files using upload-large-folder tool

LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0022000_logistic_normal_t1p45.log

@@ -0,0 +1,76 @@
+[watch-lognormal-sde] 2026-05-23_00:36:48 infer runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0022000.pt -> docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0022000
+[load] runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0022000.pt
+[ckpt] step=22000
+[sde] generated 16/256
+[sde] generated 32/256
+[sde] generated 48/256
+[sde] generated 64/256
+[sde] generated 80/256
+[sde] generated 96/256
+[sde] generated 112/256
+[sde] generated 128/256
+[sde] generated 144/256
+[sde] generated 160/256
+[sde] generated 176/256
+[sde] generated 192/256
+[sde] generated 208/256
+[sde] generated 224/256
+[sde] generated 240/256
+[sde] generated 256/256
+[score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
+[summary] {
+  "type": "summary",
+  "checkpoint": "runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0022000.pt",
+  "step": 22000,
+  "decode": {
+    "decode_rule": "logistic_normal_resample_sde",
+    "steps": 128,
+    "model_t_mode": "const0.5",
+    "mean_mode": "anchor_semantic",
+    "endpoint_floor": 0.0,
+    "concentration_min": 1.0,
+    "concentration_max": 1024.0,
+    "endpoint_temp": 1.45,
+    "support_power": 1.0,
+    "semantic_power": 1.0,
+    "noise_init": "logistic_normal",
+    "noise_sigma": 3.0,
+    "noise_dirichlet_concentration": 1.0,
+    "sde_resample": "logistic_normal",
+    "logistic_normal_sigma_min": 0.18,
+    "logistic_normal_sigma_max": 3.0,
+    "logistic_normal_tau_min": 0.65,
+    "logistic_normal_tau_max": 1.0,
+    "final_from": "blend_0.5",
+    "n_samples": 256,
+    "seed": 20260522
+  },
+  "raw_genppl": {
+    "ppl": 36.203633463316066,
+    "nll_per_token": 3.5891594857096103,
+    "tokens": 37429,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "stripped_genppl": {
+    "ppl": 52.686914223709124,
+    "nll_per_token": 3.9643671177635507,
+    "tokens": 30976,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "diversity": {
+    "sample_entropy": 3.8378421959993116,
+    "unique_tokens": 2685,
+    "token_count": 32768,
+    "distinct_1": 0.081939697265625,
+    "distinct_2": 0.3808132381889764,
+    "top_token_mass": 0.087738037109375
+  }
+}
+[done] docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0022000/sde_steps128_samples256_scored.jsonl
+[watch-lognormal-sde] 2026-05-23_00:38:17 done step_0022000

LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0082000_logistic_normal_t1p45.log

@@ -0,0 +1,76 @@
+[watch-lognormal-sde] 2026-05-23_06:11:48 infer runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0082000.pt -> docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0082000
+[load] runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0082000.pt
+[ckpt] step=82000
+[sde] generated 16/256
+[sde] generated 32/256
+[sde] generated 48/256
+[sde] generated 64/256
+[sde] generated 80/256
+[sde] generated 96/256
+[sde] generated 112/256
+[sde] generated 128/256
+[sde] generated 144/256
+[sde] generated 160/256
+[sde] generated 176/256
+[sde] generated 192/256
+[sde] generated 208/256
+[sde] generated 224/256
+[sde] generated 240/256
+[sde] generated 256/256
+[score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
+[summary] {
+  "type": "summary",
+  "checkpoint": "runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0082000.pt",
+  "step": 82000,
+  "decode": {
+    "decode_rule": "logistic_normal_resample_sde",
+    "steps": 128,
+    "model_t_mode": "const0.5",
+    "mean_mode": "anchor_semantic",
+    "endpoint_floor": 0.0,
+    "concentration_min": 1.0,
+    "concentration_max": 1024.0,
+    "endpoint_temp": 1.45,
+    "support_power": 1.0,
+    "semantic_power": 1.0,
+    "noise_init": "logistic_normal",
+    "noise_sigma": 3.0,
+    "noise_dirichlet_concentration": 1.0,
+    "sde_resample": "logistic_normal",
+    "logistic_normal_sigma_min": 0.18,
+    "logistic_normal_sigma_max": 3.0,
+    "logistic_normal_tau_min": 0.65,
+    "logistic_normal_tau_max": 1.0,
+    "final_from": "blend_0.5",
+    "n_samples": 256,
+    "seed": 20260522
+  },
+  "raw_genppl": {
+    "ppl": 32.86963269699827,
+    "nll_per_token": 3.4925492131992817,
+    "tokens": 35961,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "stripped_genppl": {
+    "ppl": 41.948984818093216,
+    "nll_per_token": 3.7364542328133203,
+    "tokens": 30476,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "diversity": {
+    "sample_entropy": 3.5578483548277156,
+    "unique_tokens": 2118,
+    "token_count": 32768,
+    "distinct_1": 0.06463623046875,
+    "distinct_2": 0.33544537401574803,
+    "top_token_mass": 0.09210205078125
+  }
+}
+[done] docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0082000/sde_steps128_samples256_scored.jsonl
+[watch-lognormal-sde] 2026-05-23_06:13:17 done step_0082000

LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0116000_logistic_normal_t1p45.log

@@ -0,0 +1,76 @@
+[watch-lognormal-sde] 2026-05-23_09:21:40 infer runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0116000.pt -> docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0116000
+[load] runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0116000.pt
+[ckpt] step=116000
+[sde] generated 16/256
+[sde] generated 32/256
+[sde] generated 48/256
+[sde] generated 64/256
+[sde] generated 80/256
+[sde] generated 96/256
+[sde] generated 112/256
+[sde] generated 128/256
+[sde] generated 144/256
+[sde] generated 160/256
+[sde] generated 176/256
+[sde] generated 192/256
+[sde] generated 208/256
+[sde] generated 224/256
+[sde] generated 240/256
+[sde] generated 256/256
+[score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
+[summary] {
+  "type": "summary",
+  "checkpoint": "runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0116000.pt",
+  "step": 116000,
+  "decode": {
+    "decode_rule": "logistic_normal_resample_sde",
+    "steps": 128,
+    "model_t_mode": "const0.5",
+    "mean_mode": "anchor_semantic",
+    "endpoint_floor": 0.0,
+    "concentration_min": 1.0,
+    "concentration_max": 1024.0,
+    "endpoint_temp": 1.45,
+    "support_power": 1.0,
+    "semantic_power": 1.0,
+    "noise_init": "logistic_normal",
+    "noise_sigma": 3.0,
+    "noise_dirichlet_concentration": 1.0,
+    "sde_resample": "logistic_normal",
+    "logistic_normal_sigma_min": 0.18,
+    "logistic_normal_sigma_max": 3.0,
+    "logistic_normal_tau_min": 0.65,
+    "logistic_normal_tau_max": 1.0,
+    "final_from": "blend_0.5",
+    "n_samples": 256,
+    "seed": 20260522
+  },
+  "raw_genppl": {
+    "ppl": 33.84903888580392,
+    "nll_per_token": 3.5219106056259073,
+    "tokens": 33058,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "stripped_genppl": {
+    "ppl": 46.07219135564344,
+    "nll_per_token": 3.8302095436000574,
+    "tokens": 27448,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "diversity": {
+    "sample_entropy": 3.3640971283558763,
+    "unique_tokens": 2189,
+    "token_count": 32768,
+    "distinct_1": 0.066802978515625,
+    "distinct_2": 0.3188976377952756,
+    "top_token_mass": 0.18438720703125
+  }
+}
+[done] docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0116000/sde_steps128_samples256_scored.jsonl
+[watch-lognormal-sde] 2026-05-23_09:23:07 done step_0116000

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/deit/__init__.py

@@ -0,0 +1,29 @@
+# Copyright 2024 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import TYPE_CHECKING
+
+from ...utils import _LazyModule
+from ...utils.import_utils import define_import_structure
+
+
+if TYPE_CHECKING:
+    from .configuration_deit import *
+    from .image_processing_deit import *
+    from .image_processing_pil_deit import *
+    from .modeling_deit import *
+else:
+    import sys
+
+    _file = globals()["__file__"]
+    sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__)

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/deit/configuration_deit.py

@@ -0,0 +1,72 @@
+# Copyright 2021 Facebook AI Research (FAIR) and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""DeiT model configuration"""
+
+from huggingface_hub.dataclasses import strict
+
+from ...configuration_utils import PreTrainedConfig
+from ...utils import auto_docstring
+
+
+@auto_docstring(checkpoint="facebook/deit-base-distilled-patch16-224")
+@strict
+class DeiTConfig(PreTrainedConfig):
+    r"""
+    encoder_stride (`int`, *optional*, defaults to 16):
+        Factor to increase the spatial resolution by in the decoder head for masked image modeling.
+    pooler_output_size (`int`, *optional*):
+        Dimensionality of the pooler layer. If None, defaults to `hidden_size`.
+    pooler_act (`str`, *optional*, defaults to `"tanh"`):
+        The activation function to be used by the pooler.
+
+    Example:
+
+    ```python
+    >>> from transformers import DeiTConfig, DeiTModel
+
+    >>> # Initializing a DeiT deit-base-distilled-patch16-224 style configuration
+    >>> configuration = DeiTConfig()
+
+    >>> # Initializing a model (with random weights) from the deit-base-distilled-patch16-224 style configuration
+    >>> model = DeiTModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "deit"
+
+    hidden_size: int = 768
+    num_hidden_layers: int = 12
+    num_attention_heads: int = 12
+    intermediate_size: int = 3072
+    hidden_act: str = "gelu"
+    hidden_dropout_prob: float | int = 0.0
+    attention_probs_dropout_prob: float | int = 0.0
+    initializer_range: float = 0.02
+    layer_norm_eps: float = 1e-12
+    image_size: int | list[int] | tuple[int, int] = 224
+    patch_size: int | list[int] | tuple[int, int] = 16
+    num_channels: int = 3
+    qkv_bias: bool = True
+    encoder_stride: int = 16
+    pooler_output_size: int | None = None
+    pooler_act: str = "tanh"
+
+    def __post_init__(self, **kwargs):
+        self.pooler_output_size = self.pooler_output_size if self.pooler_output_size else self.hidden_size
+        super().__post_init__(**kwargs)
+
+
+__all__ = ["DeiTConfig"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/deit/image_processing_deit.py

@@ -0,0 +1,34 @@
+# Copyright 2022 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Image processor class for DeiT."""
+
+from ...image_processing_backends import TorchvisionBackend
+from ...image_utils import IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD, PILImageResampling
+from ...utils import auto_docstring
+
+
+@auto_docstring
+class DeiTImageProcessor(TorchvisionBackend):
+    resample = PILImageResampling.BICUBIC
+    image_mean = IMAGENET_STANDARD_MEAN
+    image_std = IMAGENET_STANDARD_STD
+    size = {"height": 256, "width": 256}
+    crop_size = {"height": 224, "width": 224}
+    do_resize = True
+    do_center_crop = True
+    do_rescale = True
+    do_normalize = True
+
+
+__all__ = ["DeiTImageProcessor"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/deit/image_processing_pil_deit.py

@@ -0,0 +1,34 @@
+# Copyright 2022 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Image processor class for DeiT."""
+
+from ...image_processing_backends import PilBackend
+from ...image_utils import IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD, PILImageResampling
+from ...utils import auto_docstring
+
+
+@auto_docstring
+class DeiTImageProcessorPil(PilBackend):
+    resample = PILImageResampling.BICUBIC
+    image_mean = IMAGENET_STANDARD_MEAN
+    image_std = IMAGENET_STANDARD_STD
+    size = {"height": 256, "width": 256}
+    crop_size = {"height": 224, "width": 224}
+    do_resize = True
+    do_center_crop = True
+    do_rescale = True
+    do_normalize = True
+
+
+__all__ = ["DeiTImageProcessorPil"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/deit/modeling_deit.py

@@ -0,0 +1,671 @@
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+#           This file was automatically generated from src/transformers/models/deit/modular_deit.py.
+#               Do NOT edit this file manually as any edits will be overwritten by the generation of
+#             the file from the modular. If any change should be done, please apply the change to the
+#                          modular_deit.py file directly. One of our CI enforces this.
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# Copyright 2021 Facebook AI Research & The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass
+
+import torch
+from torch import nn
+
+from ... import initialization as init
+from ...activations import ACT2FN
+from ...masking_utils import create_bidirectional_mask
+from ...modeling_layers import GradientCheckpointingLayer
+from ...modeling_outputs import BaseModelOutputWithPooling, ImageClassifierOutput, MaskedImageModelingOutput
+from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
+from ...processing_utils import Unpack
+from ...utils import ModelOutput, TransformersKwargs, auto_docstring, torch_int
+from ...utils.generic import can_return_tuple, merge_with_config_defaults
+from ...utils.output_capturing import capture_outputs
+from .configuration_deit import DeiTConfig
+
+
+class DeiTPatchEmbeddings(nn.Module):
+    """
+    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
+    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
+    Transformer.
+    """
+
+    def __init__(self, config: DeiTConfig):
+        super().__init__()
+        image_size = config.image_size
+        patch_size = config.patch_size
+        image_size = image_size if isinstance(image_size, Iterable) else (image_size, image_size)
+        patch_size = patch_size if isinstance(patch_size, Iterable) else (patch_size, patch_size)
+
+        self.num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
+        self.image_size = image_size
+        self.patch_size = patch_size
+        self.num_channels = config.num_channels
+        self.projection = nn.Conv2d(config.num_channels, config.hidden_size, kernel_size=patch_size, stride=patch_size)
+
+    def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
+        num_channels = pixel_values.shape[1]
+        if num_channels != self.num_channels:
+            raise ValueError(
+                "Make sure that the channel dimension of the pixel values match with the one set in the configuration."
+                f" Expected {self.num_channels} but got {num_channels}."
+            )
+        return self.projection(pixel_values).flatten(2).transpose(1, 2)
+
+
+class DeiTEmbeddings(nn.Module):
+    """
+    Construct the CLS token, distillation token, position and patch embeddings. Optionally, also the mask token.
+
+    Differences from ViTEmbeddings:
+    - Adds a distillation token (for distillation pre-training).
+    - Position embeddings include +2 slots (CLS + distillation) instead of +1.
+    - interpolate_pos_encoding handles 2 special tokens instead of 1.
+    - forward concatenates distillation token and handles position encoding for both.
+    """
+
+    def __init__(self, config: DeiTConfig, use_mask_token: bool = False) -> None:
+        super().__init__()
+        self.cls_token = nn.Parameter(torch.zeros(1, 1, config.hidden_size))
+        self.mask_token = nn.Parameter(torch.zeros(1, 1, config.hidden_size)) if use_mask_token else None
+        self.patch_embeddings = DeiTPatchEmbeddings(config)
+        num_patches = self.patch_embeddings.num_patches
+        # +2: one slot for CLS, one for distillation token
+        self.position_embeddings = nn.Parameter(torch.zeros(1, num_patches + 2, config.hidden_size))
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+        self.patch_size = config.patch_size
+        self.image_size = self.patch_embeddings.image_size
+        self.distillation_token = nn.Parameter(torch.zeros(1, 1, config.hidden_size))
+
+    def interpolate_pos_encoding(self, embeddings: torch.Tensor, height: int, width: int) -> torch.Tensor:
+        """
+        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
+        images. This method is also adapted to support torch.jit tracing and 2 class embeddings.
+
+        Adapted from:
+        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
+        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
+        """
+
+        num_patches = embeddings.shape[1] - 2
+        num_positions = self.position_embeddings.shape[1] - 2
+
+        # always interpolate when tracing to ensure the exported model works for dynamic input shapes
+        if not torch.jit.is_tracing() and num_patches == num_positions and height == width:
+            return self.position_embeddings
+
+        class_and_dist_pos_embed = self.position_embeddings[:, :2]
+        patch_pos_embed = self.position_embeddings[:, 2:]
+
+        dim = embeddings.shape[-1]
+
+        new_height = height // self.patch_size
+        new_width = width // self.patch_size
+
+        sqrt_num_positions = torch_int(num_positions**0.5)
+        patch_pos_embed = patch_pos_embed.reshape(1, sqrt_num_positions, sqrt_num_positions, dim)
+        patch_pos_embed = patch_pos_embed.permute(0, 3, 1, 2)
+
+        patch_pos_embed = nn.functional.interpolate(
+            patch_pos_embed,
+            size=(new_height, new_width),
+            mode="bicubic",
+            align_corners=False,
+        )
+
+        patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim)
+
+        return torch.cat((class_and_dist_pos_embed, patch_pos_embed), dim=1)
+
+    def forward(
+        self,
+        pixel_values: torch.Tensor,
+        bool_masked_pos: torch.BoolTensor | None = None,
+        interpolate_pos_encoding: bool = False,
+    ) -> torch.Tensor:
+        _, _, height, width = pixel_values.shape
+        embeddings = self.patch_embeddings(pixel_values)
+
+        batch_size, seq_length, _ = embeddings.size()
+
+        if bool_masked_pos is not None:
+            mask_tokens = self.mask_token.expand(batch_size, seq_length, -1)
+            # replace the masked visual tokens by mask_tokens
+            mask = bool_masked_pos.unsqueeze(-1).type_as(mask_tokens)
+            embeddings = embeddings * (1.0 - mask) + mask_tokens * mask
+
+        cls_tokens = self.cls_token.expand(batch_size, -1, -1)
+        distillation_tokens = self.distillation_token.expand(batch_size, -1, -1)
+        embeddings = torch.cat((cls_tokens, distillation_tokens, embeddings), dim=1)
+
+        if interpolate_pos_encoding:
+            embeddings = embeddings + self.interpolate_pos_encoding(embeddings, height, width)
+        else:
+            if height != self.image_size[0] or width != self.image_size[1]:
+                raise ValueError(
+                    f"Input image size ({height}*{width}) doesn't match model"
+                    f" ({self.image_size[0]}*{self.image_size[1]})."
+                )
+            embeddings = embeddings + self.position_embeddings
+
+        embeddings = self.dropout(embeddings)
+        return embeddings
+
+
+def eager_attention_forward(
+    module: nn.Module,
+    query: torch.Tensor,
+    key: torch.Tensor,
+    value: torch.Tensor,
+    attention_mask: torch.Tensor | None,
+    scaling: float | None = None,
+    dropout: float = 0.0,
+    **kwargs: Unpack[TransformersKwargs],
+):
+    if scaling is None:
+        scaling = query.size(-1) ** -0.5
+
+    # Take the dot product between "query" and "key" to get the raw attention scores.
+    attn_weights = torch.matmul(query, key.transpose(2, 3)) * scaling
+
+    if attention_mask is not None:
+        attn_weights = attn_weights + attention_mask
+
+    attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)
+    attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)
+
+    attn_output = torch.matmul(attn_weights, value)
+    attn_output = attn_output.transpose(1, 2).contiguous()
+
+    return attn_output, attn_weights
+
+
+class DeiTAttention(nn.Module):
+    def __init__(self, config: DeiTConfig):
+        super().__init__()
+        self.config = config
+        self.num_attention_heads = config.num_attention_heads
+        self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
+        self.attention_dropout = config.attention_probs_dropout_prob
+        self.scaling = self.head_dim**-0.5
+        self.is_causal = False
+
+        self.q_proj = nn.Linear(config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.qkv_bias)
+        self.k_proj = nn.Linear(config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.qkv_bias)
+        self.v_proj = nn.Linear(config.hidden_size, config.num_attention_heads * self.head_dim, bias=config.qkv_bias)
+        self.o_proj = nn.Linear(config.num_attention_heads * self.head_dim, config.hidden_size, bias=True)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: torch.Tensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple[torch.Tensor, torch.Tensor]:
+        input_shape = hidden_states.shape[:-1]
+        hidden_shape = (*input_shape, -1, self.head_dim)
+
+        query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+        key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+        value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+
+        attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface(
+            self.config._attn_implementation, eager_attention_forward
+        )
+
+        attn_output, attn_weights = attention_interface(
+            self,
+            query_states,
+            key_states,
+            value_states,
+            attention_mask,
+            dropout=0.0 if not self.training else self.attention_dropout,
+            scaling=self.scaling,
+            **kwargs,
+        )
+
+        attn_output = attn_output.reshape(*input_shape, -1).contiguous()
+        attn_output = self.o_proj(attn_output)
+
+        return attn_output, attn_weights
+
+
+class DeiTMLP(nn.Module):
+    def __init__(self, config: DeiTConfig):
+        super().__init__()
+        self.config = config
+        self.activation_fn = ACT2FN[config.hidden_act]
+        self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
+        self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        hidden_states = self.fc1(hidden_states)
+        hidden_states = self.activation_fn(hidden_states)
+        hidden_states = self.fc2(hidden_states)
+
+        return hidden_states
+
+
+class DeiTLayer(GradientCheckpointingLayer):
+    def __init__(self, config: DeiTConfig):
+        super().__init__()
+        self.attention = DeiTAttention(config)
+        self.layernorm_before = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.layernorm_after = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.mlp = DeiTMLP(config)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: torch.Tensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> torch.Tensor:
+        # Self Attention
+        residual = hidden_states
+        hidden_states = self.layernorm_before(hidden_states)
+        hidden_states, _ = self.attention(hidden_states, attention_mask, **kwargs)
+        hidden_states = self.dropout(hidden_states)
+        hidden_states = hidden_states + residual
+
+        # Fully Connected
+        residual = hidden_states
+        hidden_states = self.layernorm_after(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = self.dropout(hidden_states)
+        hidden_states = hidden_states + residual
+
+        return hidden_states
+
+
+@auto_docstring
+class DeiTPreTrainedModel(PreTrainedModel):
+    config: DeiTConfig
+    base_model_prefix = "deit"
+    main_input_name = "pixel_values"
+    input_modalities = ("image",)
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["DeiTEmbeddings", "DeiTLayer"]
+    _supports_sdpa = True
+    _supports_flash_attn = True
+    _supports_flex_attn = True
+    _supports_attention_backend = True
+    _can_compile_fullgraph = True
+    _can_record_outputs = {
+        "hidden_states": DeiTLayer,
+        "attentions": DeiTAttention,
+    }
+    _input_embed_layer = "patch_embeddings"
+
+    @torch.no_grad()
+    def _init_weights(self, module: nn.Linear | nn.Conv2d | nn.LayerNorm) -> None:
+        """Initialize the weights"""
+        super()._init_weights(module)
+        if isinstance(module, DeiTEmbeddings):
+            if module.position_embeddings is not None:
+                init.trunc_normal_(module.position_embeddings, mean=0.0, std=self.config.initializer_range)
+            init.trunc_normal_(module.cls_token, mean=0.0, std=self.config.initializer_range)
+            if module.mask_token is not None:
+                init.zeros_(module.mask_token)
+        if isinstance(module, DeiTEmbeddings):
+            init.zeros_(module.cls_token)
+            init.zeros_(module.position_embeddings)
+            init.zeros_(module.distillation_token)
+            if module.mask_token is not None:
+                init.zeros_(module.mask_token)
+
+
+class DeiTPooler(nn.Module):
+    def __init__(self, config: DeiTConfig):
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.pooler_output_size)
+        self.activation = ACT2FN[config.pooler_act]
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        # We "pool" the model by simply taking the hidden state corresponding
+        # to the first token.
+        first_token_tensor = hidden_states[:, 0]
+        pooled_output = self.dense(first_token_tensor)
+        pooled_output = self.activation(pooled_output)
+        return pooled_output
+
+
+@auto_docstring
+class DeiTModel(DeiTPreTrainedModel):
+    def __init__(self, config: DeiTConfig, add_pooling_layer: bool = True, use_mask_token: bool = False):
+        r"""
+        add_pooling_layer (bool, *optional*, defaults to `True`):
+            Whether to add a pooling layer
+        use_mask_token (`bool`, *optional*, defaults to `False`):
+            Whether to use a mask token for masked image modeling.
+        """
+        super().__init__(config)
+        self.config = config
+        self.embeddings = DeiTEmbeddings(config, use_mask_token=use_mask_token)
+        self.layers = nn.ModuleList([DeiTLayer(config) for _ in range(config.num_hidden_layers)])
+        self.layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.pooler = DeiTPooler(config) if add_pooling_layer else None
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    @merge_with_config_defaults
+    @capture_outputs(tie_last_hidden_states=False)
+    @auto_docstring
+    def forward(
+        self,
+        pixel_values: torch.Tensor | None = None,
+        bool_masked_pos: torch.BoolTensor | None = None,
+        interpolate_pos_encoding: bool | None = None,
+        attention_mask: torch.Tensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> BaseModelOutputWithPooling:
+        r"""
+        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
+            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
+        """
+        # Kept for BC, but this should be handled by users on the processed inputs directly.
+        expected_dtype = self.embeddings.patch_embeddings.projection.weight.dtype
+        if pixel_values.dtype != expected_dtype:
+            pixel_values = pixel_values.to(expected_dtype)
+
+        embedding_output = self.embeddings(
+            pixel_values, bool_masked_pos=bool_masked_pos, interpolate_pos_encoding=interpolate_pos_encoding
+        )
+        attention_mask = create_bidirectional_mask(
+            config=self.config,
+            inputs_embeds=embedding_output,
+            attention_mask=attention_mask,
+        )
+        hidden_states = embedding_output
+        for layer in self.layers:
+            hidden_states = layer(hidden_states, attention_mask, **kwargs)
+
+        sequence_output = self.layernorm(hidden_states)
+        pooled_output = self.pooler(sequence_output) if self.pooler is not None else None
+
+        return BaseModelOutputWithPooling(last_hidden_state=sequence_output, pooler_output=pooled_output)
+
+
+@auto_docstring(
+    custom_intro="""
+    DeiT Model with a decoder on top for masked image modeling, as proposed in [SimMIM](https://huggingface.co/papers/2111.09886).
+
+    <Tip>
+
+    Note that we provide a script to pre-train this model on custom data in our [examples
+    directory](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-pretraining).
+
+    </Tip>
+    """
+)
+class DeiTForMaskedImageModeling(DeiTPreTrainedModel):
+    def __init__(self, config: DeiTConfig):
+        super().__init__(config)
+
+        self.deit = DeiTModel(config, add_pooling_layer=False, use_mask_token=True)
+
+        self.decoder = nn.Sequential(
+            nn.Conv2d(
+                in_channels=config.hidden_size,
+                out_channels=config.encoder_stride**2 * config.num_channels,
+                kernel_size=1,
+            ),
+            nn.PixelShuffle(config.encoder_stride),
+        )
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        pixel_values: torch.Tensor | None = None,
+        bool_masked_pos: torch.BoolTensor | None = None,
+        interpolate_pos_encoding: bool = False,
+        attention_mask: torch.Tensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> MaskedImageModelingOutput:
+        r"""
+        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`):
+            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
+
+        Examples:
+        ```python
+        >>> from transformers import AutoImageProcessor, DeiTForMaskedImageModeling
+        >>> import torch
+        >>> from PIL import Image
+        >>> import requests
+
+        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+        >>> image = Image.open(requests.get(url, stream=True).raw)
+
+        >>> image_processor = AutoImageProcessor.from_pretrained("facebook/deit-base-distilled-patch16-224")
+        >>> model = DeiTForMaskedImageModeling.from_pretrained("facebook/deit-base-distilled-patch16-224")
+
+        >>> num_patches = (model.config.image_size // model.config.patch_size) ** 2
+        >>> pixel_values = image_processor(images=image, return_tensors="pt").pixel_values
+        >>> # create random boolean mask of shape (batch_size, num_patches)
+        >>> bool_masked_pos = torch.randint(low=0, high=2, size=(1, num_patches)).bool()
+
+        >>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos)
+        >>> loss, reconstructed_pixel_values = outputs.loss, outputs.reconstruction
+        >>> list(reconstructed_pixel_values.shape)
+        [1, 3, 224, 224]
+        ```"""
+
+        outputs: BaseModelOutputWithPooling = self.deit(
+            pixel_values,
+            bool_masked_pos=bool_masked_pos,
+            interpolate_pos_encoding=interpolate_pos_encoding,
+            attention_mask=attention_mask,
+            **kwargs,
+        )
+
+        sequence_output = outputs.last_hidden_state
+
+        # Reshape to (batch_size, num_channels, height, width)
+        # Remove the [CLS] token (index 0) and distillation token (index 1), keep only patch embeddings
+        sequence_output = sequence_output[:, 2:]
+        batch_size, sequence_length, num_channels = sequence_output.shape
+        height = width = int(sequence_length**0.5)
+        sequence_output = sequence_output.permute(0, 2, 1).reshape(batch_size, num_channels, height, width)
+
+        # Reconstruct pixel values
+        reconstructed_pixel_values = self.decoder(sequence_output)
+
+        masked_im_loss = None
+        if bool_masked_pos is not None:
+            size = self.config.image_size // self.config.patch_size
+            bool_masked_pos = bool_masked_pos.reshape(-1, size, size)
+            mask = (
+                bool_masked_pos.repeat_interleave(self.config.patch_size, 1)
+                .repeat_interleave(self.config.patch_size, 2)
+                .unsqueeze(1)
+                .contiguous()
+            )
+            reconstruction_loss = nn.functional.l1_loss(pixel_values, reconstructed_pixel_values, reduction="none")
+            masked_im_loss = (reconstruction_loss * mask).sum() / (mask.sum() + 1e-5) / self.config.num_channels
+
+        return MaskedImageModelingOutput(
+            loss=masked_im_loss,
+            reconstruction=reconstructed_pixel_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+@auto_docstring(
+    custom_intro="""
+    DeiT Model transformer with an image classification head on top (a linear layer on top of the final hidden state of
+    the [CLS] token) e.g. for ImageNet.
+
+    <Tip>
+
+        Note that it's possible to fine-tune DeiT on higher resolution images than the ones it has been trained on, by
+        setting `interpolate_pos_encoding` to `True` in the forward of the model. This will interpolate the pre-trained
+        position embeddings to the higher resolution.
+
+    </Tip>
+    """
+)
+class DeiTForImageClassification(DeiTPreTrainedModel):
+    def __init__(self, config: DeiTConfig):
+        super().__init__(config)
+
+        self.num_labels = config.num_labels
+        self.deit = DeiTModel(config, add_pooling_layer=False)
+
+        # Classifier head
+        self.classifier = nn.Linear(config.hidden_size, config.num_labels) if config.num_labels > 0 else nn.Identity()
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        pixel_values: torch.Tensor | None = None,
+        labels: torch.Tensor | None = None,
+        interpolate_pos_encoding: bool | None = None,
+        attention_mask: torch.Tensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> ImageClassifierOutput:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
+            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+        """
+
+        outputs: BaseModelOutputWithPooling = self.deit(
+            pixel_values,
+            interpolate_pos_encoding=interpolate_pos_encoding,
+            attention_mask=attention_mask,
+            **kwargs,
+        )
+
+        sequence_output = outputs.last_hidden_state
+        pooled_output = sequence_output[:, 0, :]
+        logits = self.classifier(pooled_output)
+
+        loss = None
+        if labels is not None:
+            loss = self.loss_function(labels, logits, self.config, **kwargs)
+
+        return ImageClassifierOutput(
+            loss=loss,
+            logits=logits,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+@auto_docstring(
+    custom_intro="""
+    Output type of [`DeiTForImageClassificationWithTeacher`].
+    """
+)
+@dataclass
+class DeiTForImageClassificationWithTeacherOutput(ModelOutput):
+    r"""
+    logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
+        Prediction scores as the average of the cls_logits and distillation logits.
+    cls_logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
+        Prediction scores of the classification head (i.e. the linear layer on top of the final hidden state of the
+        class token).
+    distillation_logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
+        Prediction scores of the distillation head (i.e. the linear layer on top of the final hidden state of the
+        distillation token).
+    """
+
+    logits: torch.FloatTensor | None = None
+    cls_logits: torch.FloatTensor | None = None
+    distillation_logits: torch.FloatTensor | None = None
+    hidden_states: tuple[torch.FloatTensor] | None = None
+    attentions: tuple[torch.FloatTensor] | None = None
+
+
+@auto_docstring(
+    custom_intro="""
+    DeiT Model transformer with image classification heads on top (a linear layer on top of the final hidden state of
+    the [CLS] token and a linear layer on top of the final hidden state of the distillation token) e.g. for ImageNet.
+
+    .. warning::
+
+           This model supports inference-only. Fine-tuning with distillation (i.e. with a teacher) is not yet
+           supported.
+    """
+)
+class DeiTForImageClassificationWithTeacher(DeiTPreTrainedModel):
+    def __init__(self, config: DeiTConfig) -> None:
+        super().__init__(config)
+
+        self.num_labels = config.num_labels
+        self.deit = DeiTModel(config, add_pooling_layer=False)
+
+        # Classifier heads
+        self.cls_classifier = (
+            nn.Linear(config.hidden_size, config.num_labels) if config.num_labels > 0 else nn.Identity()
+        )
+        self.distillation_classifier = (
+            nn.Linear(config.hidden_size, config.num_labels) if config.num_labels > 0 else nn.Identity()
+        )
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        pixel_values: torch.Tensor | None = None,
+        interpolate_pos_encoding: bool = False,
+        attention_mask: torch.Tensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> DeiTForImageClassificationWithTeacherOutput:
+        outputs: BaseModelOutputWithPooling = self.deit(
+            pixel_values,
+            interpolate_pos_encoding=interpolate_pos_encoding,
+            attention_mask=attention_mask,
+            **kwargs,
+        )
+
+        sequence_output = outputs.last_hidden_state
+
+        cls_logits = self.cls_classifier(sequence_output[:, 0, :])
+        distillation_logits = self.distillation_classifier(sequence_output[:, 1, :])
+
+        # during inference, return the average of both classifier predictions
+        logits = (cls_logits + distillation_logits) / 2
+
+        return DeiTForImageClassificationWithTeacherOutput(
+            logits=logits,
+            cls_logits=cls_logits,
+            distillation_logits=distillation_logits,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+__all__ = [
+    "DeiTForImageClassification",
+    "DeiTForImageClassificationWithTeacher",
+    "DeiTForMaskedImageModeling",
+    "DeiTModel",
+    "DeiTPreTrainedModel",
+]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/imagegpt/configuration_imagegpt.py

@@ -0,0 +1,79 @@
+# Copyright 2021 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""OpenAI ImageGPT configuration"""
+
+from huggingface_hub.dataclasses import strict
+
+from ...configuration_utils import PreTrainedConfig
+from ...utils import auto_docstring
+
+
+@auto_docstring(checkpoint="openai/imagegpt-small")
+@strict
+class ImageGPTConfig(PreTrainedConfig):
+    r"""
+    scale_attn_by_inverse_layer_idx (`bool`, *optional*, defaults to `False`):
+        Whether to additionally scale attention weights by `1 / layer_idx + 1`.
+    reorder_and_upcast_attn (`bool`, *optional*, defaults to `False`):
+        Whether to scale keys (K) prior to computing attention (dot-product) and upcast attention
+        dot-product/softmax to float() when training with mixed precision.
+
+    Example:
+
+    ```python
+    >>> from transformers import ImageGPTConfig, ImageGPTModel
+
+    >>> # Initializing a ImageGPT configuration
+    >>> configuration = ImageGPTConfig()
+
+    >>> # Initializing a model (with random weights) from the configuration
+    >>> model = ImageGPTModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "imagegpt"
+    keys_to_ignore_at_inference = ["past_key_values"]
+    attribute_map = {
+        "hidden_size": "n_embd",
+        "max_position_embeddings": "n_positions",
+        "num_attention_heads": "n_head",
+        "num_hidden_layers": "n_layer",
+    }
+
+    vocab_size: int = 512 + 1  # add one for start of sentence (sos) token
+    n_positions: int = 32 * 32
+    n_embd: int = 512
+    n_layer: int = 24
+    n_head: int = 8
+    n_inner: int | None = None
+    activation_function: str = "quick_gelu"
+    resid_pdrop: float | int = 0.1
+    embd_pdrop: float | int = 0.1
+    attn_pdrop: float | int = 0.1
+    layer_norm_epsilon: float = 1e-5
+    initializer_range: float = 0.02
+    scale_attn_weights: bool = True
+    use_cache: bool = True
+    tie_word_embeddings: bool = False
+    scale_attn_by_inverse_layer_idx: bool = False
+    reorder_and_upcast_attn: bool = False
+    add_cross_attention: bool = False
+    pad_token_id: int | None = None
+    bos_token_id: int | None = None
+    eos_token_id: int | list[int] | None = None
+
+
+__all__ = ["ImageGPTConfig"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/imagegpt/image_processing_imagegpt.py

@@ -0,0 +1,192 @@
+# Copyright 2022 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Image processor class for ImageGPT."""
+
+from typing import Union
+
+import numpy as np
+import torch
+from torchvision.transforms.v2 import functional as tvF
+
+from ...image_processing_backends import TorchvisionBackend
+from ...image_processing_utils import BatchFeature
+from ...image_transforms import group_images_by_shape, reorder_images
+from ...image_utils import PILImageResampling, SizeDict
+from ...processing_utils import ImagesKwargs, Unpack
+from ...utils import (
+    TensorType,
+    auto_docstring,
+)
+
+
+class ImageGPTImageProcessorKwargs(ImagesKwargs, total=False):
+    r"""
+    clusters (`np.ndarray` or `list[list[int]]` or `torch.Tensor`, *optional*, defaults to `self.clusters`):
+        The color clusters to use, of shape `(n_clusters, 3)` when color quantizing. Can be overridden by `clusters`
+        in `preprocess`.
+    do_color_quantize (`bool`, *optional*, defaults to `self.do_color_quantize`):
+        Controls whether to apply color quantization to convert continuous pixel values to discrete cluster indices.
+        When True, each pixel is assigned to its nearest color cluster, enabling ImageGPT's discrete token modeling.
+    """
+
+    clusters: Union[np.ndarray, list[list[int]], "torch.Tensor"] | None
+    do_color_quantize: bool
+
+
+def squared_euclidean_distance_torch(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
+    """
+    Compute squared Euclidean distances between all pixels and clusters.
+
+    Args:
+        a: (N, 3) tensor of pixel RGB values
+        b: (M, 3) tensor of cluster RGB values
+
+    Returns:
+        (N, M) tensor of squared distances
+    """
+    b = b.t()  # (3, M)
+    a2 = torch.sum(a**2, dim=1)  # (N,)
+    b2 = torch.sum(b**2, dim=0)  # (M,)
+    ab = torch.matmul(a, b)  # (N, M)
+    d = a2[:, None] - 2 * ab + b2[None, :]  # Squared Euclidean Distance: a^2 - 2ab + b^2
+    return d  # (N, M) tensor of squared distances
+
+
+def color_quantize_torch(x: torch.Tensor, clusters: torch.Tensor) -> torch.Tensor:
+    """
+    Assign each pixel to its nearest color cluster.
+
+    Args:
+        x: (H*W, 3) tensor of flattened pixel RGB values
+        clusters: (n_clusters, 3) tensor of cluster RGB values
+
+    Returns:
+        (H*W,) tensor of cluster indices
+    """
+    d = squared_euclidean_distance_torch(x, clusters)
+    return torch.argmin(d, dim=1)
+
+
+@auto_docstring
+class ImageGPTImageProcessor(TorchvisionBackend):
+    model_input_names = ["input_ids"]
+    valid_kwargs = ImageGPTImageProcessorKwargs
+    resample = PILImageResampling.BILINEAR
+    do_color_quantize = True
+    clusters = None
+    image_mean = [0.5, 0.5, 0.5]
+    image_std = [0.5, 0.5, 0.5]
+    do_rescale = True
+    do_normalize = True
+    size = {"height": 256, "width": 256}
+    do_resize = True
+
+    def __init__(
+        self,
+        clusters: list | np.ndarray | torch.Tensor | None = None,  # keep as arg for backwards compatibility
+        **kwargs: Unpack[ImageGPTImageProcessorKwargs],
+    ):
+        r"""
+        clusters (`np.ndarray` or `list[list[int]]` or `torch.Tensor`, *optional*):
+            The color clusters to use, of shape `(n_clusters, 3)` when color quantizing. Can be overridden by `clusters`
+            in `preprocess`.
+        """
+        clusters = torch.as_tensor(clusters, dtype=torch.float32) if clusters is not None else None
+        super().__init__(clusters=clusters, **kwargs)
+
+    def _preprocess(
+        self,
+        images: list["torch.Tensor"],
+        do_resize: bool,
+        size: SizeDict,
+        resample: "PILImageResampling | tvF.InterpolationMode | int | None",
+        do_center_crop: bool,
+        crop_size: SizeDict,
+        do_rescale: bool,
+        rescale_factor: float,
+        do_normalize: bool,
+        image_mean: float | list[float] | None,
+        image_std: float | list[float] | None,
+        disable_grouping: bool | None,
+        return_tensors: str | TensorType | None,
+        do_color_quantize: bool | None = None,
+        clusters: list | np.ndarray | torch.Tensor | None = None,
+        **kwargs,
+    ):
+        # Group images by size for batched resizing
+        grouped_images, grouped_images_index = group_images_by_shape(images, disable_grouping=disable_grouping)
+        resized_images_grouped = {}
+        for shape, stacked_images in grouped_images.items():
+            if do_resize:
+                stacked_images = self.resize(image=stacked_images, size=size, resample=resample)
+            resized_images_grouped[shape] = stacked_images
+        resized_images = reorder_images(resized_images_grouped, grouped_images_index)
+
+        # Group images by size for further processing
+        # Needed in case do_resize is False, or resize returns images with different sizes
+        grouped_images, grouped_images_index = group_images_by_shape(resized_images, disable_grouping=disable_grouping)
+        processed_images_grouped = {}
+        for shape, stacked_images in grouped_images.items():
+            if do_center_crop:
+                stacked_images = self.center_crop(stacked_images, crop_size)
+            # Fused rescale and normalize
+            stacked_images = self.rescale_and_normalize(
+                stacked_images, do_rescale, rescale_factor, do_normalize, image_mean, image_std
+            )
+            processed_images_grouped[shape] = stacked_images
+
+        pixel_values = reorder_images(processed_images_grouped, grouped_images_index)
+
+        # If color quantization is requested, perform it; otherwise return pixel values
+        if do_color_quantize:
+            # Prepare clusters
+            if clusters is None:
+                raise ValueError("Clusters must be provided for color quantization.")
+            # Convert to torch tensor if needed (clusters might be passed as list/numpy)
+            clusters_torch = (
+                torch.as_tensor(clusters, dtype=torch.float32) if not isinstance(clusters, torch.Tensor) else clusters
+            ).to(pixel_values[0].device, dtype=pixel_values[0].dtype)
+
+            # Group images by shape for batch processing
+            # We need to check if the pixel values are a tensor or a list of tensors
+            grouped_images, grouped_images_index = group_images_by_shape(
+                pixel_values, disable_grouping=disable_grouping
+            )
+            # Process each group
+            input_ids_grouped = {}
+
+            for shape, stacked_images in grouped_images.items():
+                input_ids = color_quantize_torch(
+                    stacked_images.permute(0, 2, 3, 1).reshape(-1, 3), clusters_torch
+                )  # (B*H*W, C)
+                input_ids_grouped[shape] = input_ids.reshape(stacked_images.shape[0], -1).reshape(
+                    stacked_images.shape[0], -1
+                )  # (B, H, W)
+
+            input_ids = reorder_images(input_ids_grouped, grouped_images_index)
+
+            return BatchFeature(data={"input_ids": input_ids}, tensor_type=return_tensors)
+
+        return BatchFeature(data={"pixel_values": pixel_values}, tensor_type=return_tensors)
+
+    def to_dict(self):
+        # Convert torch tensors to lists for JSON serialization
+        output = super().to_dict()
+        if output.get("clusters") is not None and isinstance(output["clusters"], torch.Tensor):
+            output["clusters"] = output["clusters"].tolist()
+
+        return output
+
+
+__all__ = ["ImageGPTImageProcessor"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/imagegpt/image_processing_pil_imagegpt.py

@@ -0,0 +1,155 @@
+# Copyright 2022 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Image processor class for ImageGPT."""
+
+from typing import Union
+
+import numpy as np
+
+from ...image_processing_backends import PilBackend
+from ...image_processing_utils import BatchFeature
+from ...image_utils import (
+    PILImageResampling,
+    SizeDict,
+)
+from ...processing_utils import ImagesKwargs, Unpack
+from ...utils import (
+    TensorType,
+    auto_docstring,
+    is_torch_available,
+)
+
+
+if is_torch_available():
+    import torch
+
+
+def squared_euclidean_distance(a, b):
+    b = b.T
+    a2 = np.sum(np.square(a), axis=1)
+    b2 = np.sum(np.square(b), axis=0)
+    ab = np.matmul(a, b)
+    d = a2[:, None] - 2 * ab + b2[None, :]
+    return d
+
+
+def color_quantize(x, clusters):
+    x = x.reshape(-1, 3)
+    d = squared_euclidean_distance(x, clusters)
+    return np.argmin(d, axis=1)
+
+
+# Adapted from transformers.models.imagegpt.image_processing_imagegpt.ImageGPTImageProcessorKwargs
+class ImageGPTImageProcessorKwargs(ImagesKwargs, total=False):
+    r"""
+    clusters (`np.ndarray` or `list[list[int]]` or `torch.Tensor`, *optional*, defaults to `self.clusters`):
+        The color clusters to use, of shape `(n_clusters, 3)` when color quantizing. Can be overridden by `clusters`
+        in `preprocess`.
+    do_color_quantize (`bool`, *optional*, defaults to `self.do_color_quantize`):
+        Controls whether to apply color quantization to convert continuous pixel values to discrete cluster indices.
+        When True, each pixel is assigned to its nearest color cluster, enabling ImageGPT's discrete token modeling.
+    """
+
+    clusters: Union[np.ndarray, list[list[int]], "torch.Tensor"] | None
+    do_color_quantize: bool
+
+
+@auto_docstring
+class ImageGPTImageProcessorPil(PilBackend):
+    model_input_names = ["input_ids"]
+    valid_kwargs = ImageGPTImageProcessorKwargs
+    resample = PILImageResampling.BILINEAR
+    do_color_quantize = True
+    clusters = None
+    image_mean = [0.5, 0.5, 0.5]
+    image_std = [0.5, 0.5, 0.5]
+    do_rescale = True
+    do_normalize = True
+    size = {"height": 256, "width": 256}
+    do_resize = True
+
+    def __init__(
+        self,
+        clusters: "list | np.ndarray | torch.Tensor | None" = None,  # keep as arg for backwards compatibility
+        **kwargs: Unpack[ImageGPTImageProcessorKwargs],
+    ):
+        r"""
+        clusters (`np.ndarray` or `list[list[int]]` or `torch.Tensor`, *optional*):
+            The color clusters to use, of shape `(n_clusters, 3)` when color quantizing. Can be overridden by `clusters`
+            in `preprocess`.
+        """
+        if clusters is not None:
+            clusters = np.array(clusters)
+        super().__init__(clusters=clusters, **kwargs)
+
+    def _preprocess(
+        self,
+        images: list[np.ndarray],
+        do_resize: bool,
+        size: SizeDict,
+        resample: "PILImageResampling | None",
+        do_rescale: bool,
+        rescale_factor: float,
+        do_normalize: bool,
+        image_mean: float | list[float] | None,
+        image_std: float | list[float] | None,
+        return_tensors: str | TensorType | None,
+        do_color_quantize: bool | None = None,
+        clusters: "list | np.ndarray | torch.Tensor | None" = None,
+        **kwargs,
+    ):
+        processed_images = []
+        for image in images:
+            if do_resize:
+                image = self.resize(image, size, resample)
+            if do_rescale:
+                image = self.rescale(image, rescale_factor)
+            if do_normalize:
+                image = self.normalize(image, image_mean, image_std)
+            processed_images.append(image)
+
+        # If color quantization is requested, perform it; otherwise return pixel values
+        if do_color_quantize:
+            # Prepare clusters
+            if clusters is None:
+                raise ValueError("Clusters must be provided for color quantization.")
+            # Convert to numpy array if needed
+            clusters_np = np.array(clusters) if not isinstance(clusters, np.ndarray) else clusters
+
+            # Stack channel-first images (B, C, H, W) and transpose to (B, H, W, C) for color quantization
+            images_array = np.array(processed_images)
+            images_hwc = images_array.transpose(0, 2, 3, 1)
+            input_ids = color_quantize(images_hwc, clusters_np).reshape(
+                images_array.shape[0], images_array.shape[2], images_array.shape[3]
+            )
+
+            # flatten to (batch_size, height*width)
+            batch_size = input_ids.shape[0]
+            input_ids = input_ids.reshape(batch_size, -1)
+
+            # We need to convert back to a list to keep consistent behaviour across processors.
+            input_ids = list(input_ids)
+            return BatchFeature(data={"input_ids": input_ids}, tensor_type=return_tensors)
+
+        return BatchFeature(data={"pixel_values": processed_images}, tensor_type=return_tensors)
+
+    def to_dict(self):
+        output = super().to_dict()
+        if output.get("clusters") is not None and isinstance(output["clusters"], np.ndarray | torch.Tensor):
+            output["clusters"] = output["clusters"].tolist()
+
+        return output
+
+
+__all__ = ["ImageGPTImageProcessorPil"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/mobilenet_v1/image_processing_mobilenet_v1.py

@@ -0,0 +1,40 @@
+# Copyright 2022 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Image processor class for MobileNetV1."""
+
+from ...image_processing_backends import TorchvisionBackend
+from ...image_utils import (
+    IMAGENET_STANDARD_MEAN,
+    IMAGENET_STANDARD_STD,
+    PILImageResampling,
+)
+from ...utils import auto_docstring
+
+
+@auto_docstring(custom_intro="Constructs a MobileNetV1 image processor.")
+class MobileNetV1ImageProcessor(TorchvisionBackend):
+    resample = PILImageResampling.BILINEAR
+    image_mean = IMAGENET_STANDARD_MEAN
+    image_std = IMAGENET_STANDARD_STD
+    size = {"shortest_edge": 256}
+    default_to_square = False
+    crop_size = {"height": 224, "width": 224}
+    do_resize = True
+    do_center_crop = True
+    do_rescale = True
+    do_normalize = True
+    do_convert_rgb = None
+
+
+__all__ = ["MobileNetV1ImageProcessor"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/mobilenet_v1/image_processing_pil_mobilenet_v1.py

@@ -0,0 +1,40 @@
+# Copyright 2022 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Image processor class for MobileNetV1."""
+
+from ...image_processing_backends import PilBackend
+from ...image_utils import (
+    IMAGENET_STANDARD_MEAN,
+    IMAGENET_STANDARD_STD,
+    PILImageResampling,
+)
+from ...utils import auto_docstring
+
+
+@auto_docstring(custom_intro="Constructs a MobileNetV1 image processor.")
+class MobileNetV1ImageProcessorPil(PilBackend):
+    resample = PILImageResampling.BILINEAR
+    image_mean = IMAGENET_STANDARD_MEAN
+    image_std = IMAGENET_STANDARD_STD
+    size = {"shortest_edge": 256}
+    default_to_square = False
+    crop_size = {"height": 224, "width": 224}
+    do_resize = True
+    do_center_crop = True
+    do_rescale = True
+    do_normalize = True
+    do_convert_rgb = None
+
+
+__all__ = ["MobileNetV1ImageProcessorPil"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/slanet/__init__.py

@@ -0,0 +1,28 @@
+# Copyright 2026 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING
+
+from ...utils import _LazyModule
+from ...utils.import_utils import define_import_structure
+
+
+if TYPE_CHECKING:
+    from .configuration_slanet import *
+    from .modeling_slanet import *
+else:
+    import sys
+
+    _file = globals()["__file__"]
+    sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__)

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/slanet/configuration_slanet.py

@@ -0,0 +1,77 @@
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+#           This file was automatically generated from src/transformers/models/slanet/modular_slanet.py.
+#               Do NOT edit this file manually as any edits will be overwritten by the generation of
+#             the file from the modular. If any change should be done, please apply the change to the
+#                          modular_slanet.py file directly. One of our CI enforces this.
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# Copyright 2026 The PaddlePaddle Team and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from huggingface_hub.dataclasses import strict
+
+from ...backbone_utils import consolidate_backbone_kwargs_to_config
+from ...configuration_utils import PreTrainedConfig
+from ...utils import auto_docstring
+from ..auto import AutoConfig
+
+
+@auto_docstring(checkpoint="PaddlePaddle/SLANet_plus_safetensors")
+@strict
+class SLANetConfig(PreTrainedConfig):
+    r"""
+    post_conv_out_channels (`int`, *optional*, defaults to 96):
+        Number of output channels for the post-encoder convolution layer.
+    out_channels (`int`, *optional*, defaults to 50):
+        Vocabulary size for the table structure token prediction head, i.e., the number of distinct structure
+        tokens the model can predict.
+    hidden_size (`int`, *optional*, defaults to 256):
+        Dimensionality of the hidden states in the attention GRU cell and the structure/location prediction heads.
+    max_text_length (`int`, *optional*, defaults to 500):
+        Maximum number of autoregressive decoding steps (tokens) for the structure and location decoder.
+    csp_kernel_size (`int`, *optional*, defaults to 5):
+        The kernel size of the Cross Stage Partial (CSP) layer.
+    csp_num_blocks (`int`, *optional*, defaults to 1):
+        Number of blocks within the Cross Stage Partial (CSP) layer.
+    """
+
+    model_type = "slanet"
+
+    sub_configs = {"backbone_config": AutoConfig}
+    post_conv_out_channels: int = 96
+    out_channels: int = 50
+    hidden_size: int = 256
+    max_text_length: int = 500
+    backbone_config: dict | PreTrainedConfig | None = None
+
+    hidden_act: str = "hardswish"
+    csp_kernel_size: int = 5
+    csp_num_blocks: int = 1
+
+    def __post_init__(self, **kwargs):
+        self.backbone_config, kwargs = consolidate_backbone_kwargs_to_config(
+            backbone_config=self.backbone_config,
+            default_config_type="pp_lcnet",
+            default_config_kwargs={
+                "scale": 1,
+                "out_features": ["stage2", "stage3", "stage4", "stage5"],
+                "out_indices": [2, 3, 4, 5],
+                "divisor": 16,
+            },
+            **kwargs,
+        )
+        super().__post_init__(**kwargs)
+
+
+__all__ = ["SLANetConfig"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/slanet/modeling_slanet.py

@@ -0,0 +1,480 @@
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+#           This file was automatically generated from src/transformers/models/slanet/modular_slanet.py.
+#               Do NOT edit this file manually as any edits will be overwritten by the generation of
+#             the file from the modular. If any change should be done, please apply the change to the
+#                          modular_slanet.py file directly. One of our CI enforces this.
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# Copyright 2026 The PaddlePaddle Team and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import math
+from dataclasses import dataclass
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from ... import initialization as init
+from ...activations import ACT2CLS, ACT2FN
+from ...backbone_utils import filter_output_hidden_states, load_backbone
+from ...modeling_layers import GradientCheckpointingLayer
+from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithNoAttention
+from ...modeling_utils import PreTrainedModel
+from ...processing_utils import Unpack
+from ...utils import TransformersKwargs, auto_docstring, can_return_tuple
+from ...utils.generic import merge_with_config_defaults
+from ...utils.output_capturing import capture_outputs
+from .configuration_slanet import SLANetConfig
+
+
+class SLANetPreTrainedModel(PreTrainedModel):
+    config: SLANetConfig
+    base_model_prefix = "backbone"
+    main_input_name = "pixel_values"
+    input_modalities = ("image",)
+    supports_gradient_checkpointing = True
+    _keep_in_fp32_modules_strict = []
+
+    @torch.no_grad()
+    def _init_weights(self, module):
+        """Initialize the weights"""
+        super()._init_weights(module)
+
+        # Initialize GRUCell (replicates PyTorch default reset_parameters)
+        if isinstance(module, nn.GRUCell):
+            std = 1.0 / math.sqrt(module.hidden_size) if module.hidden_size > 0 else 0
+            init.uniform_(module.weight_ih, -std, std)
+            init.uniform_(module.weight_hh, -std, std)
+            if module.bias_ih is not None:
+                init.uniform_(module.bias_ih, -std, std)
+            if module.bias_hh is not None:
+                init.uniform_(module.bias_hh, -std, std)
+
+        # Initialize SLAHead layers
+        if isinstance(module, SLANetSLAHead):
+            std = 1.0 / math.sqrt(self.config.hidden_size * 1.0)
+            # Initialize structure_generator and loc_generator layers
+            for generator in (module.structure_generator,):
+                for layer in generator.children():
+                    if isinstance(layer, nn.Linear):
+                        init.uniform_(layer.weight, -std, std)
+                        if layer.bias is not None:
+                            init.uniform_(layer.bias, -std, std)
+
+
+@auto_docstring
+@dataclass
+class SLANetForTableRecognitionOutput(BaseModelOutputWithNoAttention):
+    r"""
+    head_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
+        Hidden-states of the SLANetSLAHead at each prediction step, varies up to max `self.config.max_text_length` states (depending on early exits).
+    head_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
+        Attentions of the SLANetSLAHead at each prediction step, varies up to max `self.config.max_text_length` attentions (depending on early exits).
+    """
+
+    head_hidden_states: torch.FloatTensor | None = None
+    head_attentions: torch.FloatTensor | None = None
+
+
+class SLANetAttentionGRUCell(nn.Module):
+    def __init__(self, input_size, hidden_size, num_embeddings):
+        super().__init__()
+
+        self.input_to_hidden = nn.Linear(input_size, hidden_size, bias=False)
+        self.hidden_to_hidden = nn.Linear(hidden_size, hidden_size)
+        self.score = nn.Linear(hidden_size, 1, bias=False)
+
+        self.rnn = nn.GRUCell(input_size + num_embeddings, hidden_size)
+
+    def forward(
+        self,
+        prev_hidden: torch.FloatTensor,
+        batch_hidden: torch.FloatTensor,
+        char_onehots: torch.FloatTensor,
+        **kwargs: Unpack[TransformersKwargs],
+    ):
+        batch_hidden_proj = self.input_to_hidden(batch_hidden)
+        prev_hidden_proj = self.hidden_to_hidden(prev_hidden).unsqueeze(1)
+
+        attention_scores = batch_hidden_proj + prev_hidden_proj
+        attention_scores = torch.tanh(attention_scores)
+        attention_scores = self.score(attention_scores)
+
+        attn_weights = F.softmax(attention_scores, dim=1, dtype=torch.float32).to(attention_scores.dtype)
+        attn_weights = attn_weights.transpose(1, 2)
+        context = torch.matmul(attn_weights, batch_hidden).squeeze(1)
+        concat_context = torch.cat([context, char_onehots], 1)
+        hidden_states = self.rnn(concat_context, prev_hidden)
+
+        return hidden_states, attn_weights
+
+
+class SLANetMLP(nn.Module):
+    def __init__(self, hidden_size, out_channels, activation=None):
+        super().__init__()
+        self.fc1 = nn.Linear(hidden_size, hidden_size)
+        self.fc2 = nn.Linear(hidden_size, out_channels)
+        self.act_fn = nn.Identity() if activation is None else ACT2CLS[activation]()
+
+    def forward(self, hidden_states):
+        hidden_states = self.fc1(hidden_states)
+        hidden_states = self.fc2(hidden_states)
+        hidden_states = self.act_fn(hidden_states)
+        return hidden_states
+
+
+class SLANetSLAHead(SLANetPreTrainedModel):
+    _can_record_outputs = {
+        "attentions": SLANetAttentionGRUCell,
+    }
+
+    def __init__(
+        self,
+        config: dict | None = None,
+        **kwargs,
+    ):
+        super().__init__(config)
+
+        self.structure_attention_cell = SLANetAttentionGRUCell(
+            config.post_conv_out_channels, config.hidden_size, config.out_channels
+        )
+        self.structure_generator = SLANetMLP(config.hidden_size, config.out_channels)
+
+        self.post_init()
+
+    @merge_with_config_defaults
+    @capture_outputs
+    @filter_output_hidden_states
+    def forward(
+        self,
+        hidden_states: torch.FloatTensor,
+        targets: torch.Tensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ):
+        features = torch.zeros(
+            (hidden_states.shape[0], self.config.hidden_size), dtype=torch.float32, device=hidden_states.device
+        )
+        predicted_chars = torch.zeros(size=[hidden_states.shape[0]], dtype=torch.long, device=hidden_states.device)
+
+        structure_preds_list = []
+        structure_ids_list = []
+        for _ in range(self.config.max_text_length + 1):
+            embedding_feature = F.one_hot(predicted_chars, self.config.out_channels).float()
+            features, _ = self.structure_attention_cell(features, hidden_states.float(), embedding_feature)
+            structure_step = self.structure_generator(features)
+            predicted_chars = structure_step.argmax(dim=1)
+
+            structure_preds_list.append(structure_step)
+            structure_ids_list.append(predicted_chars)
+            if torch.stack(structure_ids_list, dim=1).eq(self.config.out_channels - 1).any(-1).all():
+                break
+        structure_preds = F.softmax(torch.stack(structure_preds_list, dim=1), dim=-1, dtype=torch.float32).to(
+            hidden_states.dtype
+        )
+
+        return BaseModelOutput(last_hidden_state=structure_preds, hidden_states=structure_preds_list)
+
+
+class SLANetConvLayer(nn.Module):
+    def __init__(
+        self,
+        in_channels: int,
+        out_channels: int,
+        kernel_size: int = 3,
+        stride: int = 1,
+        bias: bool = False,
+        dilation: int | tuple[int, int] = 1,
+        groups: int = 1,
+        activation: str = "hardswish",
+    ):
+        super().__init__()
+        self.convolution = nn.Conv2d(
+            in_channels=in_channels,
+            out_channels=out_channels,
+            kernel_size=kernel_size,
+            stride=stride,
+            padding=kernel_size // 2,
+            bias=bias,
+            dilation=dilation,
+            groups=groups,
+        )
+        self.normalization = nn.BatchNorm2d(out_channels)
+        self.activation = ACT2FN[activation] if activation is not None else nn.Identity()
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        hidden_states = self.convolution(hidden_states)
+        hidden_states = self.normalization(hidden_states)
+        hidden_states = self.activation(hidden_states)
+        return hidden_states
+
+
+class SLANetDepthwiseSeparableConvLayer(GradientCheckpointingLayer):
+    """
+    Depthwise Separable Convolution Layer: Depthwise Conv -> Pointwise Conv
+    Core component of lightweight models (e.g., MobileNet, PP-LCNet) that significantly reduces
+    the number of parameters and computational cost.
+    """
+
+    def __init__(
+        self,
+        in_channels,
+        out_channels,
+        stride,
+        kernel_size,
+        config,
+    ):
+        super().__init__()
+        self.depthwise_convolution = SLANetConvLayer(
+            in_channels=in_channels,
+            out_channels=in_channels,
+            kernel_size=kernel_size,
+            stride=stride,
+            groups=in_channels,
+            activation=config.hidden_act,
+        )
+        self.squeeze_excitation_module = nn.Identity()
+        self.pointwise_convolution = SLANetConvLayer(
+            in_channels=in_channels,
+            kernel_size=1,
+            out_channels=out_channels,
+            stride=1,
+            activation=config.hidden_act,
+        )
+
+    def forward(self, hidden_state):
+        hidden_state = self.depthwise_convolution(hidden_state)
+        hidden_state = self.squeeze_excitation_module(hidden_state)
+        hidden_state = self.pointwise_convolution(hidden_state)
+
+        return hidden_state
+
+
+class SLANetBottleneck(nn.Module):
+    def __init__(
+        self,
+        in_channels,
+        out_channels,
+        kernel_size,
+        activation,
+        config,
+    ):
+        super().__init__()
+        self.conv1 = SLANetConvLayer(
+            in_channels=in_channels, out_channels=out_channels, kernel_size=1, activation=activation
+        )
+        self.conv2 = SLANetDepthwiseSeparableConvLayer(
+            in_channels=out_channels,
+            out_channels=out_channels,
+            kernel_size=kernel_size,
+            stride=1,
+            config=config,
+        )
+
+    def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor:
+        hidden_states = self.conv1(hidden_states)
+        hidden_states = self.conv2(hidden_states)
+
+        return hidden_states
+
+
+class SLANetCSPLayer(nn.Module):
+    """
+    Cross Stage Partial (CSP) network layer. Similar in structure to DFineCSPRepLayer, but with a different forward computation.
+    """
+
+    def __init__(
+        self,
+        config,
+        in_channels,
+        out_channels,
+        kernel_size=3,
+        expansion=0.5,
+        num_blocks=1,
+        activation="hardswish",
+    ):
+        super().__init__()
+        hidden_channels = int(out_channels * expansion)
+        self.conv1 = SLANetConvLayer(in_channels, hidden_channels, 1, activation=activation)
+        self.conv2 = SLANetConvLayer(in_channels, hidden_channels, 1, activation=activation)
+        self.conv3 = SLANetConvLayer(2 * hidden_channels, out_channels, 1, activation=activation)
+        self.bottlenecks = nn.ModuleList(
+            [
+                SLANetBottleneck(hidden_channels, hidden_channels, kernel_size, activation, config)
+                for _ in range(num_blocks)
+            ]
+        )
+
+    def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor:
+        residual = self.conv1(hidden_states)
+
+        hidden_states = self.conv2(hidden_states)
+        for bottleneck in self.bottlenecks:
+            hidden_states = bottleneck(hidden_states)
+
+        hidden_states = torch.cat((hidden_states, residual), dim=1)
+        hidden_states = self.conv3(hidden_states)
+
+        return hidden_states
+
+
+class SLANetCSPPAN(nn.Module):
+    """
+    CSP-PAN: Path Aggregation Network with CSP layers
+    """
+
+    def __init__(
+        self,
+        config,
+        in_channel_list,
+    ):
+        super().__init__()
+        out_channels = config.post_conv_out_channels
+        activation = config.hidden_act
+        kernel_size = config.csp_kernel_size
+        csp_num_blocks = config.csp_num_blocks
+
+        self.channel_projector = nn.ModuleList(
+            [
+                SLANetConvLayer(
+                    in_channels=in_channel_list[i], out_channels=out_channels, kernel_size=1, activation=activation
+                )
+                for i in range(len(in_channel_list))
+            ]
+        )
+
+        # build top-down blocks
+        self.upsample = nn.Upsample(scale_factor=2, mode="nearest")
+        self.top_down_blocks = nn.ModuleList(
+            [
+                SLANetCSPLayer(
+                    config,
+                    out_channels * 2,
+                    out_channels,
+                    kernel_size=kernel_size,
+                    num_blocks=csp_num_blocks,
+                    activation=activation,
+                )
+                for _ in range(len(in_channel_list) - 1, 0, -1)
+            ]
+        )
+
+        # build bottom-up blocks
+        self.downsamples = nn.ModuleList(
+            [
+                SLANetDepthwiseSeparableConvLayer(
+                    out_channels,
+                    out_channels,
+                    kernel_size=kernel_size,
+                    stride=2,
+                    config=config,
+                )
+                for _ in range(len(in_channel_list) - 1)
+            ]
+        )
+        self.bottom_up_blocks = nn.ModuleList(
+            [
+                SLANetCSPLayer(
+                    config,
+                    out_channels * 2,
+                    out_channels,
+                    kernel_size=kernel_size,
+                    num_blocks=csp_num_blocks,
+                    activation=activation,
+                )
+                for _ in range(len(in_channel_list) - 1)
+            ]
+        )
+
+    def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor:
+        projected_features = []
+        for idx in range(len(self.channel_projector)):
+            projected_features.append(self.channel_projector[idx](hidden_states[idx]))
+
+        top_down_features = [projected_features[-1]]
+        for top_down_block, low_level_feature in zip(self.top_down_blocks, reversed(projected_features[:-1])):
+            high_level_feature = top_down_features[-1]
+            upsampled_feature = F.interpolate(
+                high_level_feature,
+                size=low_level_feature.shape[-2:],
+                mode="nearest",
+            )
+            fused_feature = top_down_block(torch.cat([upsampled_feature, low_level_feature], dim=1))
+            top_down_features.append(fused_feature)
+
+        pyramid_features = list(reversed(top_down_features))
+        output_feature = pyramid_features[0]
+        for downsample_layer, bottom_up_block, high_level_feature in zip(
+            self.downsamples, self.bottom_up_blocks, pyramid_features[1:]
+        ):
+            downsampled_feature = downsample_layer(output_feature)
+            output_feature = bottom_up_block(torch.cat([downsampled_feature, high_level_feature], dim=1))
+
+        hidden_states = output_feature.flatten(2).transpose(1, 2)
+        return hidden_states
+
+
+class SLANetBackbone(SLANetPreTrainedModel):
+    def __init__(self, config: SLANetConfig):
+        super().__init__(config)
+        self.vision_backbone = load_backbone(config)
+        self.post_csp_pan = SLANetCSPPAN(config, self.vision_backbone.num_features[2:])
+
+        self.post_init()
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self, hidden_states: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]
+    ) -> tuple[torch.FloatTensor] | BaseModelOutputWithNoAttention:
+        outputs = self.vision_backbone(hidden_states, **kwargs)
+        hidden_states = self.post_csp_pan(outputs.feature_maps)
+        return BaseModelOutputWithNoAttention(
+            last_hidden_state=hidden_states,
+            hidden_states=outputs.hidden_states,
+        )
+
+
+@auto_docstring(
+    custom_intro="""
+    SLANet Table Recognition model for table recognition tasks. Wraps the core SLANetPreTrainedModel
+    and returns outputs compatible with the Transformers table recognition API.
+    """
+)
+class SLANetForTableRecognition(SLANetPreTrainedModel):
+    _keys_to_ignore_on_load_missing = ["num_batches_tracked"]
+
+    def __init__(self, config: SLANetConfig):
+        super().__init__(config)
+        self.backbone = SLANetBackbone(config=config)
+        self.head = SLANetSLAHead(config=config)
+        self.post_init()
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]
+    ) -> tuple[torch.FloatTensor] | SLANetForTableRecognitionOutput:
+        outputs = self.backbone(pixel_values, **kwargs)
+        head_outputs = self.head(outputs.last_hidden_state, **kwargs)
+        # Key difference: no attentions in its vision model
+        return SLANetForTableRecognitionOutput(
+            last_hidden_state=head_outputs.last_hidden_state,
+            hidden_states=outputs.hidden_states,
+            head_hidden_states=head_outputs.hidden_states,
+            head_attentions=head_outputs.attentions,
+        )
+
+
+__all__ = ["SLANetForTableRecognition", "SLANetPreTrainedModel", "SLANetSLAHead", "SLANetBackbone"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/slanet/modular_slanet.py

@@ -0,0 +1,372 @@
+# Copyright 2026 The PaddlePaddle Team and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import math
+from dataclasses import dataclass
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from huggingface_hub.dataclasses import strict
+
+from ... import initialization as init
+from ...backbone_utils import consolidate_backbone_kwargs_to_config, load_backbone
+from ...configuration_utils import PreTrainedConfig
+from ...modeling_outputs import BaseModelOutputWithNoAttention
+from ...modeling_utils import PreTrainedModel
+from ...processing_utils import Unpack
+from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging
+from ..auto import AutoConfig
+from ..pp_lcnet.modeling_pp_lcnet import PPLCNetConvLayer, PPLCNetDepthwiseSeparableConvLayer
+from ..slanext.configuration_slanext import SLANeXtConfig
+from ..slanext.modeling_slanext import (
+    SLANeXtForTableRecognition,
+    SLANeXtPreTrainedModel,
+    SLANeXtSLAHead,
+)
+
+
+logger = logging.get_logger(__name__)
+
+
+@auto_docstring(checkpoint="PaddlePaddle/SLANet_plus_safetensors")
+@strict
+class SLANetConfig(SLANeXtConfig):
+    r"""
+    post_conv_out_channels (`int`, *optional*, defaults to 96):
+        Number of output channels for the post-encoder convolution layer.
+    out_channels (`int`, *optional*, defaults to 50):
+        Vocabulary size for the table structure token prediction head, i.e., the number of distinct structure
+        tokens the model can predict.
+    hidden_size (`int`, *optional*, defaults to 256):
+        Dimensionality of the hidden states in the attention GRU cell and the structure/location prediction heads.
+    max_text_length (`int`, *optional*, defaults to 500):
+        Maximum number of autoregressive decoding steps (tokens) for the structure and location decoder.
+    csp_kernel_size (`int`, *optional*, defaults to 5):
+        The kernel size of the Cross Stage Partial (CSP) layer.
+    csp_num_blocks (`int`, *optional*, defaults to 1):
+        Number of blocks within the Cross Stage Partial (CSP) layer.
+    """
+
+    sub_configs = {"backbone_config": AutoConfig}
+
+    vision_config = AttributeError()
+    backbone_config: dict | PreTrainedConfig | None = None
+
+    post_conv_in_channels = AttributeError()
+    post_conv_out_channels: int = 96
+    hidden_size: int = 256
+
+    hidden_act: str = "hardswish"
+    csp_kernel_size: int = 5
+    csp_num_blocks: int = 1
+
+    def __post_init__(self, **kwargs):
+        self.backbone_config, kwargs = consolidate_backbone_kwargs_to_config(
+            backbone_config=self.backbone_config,
+            default_config_type="pp_lcnet",
+            default_config_kwargs={
+                "scale": 1,
+                "out_features": ["stage2", "stage3", "stage4", "stage5"],
+                "out_indices": [2, 3, 4, 5],
+                "divisor": 16,
+            },
+            **kwargs,
+        )
+        PreTrainedConfig.__post_init__(**kwargs)
+
+
+class SLANetPreTrainedModel(SLANeXtPreTrainedModel):
+    _keep_in_fp32_modules_strict = []
+
+    @torch.no_grad()
+    def _init_weights(self, module):
+        """Initialize the weights"""
+        PreTrainedModel._init_weights(module)
+
+        # Initialize GRUCell (replicates PyTorch default reset_parameters)
+        if isinstance(module, nn.GRUCell):
+            std = 1.0 / math.sqrt(module.hidden_size) if module.hidden_size > 0 else 0
+            init.uniform_(module.weight_ih, -std, std)
+            init.uniform_(module.weight_hh, -std, std)
+            if module.bias_ih is not None:
+                init.uniform_(module.bias_ih, -std, std)
+            if module.bias_hh is not None:
+                init.uniform_(module.bias_hh, -std, std)
+
+        # Initialize SLAHead layers
+        if isinstance(module, SLANetSLAHead):
+            std = 1.0 / math.sqrt(self.config.hidden_size * 1.0)
+            # Initialize structure_generator and loc_generator layers
+            for generator in (module.structure_generator,):
+                for layer in generator.children():
+                    if isinstance(layer, nn.Linear):
+                        init.uniform_(layer.weight, -std, std)
+                        if layer.bias is not None:
+                            init.uniform_(layer.bias, -std, std)
+
+
+@auto_docstring
+@dataclass
+class SLANetForTableRecognitionOutput(BaseModelOutputWithNoAttention):
+    r"""
+    head_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
+        Hidden-states of the SLANetSLAHead at each prediction step, varies up to max `self.config.max_text_length` states (depending on early exits).
+    head_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
+        Attentions of the SLANetSLAHead at each prediction step, varies up to max `self.config.max_text_length` attentions (depending on early exits).
+    """
+
+    head_hidden_states: torch.FloatTensor | None = None
+    head_attentions: torch.FloatTensor | None = None
+
+
+class SLANetSLAHead(SLANeXtSLAHead):
+    pass
+
+
+class SLANetConvLayer(PPLCNetConvLayer):
+    pass
+
+
+class SLANetDepthwiseSeparableConvLayer(PPLCNetDepthwiseSeparableConvLayer):
+    """
+    Depthwise Separable Convolution Layer: Depthwise Conv -> Pointwise Conv
+    Core component of lightweight models (e.g., MobileNet, PP-LCNet) that significantly reduces
+    the number of parameters and computational cost.
+    """
+
+    def __init__(
+        self,
+        in_channels,
+        out_channels,
+        stride,
+        kernel_size,
+        config,
+    ):
+        super().__init__()
+        self.squeeze_excitation_module = nn.Identity()
+
+
+class SLANetBottleneck(nn.Module):
+    def __init__(
+        self,
+        in_channels,
+        out_channels,
+        kernel_size,
+        activation,
+        config,
+    ):
+        super().__init__()
+        self.conv1 = SLANetConvLayer(
+            in_channels=in_channels, out_channels=out_channels, kernel_size=1, activation=activation
+        )
+        self.conv2 = SLANetDepthwiseSeparableConvLayer(
+            in_channels=out_channels,
+            out_channels=out_channels,
+            kernel_size=kernel_size,
+            stride=1,
+            config=config,
+        )
+
+    def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor:
+        hidden_states = self.conv1(hidden_states)
+        hidden_states = self.conv2(hidden_states)
+
+        return hidden_states
+
+
+class SLANetCSPLayer(nn.Module):
+    """
+    Cross Stage Partial (CSP) network layer. Similar in structure to DFineCSPRepLayer, but with a different forward computation.
+    """
+
+    def __init__(
+        self,
+        config,
+        in_channels,
+        out_channels,
+        kernel_size=3,
+        expansion=0.5,
+        num_blocks=1,
+        activation="hardswish",
+    ):
+        super().__init__()
+        hidden_channels = int(out_channels * expansion)
+        self.conv1 = SLANetConvLayer(in_channels, hidden_channels, 1, activation=activation)
+        self.conv2 = SLANetConvLayer(in_channels, hidden_channels, 1, activation=activation)
+        self.conv3 = SLANetConvLayer(2 * hidden_channels, out_channels, 1, activation=activation)
+        self.bottlenecks = nn.ModuleList(
+            [
+                SLANetBottleneck(hidden_channels, hidden_channels, kernel_size, activation, config)
+                for _ in range(num_blocks)
+            ]
+        )
+
+    def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor:
+        residual = self.conv1(hidden_states)
+
+        hidden_states = self.conv2(hidden_states)
+        for bottleneck in self.bottlenecks:
+            hidden_states = bottleneck(hidden_states)
+
+        hidden_states = torch.cat((hidden_states, residual), dim=1)
+        hidden_states = self.conv3(hidden_states)
+
+        return hidden_states
+
+
+class SLANetCSPPAN(nn.Module):
+    """
+    CSP-PAN: Path Aggregation Network with CSP layers
+    """
+
+    def __init__(
+        self,
+        config,
+        in_channel_list,
+    ):
+        super().__init__()
+        out_channels = config.post_conv_out_channels
+        activation = config.hidden_act
+        kernel_size = config.csp_kernel_size
+        csp_num_blocks = config.csp_num_blocks
+
+        self.channel_projector = nn.ModuleList(
+            [
+                SLANetConvLayer(
+                    in_channels=in_channel_list[i], out_channels=out_channels, kernel_size=1, activation=activation
+                )
+                for i in range(len(in_channel_list))
+            ]
+        )
+
+        # build top-down blocks
+        self.upsample = nn.Upsample(scale_factor=2, mode="nearest")
+        self.top_down_blocks = nn.ModuleList(
+            [
+                SLANetCSPLayer(
+                    config,
+                    out_channels * 2,
+                    out_channels,
+                    kernel_size=kernel_size,
+                    num_blocks=csp_num_blocks,
+                    activation=activation,
+                )
+                for _ in range(len(in_channel_list) - 1, 0, -1)
+            ]
+        )
+
+        # build bottom-up blocks
+        self.downsamples = nn.ModuleList(
+            [
+                SLANetDepthwiseSeparableConvLayer(
+                    out_channels,
+                    out_channels,
+                    kernel_size=kernel_size,
+                    stride=2,
+                    config=config,
+                )
+                for _ in range(len(in_channel_list) - 1)
+            ]
+        )
+        self.bottom_up_blocks = nn.ModuleList(
+            [
+                SLANetCSPLayer(
+                    config,
+                    out_channels * 2,
+                    out_channels,
+                    kernel_size=kernel_size,
+                    num_blocks=csp_num_blocks,
+                    activation=activation,
+                )
+                for _ in range(len(in_channel_list) - 1)
+            ]
+        )
+
+    def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor:
+        projected_features = []
+        for idx in range(len(self.channel_projector)):
+            projected_features.append(self.channel_projector[idx](hidden_states[idx]))
+
+        top_down_features = [projected_features[-1]]
+        for top_down_block, low_level_feature in zip(self.top_down_blocks, reversed(projected_features[:-1])):
+            high_level_feature = top_down_features[-1]
+            upsampled_feature = F.interpolate(
+                high_level_feature,
+                size=low_level_feature.shape[-2:],
+                mode="nearest",
+            )
+            fused_feature = top_down_block(torch.cat([upsampled_feature, low_level_feature], dim=1))
+            top_down_features.append(fused_feature)
+
+        pyramid_features = list(reversed(top_down_features))
+        output_feature = pyramid_features[0]
+        for downsample_layer, bottom_up_block, high_level_feature in zip(
+            self.downsamples, self.bottom_up_blocks, pyramid_features[1:]
+        ):
+            downsampled_feature = downsample_layer(output_feature)
+            output_feature = bottom_up_block(torch.cat([downsampled_feature, high_level_feature], dim=1))
+
+        hidden_states = output_feature.flatten(2).transpose(1, 2)
+        return hidden_states
+
+
+class SLANetBackbone(SLANetPreTrainedModel):
+    def __init__(self, config: SLANetConfig):
+        super().__init__(config)
+        self.vision_backbone = load_backbone(config)
+        self.post_csp_pan = SLANetCSPPAN(config, self.vision_backbone.num_features[2:])
+
+        self.post_init()
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self, hidden_states: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]
+    ) -> tuple[torch.FloatTensor] | BaseModelOutputWithNoAttention:
+        outputs = self.vision_backbone(hidden_states, **kwargs)
+        hidden_states = self.post_csp_pan(outputs.feature_maps)
+        return BaseModelOutputWithNoAttention(
+            last_hidden_state=hidden_states,
+            hidden_states=outputs.hidden_states,
+        )
+
+
+@auto_docstring(
+    custom_intro="""
+    SLANet Table Recognition model for table recognition tasks. Wraps the core SLANetPreTrainedModel
+    and returns outputs compatible with the Transformers table recognition API.
+    """
+)
+class SLANetForTableRecognition(SLANeXtForTableRecognition):
+    _keys_to_ignore_on_load_missing = ["num_batches_tracked"]
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]
+    ) -> tuple[torch.FloatTensor] | SLANetForTableRecognitionOutput:
+        outputs = self.backbone(pixel_values, **kwargs)
+        head_outputs = self.head(outputs.last_hidden_state, **kwargs)
+        # Key difference: no attentions in its vision model
+        return SLANetForTableRecognitionOutput(
+            last_hidden_state=head_outputs.last_hidden_state,
+            hidden_states=outputs.hidden_states,
+            head_hidden_states=head_outputs.hidden_states,
+            head_attentions=head_outputs.attentions,
+        )
+
+
+__all__ = ["SLANetConfig", "SLANetForTableRecognition", "SLANetPreTrainedModel", "SLANetSLAHead", "SLANetBackbone"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/videomt/configuration_videomt.py

@@ -0,0 +1,101 @@
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+#           This file was automatically generated from src/transformers/models/videomt/modular_videomt.py.
+#               Do NOT edit this file manually as any edits will be overwritten by the generation of
+#             the file from the modular. If any change should be done, please apply the change to the
+#                          modular_videomt.py file directly. One of our CI enforces this.
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# Copyright 2026 the HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from huggingface_hub.dataclasses import strict
+
+from ...configuration_utils import PreTrainedConfig
+from ...utils import auto_docstring
+
+
+@auto_docstring(checkpoint="tue-mps/videomt-dinov2-small-ytvis2019")
+@strict
+class VideomtConfig(PreTrainedConfig):
+    r"""
+    layerscale_value (`float`, *optional*, defaults to 1.0):
+        Initial value for the LayerScale parameter.
+    num_upscale_blocks (`int`, *optional*, defaults to 2):
+        Number of upsampling blocks used in the decoder or segmentation head.
+    use_swiglu_ffn (`bool`, *optional*, defaults to `False`):
+        Whether to use the SwiGLU feedforward neural network.
+    num_blocks (`int`, *optional*, defaults to 4):
+        Number of feature blocks or stages in the architecture.
+    no_object_weight (`float`, *optional*, defaults to 0.1):
+        Loss weight for the 'no object' class in panoptic/instance segmentation.
+    class_weight (`float`, *optional*, defaults to 2.0):
+        Loss weight for classification targets.
+    mask_weight (`float`, *optional*, defaults to 5.0):
+        Loss weight for mask prediction.
+    train_num_points (`int`, *optional*, defaults to 12544):
+        Number of points to sample for mask loss computation during training.
+    oversample_ratio (`float`, *optional*, defaults to 3.0):
+        Oversampling ratio used in point sampling for mask training.
+    importance_sample_ratio (`float`, *optional*, defaults to 0.75):
+        Ratio of points to sample based on importance during training.
+    num_queries (`int`, *optional*, defaults to 200):
+        Number of object queries in the Transformer.
+    num_register_tokens (`int`, *optional*, defaults to 4):
+        Number of learnable register tokens added to the transformer input.
+
+    Example:
+
+    ```python
+    >>> from transformers import VideomtConfig, VideomtForUniversalSegmentation
+
+    >>> # Initialize configuration
+    >>> config = VideomtConfig()
+
+    >>> # Initialize model
+    >>> model = VideomtForUniversalSegmentation(config)
+
+    >>> # Access config
+    >>> config = model.config
+    ```"""
+
+    model_type = "videomt"
+
+    hidden_size: int = 1024
+    num_hidden_layers: int = 24
+    num_attention_heads: int = 16
+    hidden_act: str = "gelu"
+    hidden_dropout_prob: float | int = 0.0
+    initializer_range: float = 0.02
+    layer_norm_eps: float = 1e-6
+    image_size: int | list[int] | tuple[int, int] = 640
+    patch_size: int | list[int] | tuple[int, int] = 16
+    num_channels: int = 3
+    mlp_ratio: int = 4
+    layerscale_value: float = 1.0
+    drop_path_rate: float | int = 0.0
+    num_upscale_blocks: int = 2
+    attention_dropout: float | int = 0.0
+    use_swiglu_ffn: bool = False
+    num_blocks: int = 4
+    no_object_weight: float = 0.1
+    class_weight: float = 2.0
+    mask_weight: float = 5.0
+    dice_weight: float = 5.0
+    train_num_points: int = 12544
+    oversample_ratio: float = 3.0
+    importance_sample_ratio: float = 0.75
+    num_queries: int = 200
+    num_register_tokens: int = 4
+
+
+__all__ = ["VideomtConfig"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/videomt/modular_videomt.py

@@ -0,0 +1,266 @@
+# Copyright 2026 the HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from dataclasses import dataclass
+
+import torch
+from huggingface_hub.dataclasses import strict
+from torch import nn
+
+from ...file_utils import ModelOutput
+from ...processing_utils import Unpack
+from ...utils import TransformersKwargs, auto_docstring
+from ..eomt.configuration_eomt import EomtConfig
+from ..eomt.modeling_eomt import (
+    EomtEmbeddings,
+    EomtForUniversalSegmentation,
+    EomtLayer,
+    EomtLayerNorm2d,
+    EomtLayerScale,
+    EomtMLP,
+    EomtPatchEmbeddings,
+    EomtPreTrainedModel,
+    EomtScaleBlock,
+    EomtScaleLayer,
+    EomtSwiGLUFFN,
+)
+
+
+@auto_docstring(checkpoint="tue-mps/videomt-dinov2-small-ytvis2019")
+@strict
+class VideomtConfig(EomtConfig):
+    model_type = "videomt"
+
+
+class VideomtPatchEmbeddings(EomtPatchEmbeddings):
+    def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
+        num_channels = pixel_values.shape[1]
+        if num_channels != self.num_channels:
+            raise ValueError(
+                "Make sure that the channel dimension of the pixel values match with the one set in the configuration."
+                f" Expected {self.num_channels} but got {num_channels}."
+            )
+
+        pixel_values = pixel_values.to(dtype=self.projection.weight.dtype)
+        embeddings = self.projection(pixel_values).flatten(2).transpose(1, 2)
+        return embeddings
+
+
+class VideomtEmbeddings(EomtEmbeddings):
+    def __init__(self, config: VideomtConfig):
+        super().__init__(config)
+        self.patch_embeddings = VideomtPatchEmbeddings(config)
+        self.mask_token = nn.Parameter(torch.zeros(1, 1, config.hidden_size))
+
+    def forward(self, pixel_values: torch.Tensor, bool_masked_pos: torch.Tensor | None = None) -> torch.Tensor:
+        if pixel_values.ndim == 5:
+            batch_size, num_frames, num_channels, height, width = pixel_values.shape
+            pixel_values = pixel_values.reshape(batch_size * num_frames, num_channels, height, width)
+
+            if bool_masked_pos is not None:
+                bool_masked_pos = bool_masked_pos.reshape(batch_size * num_frames, -1)
+        elif bool_masked_pos is not None and bool_masked_pos.ndim > 2:
+            bool_masked_pos = bool_masked_pos.reshape(bool_masked_pos.shape[0], -1)
+
+        batch_size = pixel_values.shape[0]
+        embeddings = self.patch_embeddings(pixel_values)
+
+        if bool_masked_pos is not None:
+            mask = bool_masked_pos.to(device=embeddings.device, dtype=torch.bool).unsqueeze(-1)
+            embeddings = torch.where(mask, self.mask_token, embeddings)
+
+        cls_tokens = self.cls_token.expand(batch_size, -1, -1)
+        register_tokens = self.register_tokens.expand(batch_size, -1, -1)
+
+        embeddings = embeddings + self.position_embeddings(self.position_ids)
+        embeddings = torch.cat([cls_tokens, register_tokens, embeddings], dim=1)
+        embeddings = self.dropout(embeddings)
+        return embeddings
+
+
+class VideomtMLP(EomtMLP):
+    pass
+
+
+class VideomtGatedMLP(EomtSwiGLUFFN):
+    pass
+
+
+class VideomtLayer(EomtLayer):
+    pass
+
+
+class VideomtLayerScale(EomtLayerScale):
+    pass
+
+
+@auto_docstring(
+    custom_intro="""
+    Class for outputs of [`VideomtForUniversalSegmentationOutput`].
+
+    This output can be directly passed to [`~VideomtVideoProcessor.post_process_semantic_segmentation`] or
+    [`~VideomtVideoProcessor.post_process_instance_segmentation`] or
+    [`~VideomtVideoProcessor.post_process_panoptic_segmentation`] to compute final segmentation maps. Please, see
+    [`~VideomtVideoProcessor`] for details regarding usage.
+    """
+)
+@dataclass
+class VideomtForUniversalSegmentationOutput(ModelOutput):
+    r"""
+    loss (`torch.Tensor`, *optional*):
+        The computed loss, returned when labels are present.
+    class_queries_logits (`torch.FloatTensor`):
+        A tensor of shape `(batch_size, num_queries, num_labels + 1)` representing the proposed classes for each
+        query. Note the `+ 1` is needed because we incorporate the null class.
+    masks_queries_logits (`torch.FloatTensor`):
+        A tensor of shape `(batch_size, num_queries, height, width)` representing the proposed masks for each
+        query.
+    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
+        Last hidden states (final feature map) of the last layer.
+    hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
+        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
+        shape `(batch_size, sequence_length, hidden_size)`. Hidden-states all layers of the model.
+    attentions (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
+        Tuple of `tuple(torch.FloatTensor)` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+        sequence_length)`. Self and Cross Attentions weights from transformer decoder.
+    """
+
+    loss: torch.FloatTensor | None = None
+    class_queries_logits: torch.FloatTensor | None = None
+    masks_queries_logits: torch.FloatTensor | None = None
+    last_hidden_state: torch.FloatTensor | None = None
+    hidden_states: tuple[torch.FloatTensor] | None = None
+    attentions: tuple[torch.FloatTensor] | None = None
+
+
+class VideomtPreTrainedModel(EomtPreTrainedModel):
+    main_input_name = "pixel_values_videos"
+    input_modalities = ("video",)
+
+    @torch.no_grad()
+    def _init_weights(self, module: nn.Module) -> None:
+        super()._init_weights(module)
+        if isinstance(module, VideomtEmbeddings):
+            nn.init.zeros_(module.mask_token)
+
+
+class VideomtLayerNorm2d(EomtLayerNorm2d):
+    pass
+
+
+class VideomtScaleLayer(EomtScaleLayer):
+    pass
+
+
+class VideomtScaleBlock(EomtScaleBlock):
+    pass
+
+
+class VideomtForUniversalSegmentation(EomtForUniversalSegmentation):
+    main_input_name = "pixel_values_videos"
+
+    def __init__(self, config: VideomtConfig):
+        super().__init__(config)
+        self.query_updater = nn.Linear(config.hidden_size, config.hidden_size)
+
+    def _disable_attention_mask(attn_mask, prob, num_query_tokens, encoder_start_tokens, device):
+        raise AttributeError("Not needed for Videomt")
+
+    def forward(
+        self,
+        pixel_values_videos: torch.Tensor | None = None,
+        mask_labels: list[torch.Tensor] | None = None,
+        class_labels: list[torch.Tensor] | None = None,
+        patch_offsets: list[torch.Tensor] | None = None,  # Unused, kept for modular compatibility.
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> VideomtForUniversalSegmentationOutput:
+        r"""
+        pixel_values_videos (`torch.Tensor`, *optional*):
+            Video inputs of shape `(batch_size, num_frames, num_channels, height, width)`.
+        mask_labels (`list[torch.Tensor]`, *optional*):
+            Not supported for 5D video inputs.
+        class_labels (`list[torch.LongTensor]`, *optional*):
+            Not supported for 5D video inputs.
+        patch_offsets (`list[torch.Tensor]`, *optional*):
+            Unused for video inputs and only kept for modular compatibility.
+        """
+        if "pixel_values" in kwargs:
+            raise ValueError("Use `pixel_values_videos` with `VideomtForUniversalSegmentation`.")
+
+        if pixel_values_videos is None:
+            raise ValueError("You have to specify pixel_values_videos")
+
+        if pixel_values_videos.ndim != 5:
+            raise ValueError(
+                "VideomtForUniversalSegmentation only supports 5D video inputs of shape "
+                "(batch_size, num_frames, channels, height, width)."
+            )
+
+        if mask_labels is not None or class_labels is not None:
+            raise ValueError(
+                "Training with 5D video inputs is not supported in `VideomtForUniversalSegmentation`. "
+                "Flatten frames and use `EomtForUniversalSegmentation` instead."
+            )
+
+        batch_size, num_frames, num_channels, height, width = pixel_values_videos.shape
+        flat_pixel_values = pixel_values_videos.reshape(batch_size * num_frames, num_channels, height, width)
+
+        hidden_states = self.embeddings(flat_pixel_values)
+        query_start_idx = self.num_hidden_layers - self.config.num_blocks
+
+        for layer_module in self.layers[:query_start_idx]:
+            hidden_states = layer_module(hidden_states)
+
+        hidden_states = hidden_states.view(batch_size, num_frames, hidden_states.shape[1], hidden_states.shape[2])
+
+        all_masks_queries_logits = []
+        all_class_queries_logits = []
+        all_last_hidden_states = []
+        propagated_query = None
+
+        for frame_idx in range(num_frames):
+            frame_hidden_states = hidden_states[:, frame_idx]
+
+            if propagated_query is None:
+                query_tokens = self.query.weight[None, :, :].expand(batch_size, -1, -1).to(frame_hidden_states.device)
+            else:
+                query_tokens = self.query_updater(propagated_query).to(frame_hidden_states.device) + self.query.weight[
+                    None, :, :
+                ].to(frame_hidden_states.device)
+            frame_hidden_states = torch.cat((query_tokens, frame_hidden_states), dim=1)
+
+            for layer_module in self.layers[query_start_idx:]:
+                frame_hidden_states = layer_module(frame_hidden_states)
+
+            sequence_output = self.layernorm(frame_hidden_states)
+            masks_queries_logits, class_queries_logits = self.predict(sequence_output)
+
+            all_masks_queries_logits.append(masks_queries_logits)
+            all_class_queries_logits.append(class_queries_logits)
+            all_last_hidden_states.append(sequence_output)
+            propagated_query = frame_hidden_states[:, : self.config.num_queries, :]
+
+        return VideomtForUniversalSegmentationOutput(
+            loss=None,  # Training not supported yet
+            masks_queries_logits=torch.cat(all_masks_queries_logits, dim=0),
+            class_queries_logits=torch.cat(all_class_queries_logits, dim=0),
+            last_hidden_state=torch.cat(all_last_hidden_states, dim=0),
+        )
+
+
+__all__ = [
+    "VideomtConfig",
+    "VideomtPreTrainedModel",
+    "VideomtForUniversalSegmentation",
+]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/videomt/video_processing_videomt.py

@@ -0,0 +1,364 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Video processor class for Videomt."""
+
+from ...image_utils import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, PILImageResampling
+from ...utils import is_torch_available, requires_backends
+from ...video_processing_utils import BaseVideoProcessor
+
+
+if is_torch_available():
+    import torch
+    import torch.nn.functional as F
+
+
+def check_segment_validity(
+    mask_labels: "torch.Tensor",
+    mask_probs: "torch.Tensor",
+    query_idx: int,
+    mask_threshold: float = 0.5,
+    overlap_mask_area_threshold: float = 0.8,
+) -> tuple[bool, "torch.Tensor"]:
+    """
+    Checks whether a predicted query produces a valid panoptic segment.
+
+    Args:
+        mask_labels (`torch.Tensor`):
+            Tensor of shape `(height, width)` containing the winning query index for each pixel.
+        mask_probs (`torch.Tensor`):
+            Tensor of shape `(num_queries, height, width)` containing per-query mask probabilities.
+        query_idx (`int`):
+            Index of the query to validate.
+        mask_threshold (`float`, *optional*, defaults to 0.5):
+            Threshold used to binarize the query mask probabilities.
+        overlap_mask_area_threshold (`float`, *optional*, defaults to 0.8):
+            Minimum overlap ratio required between the assigned query area and the original query mask area.
+
+    Returns:
+        `tuple[bool, torch.Tensor]`: A tuple containing whether the segment is valid and the final boolean mask for
+        that segment.
+    """
+    query_mask = mask_labels == query_idx
+    query_mask_area = query_mask.sum()
+
+    original_mask = mask_probs[query_idx] >= mask_threshold
+    original_area = original_mask.sum()
+
+    final_mask = query_mask & original_mask
+    final_mask_area = final_mask.sum()
+
+    mask_exists = query_mask_area > 0 and original_area > 0 and final_mask_area > 0
+
+    if mask_exists:
+        area_ratio = query_mask_area / original_area
+        if not area_ratio.item() > overlap_mask_area_threshold:
+            mask_exists = False
+
+    return mask_exists, final_mask
+
+
+def compute_segments(
+    mask_probs: "torch.Tensor",
+    pred_scores: "torch.Tensor",
+    pred_labels: "torch.Tensor",
+    label_ids_to_fuse: set[int] | None,
+    mask_threshold: float = 0.5,
+    overlap_mask_area_threshold: float = 0.8,
+    target_size: tuple[int, int] | None = None,
+) -> tuple["torch.Tensor", list[dict[str, int | float]]]:
+    """
+    Converts per-query mask predictions into a panoptic segmentation map.
+
+    Args:
+        mask_probs (`torch.Tensor`):
+            Tensor of shape `(num_queries, height, width)` containing per-query mask logits.
+        pred_scores (`torch.Tensor`):
+            Tensor of shape `(num_queries,)` containing the confidence score of each predicted query.
+        pred_labels (`torch.Tensor`):
+            Tensor of shape `(num_queries,)` containing the predicted class ID of each query.
+        label_ids_to_fuse (`set[int]`, *optional*):
+            Label IDs that should be fused across disconnected regions.
+        mask_threshold (`float`, *optional*, defaults to 0.5):
+            Threshold used to binarize the query mask probabilities.
+        overlap_mask_area_threshold (`float`, *optional*, defaults to 0.8):
+            Minimum overlap ratio required to keep a predicted segment.
+        target_size (`tuple[int, int]`, *optional*):
+            Final `(height, width)` of the segmentation map. If unset, uses the spatial size of `mask_probs`.
+
+    Returns:
+        `tuple[torch.Tensor, list[dict[str, int | float]]]`: The panoptic segmentation map and the metadata for each
+        predicted segment.
+    """
+    height = mask_probs.shape[1] if target_size is None else target_size[0]
+    width = mask_probs.shape[2] if target_size is None else target_size[1]
+
+    segmentation = torch.zeros((height, width), dtype=torch.long, device=mask_probs.device) - 1
+    segments: list[dict] = []
+
+    mask_probs = mask_probs.sigmoid()
+    mask_labels = (pred_scores[:, None, None] * mask_probs).argmax(0)
+
+    current_segment_id = 0
+    stuff_memory_list: dict[int, int] = {}
+
+    for query_idx in range(pred_labels.shape[0]):
+        pred_class = pred_labels[query_idx].item()
+
+        mask_exists, final_mask = check_segment_validity(
+            mask_labels, mask_probs, query_idx, mask_threshold, overlap_mask_area_threshold
+        )
+
+        if not mask_exists:
+            continue
+
+        if label_ids_to_fuse and pred_class in label_ids_to_fuse:
+            if pred_class in stuff_memory_list:
+                segmentation[final_mask] = stuff_memory_list[pred_class]
+                continue
+            else:
+                stuff_memory_list[pred_class] = current_segment_id
+
+        segmentation[final_mask] = current_segment_id
+        segment_score = round(pred_scores[query_idx].item(), 6)
+        segments.append(
+            {
+                "id": current_segment_id,
+                "label_id": pred_class,
+                "score": segment_score,
+            }
+        )
+        current_segment_id += 1
+    return segmentation, segments
+
+
+class VideomtVideoProcessor(BaseVideoProcessor):
+    resample = PILImageResampling.BILINEAR
+    image_mean = IMAGENET_DEFAULT_MEAN
+    image_std = IMAGENET_DEFAULT_STD
+    size = {"height": 640, "width": 640}
+    do_resize = True
+    do_center_crop = False
+    do_rescale = True
+    rescale_factor = 1 / 255
+    do_normalize = True
+    do_convert_rgb = True
+    do_sample_frames = False
+    model_input_names = ["pixel_values_videos"]
+
+    def _resize_mask_logits(
+        self,
+        masks_queries_logits: "torch.Tensor",
+        target_sizes: list[tuple[int, int]],
+    ) -> list["torch.Tensor"]:
+        """Interpolates mask logits to each frame's original resolution."""
+        resized = []
+        for idx, original_size in enumerate(target_sizes):
+            upsampled = F.interpolate(
+                masks_queries_logits[idx][None, ...],
+                size=original_size,
+                mode="bilinear",
+                align_corners=False,
+            )[0]
+            resized.append(upsampled)
+        return resized
+
+    def post_process_semantic_segmentation(
+        self,
+        outputs,
+        target_sizes: list[tuple[int, int]],
+    ) -> list["torch.Tensor"]:
+        """
+        Converts the output of [`VideomtForUniversalSegmentation`] into semantic segmentation predictions.
+
+        Args:
+            outputs ([`VideomtForUniversalSegmentationOutput`]):
+                Raw outputs of the model.
+            target_sizes (`list[tuple[int, int]]`):
+                List of `(height, width)` tuples corresponding to the requested final size of each prediction.
+                Length should match the number of frames in the output.
+
+        Returns:
+            `list[torch.Tensor]`: A list of tensors, each of shape `(height, width)`, where each value is the
+            predicted class index for the corresponding pixel.
+        """
+        requires_backends(self, ["torch"])
+
+        masks_queries_logits = outputs.masks_queries_logits  # [num_frames, num_queries, height, width]
+        class_queries_logits = outputs.class_queries_logits  # [num_frames, num_queries, num_classes+1]
+
+        # Remove the null class `[..., :-1]`
+        masks_classes = class_queries_logits.float().softmax(dim=-1)[..., :-1]
+        masks_probs = masks_queries_logits.float().sigmoid()
+
+        segmentation_logits = torch.matmul(masks_classes.transpose(1, 2), masks_probs.flatten(2))
+        segmentation_logits = segmentation_logits.reshape(
+            masks_probs.shape[0], masks_classes.shape[-1], masks_probs.shape[-2], masks_probs.shape[-1]
+        )
+
+        output_logits = self._resize_mask_logits(segmentation_logits, target_sizes)
+
+        return [logit.argmax(dim=0) for logit in output_logits]
+
+    def post_process_instance_segmentation(
+        self,
+        outputs,
+        target_sizes: list[tuple[int, int]],
+        threshold: float = 0.5,
+    ) -> list[dict]:
+        """
+        Converts the output of [`VideomtForUniversalSegmentation`] into instance segmentation predictions.
+
+        Args:
+            outputs ([`VideomtForUniversalSegmentationOutput`]):
+                Raw outputs of the model.
+            target_sizes (`list[tuple[int, int]]`):
+                List of `(height, width)` tuples corresponding to the requested final size of each prediction.
+                Length should match the number of frames in the output.
+            threshold (`float`, *optional*, defaults to 0.5):
+                Minimum combined score to keep an instance.
+
+        Returns:
+            `list[dict]`: A list of dicts (one per frame), each containing:
+                - `"segmentation"` -- A `torch.Tensor` of shape `(height, width)` with instance IDs (or -1 for background).
+                - `"segments_info"` -- A list of dicts with `"id"`, `"label_id"`, and `"score"` for each instance.
+        """
+        requires_backends(self, ["torch"])
+
+        class_queries_logits = outputs.class_queries_logits
+        masks_queries_logits = outputs.masks_queries_logits
+
+        mask_probs_batch = self._resize_mask_logits(masks_queries_logits, target_sizes)
+
+        device = masks_queries_logits.device
+        num_frames = class_queries_logits.shape[0]
+        num_queries = class_queries_logits.shape[-2]
+
+        results = []
+
+        for frame_idx in range(num_frames):
+            mask_pred = mask_probs_batch[frame_idx]
+            mask_class = class_queries_logits[frame_idx]
+
+            class_probs = mask_class.float().softmax(dim=-1)[..., :-1]
+            scores, pred_classes = class_probs.max(-1)
+            pred_masks = mask_pred > 0
+
+            mask_probs = mask_pred.float().sigmoid()
+            mask_scores = (mask_probs.flatten(1) * pred_masks.flatten(1)).sum(1) / (
+                pred_masks.flatten(1).sum(1) + 1e-6
+            )
+            pred_scores = scores * mask_scores
+
+            segmentation = torch.full(target_sizes[frame_idx], fill_value=-1, dtype=torch.long, device=device)
+
+            segments = []
+            current_segment_id = 0
+            for query_idx in range(num_queries):
+                score = pred_scores[query_idx].item()
+
+                if torch.any(pred_masks[query_idx]) and score >= threshold:
+                    segmentation[pred_masks[query_idx]] = current_segment_id
+                    segments.append(
+                        {
+                            "id": current_segment_id,
+                            "label_id": pred_classes[query_idx].item(),
+                            "score": round(score, 6),
+                        }
+                    )
+                    current_segment_id += 1
+
+            results.append({"segmentation": segmentation, "segments_info": segments})
+        return results
+
+    def post_process_panoptic_segmentation(
+        self,
+        outputs,
+        target_sizes: list[tuple[int, int]],
+        threshold: float = 0.8,
+        mask_threshold: float = 0.5,
+        overlap_mask_area_threshold: float = 0.8,
+        label_ids_to_fuse: set[int] | None = None,
+    ) -> list[dict]:
+        """
+        Converts the output of [`VideomtForUniversalSegmentation`] into panoptic segmentation predictions.
+
+        Args:
+            outputs ([`VideomtForUniversalSegmentationOutput`]):
+                Raw outputs of the model.
+            target_sizes (`list[tuple[int, int]]`):
+                List of `(height, width)` tuples corresponding to the requested final size of each prediction.
+                Length should match the number of frames in the output.
+            threshold (`float`, *optional*, defaults to 0.8):
+                Minimum score to keep a predicted segment.
+            mask_threshold (`float`, *optional*, defaults to 0.5):
+                Threshold for binarizing mask probabilities.
+            overlap_mask_area_threshold (`float`, *optional*, defaults to 0.8):
+                Overlap threshold to merge masks into a single segment.
+            label_ids_to_fuse (`set[int]`, *optional*):
+                Label IDs that should be fused across disconnected regions.
+
+        Returns:
+            `list[dict]`: A list of dicts (one per frame), each containing:
+                - `"segmentation"` -- A `torch.Tensor` of shape `(height, width)` with segment IDs (or -1 for background).
+                - `"segments_info"` -- A list of dicts with `"id"`, `"label_id"`, and `"score"` for each segment.
+        """
+        requires_backends(self, ["torch"])
+
+        masks_queries_logits = outputs.masks_queries_logits
+        class_queries_logits = outputs.class_queries_logits
+
+        num_frames = class_queries_logits.shape[0]
+        num_labels = class_queries_logits.shape[-1] - 1
+
+        mask_probs_batch = self._resize_mask_logits(masks_queries_logits, target_sizes)
+        pred_scores_batch, pred_labels_batch = class_queries_logits.float().softmax(dim=-1).max(-1)
+
+        results: list = []
+
+        for frame_idx in range(num_frames):
+            mask_probs = mask_probs_batch[frame_idx]
+            pred_scores = pred_scores_batch[frame_idx]
+            pred_labels = pred_labels_batch[frame_idx]
+
+            if not (mask_probs.shape[0] == pred_scores.shape[0] == pred_labels.shape[0]):
+                raise ValueError("mask, scores and labels must have the same shape!")
+
+            to_keep = pred_labels.ne(num_labels) & (pred_scores > threshold)
+            mask_probs = mask_probs[to_keep]
+            pred_scores = pred_scores[to_keep]
+            pred_labels = pred_labels[to_keep]
+
+            if mask_probs.shape[0] <= 0:
+                height, width = target_sizes[frame_idx] if target_sizes is not None else mask_probs.shape[1:]
+                segmentation = torch.full(
+                    (height, width), fill_value=-1, dtype=torch.long, device=masks_queries_logits.device
+                )
+                results.append({"segmentation": segmentation, "segments_info": []})
+                continue
+
+            segmentation, segments = compute_segments(
+                mask_probs=mask_probs,
+                pred_scores=pred_scores,
+                pred_labels=pred_labels,
+                label_ids_to_fuse=label_ids_to_fuse,
+                mask_threshold=mask_threshold,
+                overlap_mask_area_threshold=overlap_mask_area_threshold,
+                target_size=target_sizes[frame_idx] if target_sizes is not None else None,
+            )
+
+            results.append({"segmentation": segmentation, "segments_info": segments})
+        return results
+
+
+__all__ = ["VideomtVideoProcessor"]