ModernLayoutLM / modeling_modernlayoutlm.py

Upload folder using huggingface_hub

3fac549 verified 28 days ago

15.9 kB

"""
ModernLayoutLM: LayoutLM's 2D layout embeddings summed into a ModernBERT backbone
The graft is confined to the embedding module so attention keeps its fused kernels and RoPE
This module is copied into exported model directories and may import only torch and transformers
"""

from __future__ import annotations

from typing import Any, cast

import torch
from torch import nn
from transformers.masking_utils import (
    create_bidirectional_mask,
    create_bidirectional_sliding_window_mask,
)
from transformers.modeling_outputs import (
    BaseModelOutput,
    MaskedLMOutput,
    SequenceClassifierOutput,
    TokenClassifierOutput,
)
from transformers.models.modernbert.modeling_modernbert import (
    ModernBertEmbeddings,
    ModernBertEncoderLayer,
    ModernBertForMaskedLM,
    ModernBertForSequenceClassification,
    ModernBertForTokenClassification,
    ModernBertModel,
    ModernBertPreTrainedModel,
)
from transformers.utils.auto_docstring import auto_docstring
from transformers.utils.generic import merge_with_config_defaults
from transformers.utils.output_capturing import capture_outputs

from .configuration_modernlayoutlm import ModernLayoutLmConfig

# Bounding boxes arrive as (x0, y0, x1, y1) with the origin at the top left corner
BBOX_COORDS = 4


class LayoutEmbedding(nn.Embedding):
    """
    A coordinate embedding table, marked by its type so weight init can find it
    `_init_weights` sees leaf modules, where a plain `nn.Embedding` would pass for the token table.
    It would then take ModernBERT's default normal init instead of being zeroed.
    """


class ModernLayoutLmEmbeddings(ModernBertEmbeddings):
    """
    ModernBERT's embeddings plus LayoutLM's four 2D position embedding tables
    Args:
        config: The model config, supplying `max_2d_position_embeddings` and `zero_init_2d`.
    """

    def __init__(self, config: ModernLayoutLmConfig) -> None:
        super().__init__(config)
        max_2d = config.max_2d_position_embeddings
        self.x_position_embeddings = LayoutEmbedding(max_2d, config.hidden_size)
        self.y_position_embeddings = LayoutEmbedding(max_2d, config.hidden_size)
        self.w_position_embeddings = LayoutEmbedding(max_2d, config.hidden_size)
        self.h_position_embeddings = LayoutEmbedding(max_2d, config.hidden_size)
        if getattr(config, "zero_init_2d", True):
            self.reset_layout_parameters()

    def reset_layout_parameters(self) -> None:
        """
        Zeroes the four layout tables so the model reduces exactly to its ModernBERT backbone
        """
        for table in self.layout_tables():
            nn.init.zeros_(table.weight)

    def layout_tables(self) -> list[LayoutEmbedding]:
        """
        Returns the four 2D position embedding tables
        Used by the optimizer to give the freshly grafted parameters their own learning rate.
        Returns:
            The x, y, w and h embedding tables, in that order.
        """
        return [
            self.x_position_embeddings,
            self.y_position_embeddings,
            self.w_position_embeddings,
            self.h_position_embeddings,
        ]

    def layout_embeddings(self, bbox: torch.Tensor) -> torch.Tensor:
        """
        Looks up and sums the six layout embeddings for each box
        Coordinates clamp into the table, and a negative width or height clamps to zero.
        A malformed box therefore degrades rather than raising.
        `x0` and `x1` share the X table and `y0` and `y1` the Y table, matching LayoutLM v1.
        Args:
            bbox: Long tensor of shape (batch, seq_len, 4) holding (x0, y0, x1, y1).
        Returns:
            A tensor of shape (batch, seq_len, hidden_size).
        Raises:
            ValueError: If the last dimension of `bbox` is not 4.
        """
        if bbox.shape[-1] != BBOX_COORDS:
            raise ValueError(f"bbox must have last dimension {BBOX_COORDS}, got {bbox.shape[-1]}")

        limit = self.x_position_embeddings.num_embeddings - 1
        boxes = bbox.long().clamp(0, limit)
        x0, y0, x1, y1 = boxes.unbind(dim=-1)
        width = (x1 - x0).clamp(0, limit)
        height = (y1 - y0).clamp(0, limit)

        return (
            self.x_position_embeddings(x0)
            + self.y_position_embeddings(y0)
            + self.x_position_embeddings(x1)
            + self.y_position_embeddings(y1)
            + self.w_position_embeddings(width)
            + self.h_position_embeddings(height)
        )

    def forward(
        self,
        input_ids: torch.LongTensor | None = None,
        inputs_embeds: torch.Tensor | None = None,
        bbox: torch.Tensor | None = None,
    ) -> torch.Tensor:
        """
        Sums token and layout embeddings, then applies the inherited LayerNorm and dropout
        Args:
            input_ids: Token ids of shape (batch, seq_len).
            bbox: Boxes of shape (batch, seq_len, 4), or None to behave as plain ModernBERT.
            inputs_embeds: Precomputed token embeddings, used instead of `input_ids`.
        Returns:
            Normalized embeddings of shape (batch, seq_len, hidden_size).
        """
        if inputs_embeds is not None:
            hidden_states = inputs_embeds
        else:
            hidden_states = self.tok_embeddings(input_ids)
        if bbox is not None:
            hidden_states = hidden_states + self.layout_embeddings(bbox)
        return self.drop(self.norm(hidden_states))


class ModernLayoutLmPreTrainedModel(ModernBertPreTrainedModel):
    """
    Shared base class, extending ModernBERT's weight initialization to the layout tables
    """

    config_class = ModernLayoutLmConfig
    base_model_prefix = "model"
    _no_split_modules = ["ModernLayoutLmEmbeddings", "ModernBertEncoderLayer"]

    @torch.no_grad()
    def _init_weights(self, module: nn.Module) -> None:
        """
        Initializes a module, adding layout table handling on top of ModernBERT's rules
        ModernBERT's default init is far too large to add onto a pretrained token embedding.
        `from_pretrained` calls this for keys a checkpoint lacks, so a graft starts zeroed.
        Args:
            module: The module to initialize.
        """
        super()._init_weights(module)
        if isinstance(module, LayoutEmbedding):
            if getattr(self.config, "zero_init_2d", True):
                nn.init.zeros_(module.weight)
            else:
                nn.init.normal_(module.weight, mean=0.0, std=self.config.initializer_range)


@auto_docstring
class ModernLayoutLmModel(ModernLayoutLmPreTrainedModel, ModernBertModel):
    """
    The bare ModernLayoutLM encoder
    Identical to `ModernBertModel` except that `bbox` is threaded into the embedding module.
    """

    def __init__(self, config: ModernLayoutLmConfig) -> None:
        super().__init__(config)
        self.embeddings = ModernLayoutLmEmbeddings(config)
        self.post_init()

    @merge_with_config_defaults
    @capture_outputs
    @auto_docstring
    def forward(
        self,
        input_ids: torch.LongTensor | None = None,
        attention_mask: torch.Tensor | None = None,
        position_ids: torch.Tensor | None = None,
        inputs_embeds: torch.Tensor | None = None,
        bbox: torch.Tensor | None = None,
        **kwargs: Any,
    ) -> BaseModelOutput:
        r"""
        bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*):
            Boxes as (x0, y0, x1, y1) on `[0, config.coordinate_scale]`, origin at the top left.
            Omitting it makes the model behave as plain ModernBERT.
        """
        # Branched rather than written as an xor so that the non-None case is provable
        if inputs_embeds is not None:
            if input_ids is not None:
                raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
            seq_len, device = inputs_embeds.shape[1], inputs_embeds.device
        elif input_ids is not None:
            seq_len, device = input_ids.shape[1], input_ids.device
        else:
            raise ValueError("You must specify exactly one of input_ids or inputs_embeds")

        if position_ids is None:
            position_ids = torch.arange(seq_len, device=device).unsqueeze(0)

        hidden_states = self.embeddings(
            input_ids=input_ids,
            bbox=bbox,
            inputs_embeds=inputs_embeds,
        )

        if not isinstance(attention_mask_mapping := attention_mask, dict):
            mask_kwargs = {
                "config": self.config,
                "inputs_embeds": hidden_states,
                "attention_mask": attention_mask,
            }
            attention_mask_mapping = {
                "full_attention": create_bidirectional_mask(**mask_kwargs),
                "sliding_attention": create_bidirectional_sliding_window_mask(**mask_kwargs),
            }

        layer_types: list[str] = self.config.layer_types or []
        position_embeddings = {}
        for layer_type in set(layer_types):
            position_embeddings[layer_type] = self.rotary_emb(
                hidden_states, position_ids, layer_type
            )

        # Cast because nn.Module.__getattr__ erases the element type of self.layers
        for module in self.layers:
            encoder_layer = cast(ModernBertEncoderLayer, module)
            attention_type = encoder_layer.attention_type
            hidden_states = encoder_layer(
                hidden_states,
                attention_mask=attention_mask_mapping[attention_type],
                position_embeddings=position_embeddings[attention_type],
                **kwargs,
            )

        hidden_states = self.final_norm(hidden_states)

        return BaseModelOutput(last_hidden_state=hidden_states)


@auto_docstring
class ModernLayoutLmForMaskedLM(ModernLayoutLmPreTrainedModel, ModernBertForMaskedLM):
    """
    ModernLayoutLM with a masked language modeling head, for the layout-aware MLM stage
    """

    _tied_weights_keys = {"decoder.weight": "model.embeddings.tok_embeddings.weight"}

    def __init__(self, config: ModernLayoutLmConfig) -> None:
        super().__init__(config)
        self.model = ModernLayoutLmModel(config)
        self.post_init()

    def forward(
        self,
        input_ids: torch.LongTensor | None = None,
        attention_mask: torch.Tensor | None = None,
        position_ids: torch.Tensor | None = None,
        inputs_embeds: torch.Tensor | None = None,
        labels: torch.Tensor | None = None,
        bbox: torch.Tensor | None = None,
        **kwargs: Any,
    ) -> tuple[torch.Tensor] | MaskedLMOutput:
        """
        Runs the encoder and the MLM head
        `bbox` is named rather than left to `**kwargs`, as `Trainer` prunes columns by signature.
        Were it implicit, the boxes would be dropped before they ever reached the model.
        Args:
            input_ids: Token ids of shape (batch, seq_len).
            attention_mask: Mask of shape (batch, seq_len).
            bbox: Boxes of shape (batch, seq_len, 4), normalized to the coordinate scale.
            position_ids: Optional explicit 1D positions.
            inputs_embeds: Precomputed token embeddings, used instead of `input_ids`.
            labels: MLM targets of shape (batch, seq_len), `-100` where unmasked.
            **kwargs: Forwarded to the backbone.
        Returns:
            A MaskedLMOutput, or a tuple when the config asks for one.
        """
        return super().forward(
            input_ids=input_ids,
            attention_mask=attention_mask,
            position_ids=position_ids,
            inputs_embeds=inputs_embeds,
            labels=labels,
            # Ignore as the ModernBert head forwards bbox opaquely and pyright loses it
            bbox=bbox,  # pyright: ignore[reportCallIssue]
            **kwargs,
        )


@auto_docstring
class ModernLayoutLmForTokenClassification(
    ModernLayoutLmPreTrainedModel, ModernBertForTokenClassification
):
    """
    ModernLayoutLM with a token classification head, for form and receipt entity extraction
    """

    def __init__(self, config: ModernLayoutLmConfig) -> None:
        super().__init__(config)
        self.model = ModernLayoutLmModel(config)
        self.post_init()

    def forward(
        self,
        input_ids: torch.LongTensor | None = None,
        attention_mask: torch.Tensor | None = None,
        position_ids: torch.Tensor | None = None,
        inputs_embeds: torch.Tensor | None = None,
        labels: torch.Tensor | None = None,
        bbox: torch.Tensor | None = None,
        **kwargs: Any,
    ) -> tuple[torch.Tensor] | TokenClassifierOutput:
        """
        Runs the encoder and the per-token classifier
        Args:
            input_ids: Token ids of shape (batch, seq_len).
            attention_mask: Mask of shape (batch, seq_len).
            bbox: Boxes of shape (batch, seq_len, 4), normalized to the coordinate scale.
            position_ids: Optional explicit 1D positions.
            inputs_embeds: Precomputed token embeddings, used instead of `input_ids`.
            labels: Class index per token, `-100` to ignore.
            **kwargs: Forwarded to the backbone.
        Returns:
            A TokenClassifierOutput, or a tuple when the config asks for one.
        """
        return super().forward(
            input_ids=input_ids,
            attention_mask=attention_mask,
            position_ids=position_ids,
            inputs_embeds=inputs_embeds,
            labels=labels,
            # Ignore as the ModernBert head forwards bbox opaquely and pyright loses it
            bbox=bbox,  # pyright: ignore[reportCallIssue]
            **kwargs,
        )


@auto_docstring
class ModernLayoutLmForSequenceClassification(
    ModernLayoutLmPreTrainedModel, ModernBertForSequenceClassification
):
    """
    ModernLayoutLM with a pooled sequence classification head, for document classification
    """

    def __init__(self, config: ModernLayoutLmConfig) -> None:
        super().__init__(config)
        self.model = ModernLayoutLmModel(config)
        self.post_init()

    def forward(
        self,
        input_ids: torch.LongTensor | None = None,
        attention_mask: torch.Tensor | None = None,
        position_ids: torch.Tensor | None = None,
        inputs_embeds: torch.Tensor | None = None,
        labels: torch.Tensor | None = None,
        bbox: torch.Tensor | None = None,
        **kwargs: Any,
    ) -> tuple[torch.Tensor] | SequenceClassifierOutput:
        """
        Runs the encoder, pools per the config, then classifies the document
        Args:
            input_ids: Token ids of shape (batch, seq_len).
            attention_mask: Mask of shape (batch, seq_len).
            bbox: Boxes of shape (batch, seq_len, 4), normalized to the coordinate scale.
            position_ids: Optional explicit 1D positions.
            inputs_embeds: Precomputed token embeddings, used instead of `input_ids`.
            labels: One class index per document.
            **kwargs: Forwarded to the backbone.
        Returns:
            A SequenceClassifierOutput, or a tuple when the config asks for one.
        """
        return super().forward(
            input_ids=input_ids,
            attention_mask=attention_mask,
            position_ids=position_ids,
            inputs_embeds=inputs_embeds,
            labels=labels,
            # Ignore as the ModernBert head forwards bbox opaquely and pyright loses it
            bbox=bbox,  # pyright: ignore[reportCallIssue]
            **kwargs,
        )


__all__ = [
    "ModernLayoutLmEmbeddings",
    "ModernLayoutLmForMaskedLM",
    "ModernLayoutLmForSequenceClassification",
    "ModernLayoutLmForTokenClassification",
    "ModernLayoutLmModel",
    "ModernLayoutLmPreTrainedModel",
]

	"""
	ModernLayoutLM: LayoutLM's 2D layout embeddings summed into a ModernBERT backbone

	The graft is confined to the embedding module so attention keeps its fused kernels and RoPE
	This module is copied into exported model directories and may import only torch and transformers
	"""

	from __future__ import annotations

	from typing import Any, cast

	import torch
	from torch import nn
	from transformers.masking_utils import (
	create_bidirectional_mask,
	create_bidirectional_sliding_window_mask,
	)
	from transformers.modeling_outputs import (
	BaseModelOutput,
	MaskedLMOutput,
	SequenceClassifierOutput,
	TokenClassifierOutput,
	)
	from transformers.models.modernbert.modeling_modernbert import (
	ModernBertEmbeddings,
	ModernBertEncoderLayer,
	ModernBertForMaskedLM,
	ModernBertForSequenceClassification,
	ModernBertForTokenClassification,
	ModernBertModel,
	ModernBertPreTrainedModel,
	)
	from transformers.utils.auto_docstring import auto_docstring
	from transformers.utils.generic import merge_with_config_defaults
	from transformers.utils.output_capturing import capture_outputs

	from .configuration_modernlayoutlm import ModernLayoutLmConfig

	# Bounding boxes arrive as (x0, y0, x1, y1) with the origin at the top left corner
	BBOX_COORDS = 4


	class LayoutEmbedding(nn.Embedding):
	"""
	A coordinate embedding table, marked by its type so weight init can find it

	`_init_weights` sees leaf modules, where a plain `nn.Embedding` would pass for the token table.
	It would then take ModernBERT's default normal init instead of being zeroed.
	"""


	class ModernLayoutLmEmbeddings(ModernBertEmbeddings):
	"""
	ModernBERT's embeddings plus LayoutLM's four 2D position embedding tables

	Args:
	config: The model config, supplying `max_2d_position_embeddings` and `zero_init_2d`.
	"""

	def __init__(self, config: ModernLayoutLmConfig) -> None:
	super().__init__(config)
	max_2d = config.max_2d_position_embeddings
	self.x_position_embeddings = LayoutEmbedding(max_2d, config.hidden_size)
	self.y_position_embeddings = LayoutEmbedding(max_2d, config.hidden_size)
	self.w_position_embeddings = LayoutEmbedding(max_2d, config.hidden_size)
	self.h_position_embeddings = LayoutEmbedding(max_2d, config.hidden_size)
	if getattr(config, "zero_init_2d", True):
	self.reset_layout_parameters()

	def reset_layout_parameters(self) -> None:
	"""
	Zeroes the four layout tables so the model reduces exactly to its ModernBERT backbone
	"""
	for table in self.layout_tables():
	nn.init.zeros_(table.weight)

	def layout_tables(self) -> list[LayoutEmbedding]:
	"""
	Returns the four 2D position embedding tables

	Used by the optimizer to give the freshly grafted parameters their own learning rate.

	Returns:
	The x, y, w and h embedding tables, in that order.
	"""
	return [
	self.x_position_embeddings,
	self.y_position_embeddings,
	self.w_position_embeddings,
	self.h_position_embeddings,
	]

	def layout_embeddings(self, bbox: torch.Tensor) -> torch.Tensor:
	"""
	Looks up and sums the six layout embeddings for each box

	Coordinates clamp into the table, and a negative width or height clamps to zero.
	A malformed box therefore degrades rather than raising.
	`x0` and `x1` share the X table and `y0` and `y1` the Y table, matching LayoutLM v1.

	Args:
	bbox: Long tensor of shape (batch, seq_len, 4) holding (x0, y0, x1, y1).

	Returns:
	A tensor of shape (batch, seq_len, hidden_size).

	Raises:
	ValueError: If the last dimension of `bbox` is not 4.
	"""
	if bbox.shape[-1] != BBOX_COORDS:
	raise ValueError(f"bbox must have last dimension {BBOX_COORDS}, got {bbox.shape[-1]}")

	limit = self.x_position_embeddings.num_embeddings - 1
	boxes = bbox.long().clamp(0, limit)
	x0, y0, x1, y1 = boxes.unbind(dim=-1)
	width = (x1 - x0).clamp(0, limit)
	height = (y1 - y0).clamp(0, limit)

	return (
	self.x_position_embeddings(x0)
	+ self.y_position_embeddings(y0)
	+ self.x_position_embeddings(x1)
	+ self.y_position_embeddings(y1)
	+ self.w_position_embeddings(width)
	+ self.h_position_embeddings(height)
	)

	def forward(
	self,
	input_ids: torch.LongTensor \| None = None,
	inputs_embeds: torch.Tensor \| None = None,
	bbox: torch.Tensor \| None = None,
	) -> torch.Tensor:
	"""
	Sums token and layout embeddings, then applies the inherited LayerNorm and dropout

	Args:
	input_ids: Token ids of shape (batch, seq_len).
	bbox: Boxes of shape (batch, seq_len, 4), or None to behave as plain ModernBERT.
	inputs_embeds: Precomputed token embeddings, used instead of `input_ids`.

	Returns:
	Normalized embeddings of shape (batch, seq_len, hidden_size).
	"""
	if inputs_embeds is not None:
	hidden_states = inputs_embeds
	else:
	hidden_states = self.tok_embeddings(input_ids)
	if bbox is not None:
	hidden_states = hidden_states + self.layout_embeddings(bbox)
	return self.drop(self.norm(hidden_states))


	class ModernLayoutLmPreTrainedModel(ModernBertPreTrainedModel):
	"""
	Shared base class, extending ModernBERT's weight initialization to the layout tables
	"""

	config_class = ModernLayoutLmConfig
	base_model_prefix = "model"
	_no_split_modules = ["ModernLayoutLmEmbeddings", "ModernBertEncoderLayer"]

	@torch.no_grad()
	def _init_weights(self, module: nn.Module) -> None:
	"""
	Initializes a module, adding layout table handling on top of ModernBERT's rules

	ModernBERT's default init is far too large to add onto a pretrained token embedding.
	`from_pretrained` calls this for keys a checkpoint lacks, so a graft starts zeroed.

	Args:
	module: The module to initialize.
	"""
	super()._init_weights(module)
	if isinstance(module, LayoutEmbedding):
	if getattr(self.config, "zero_init_2d", True):
	nn.init.zeros_(module.weight)
	else:
	nn.init.normal_(module.weight, mean=0.0, std=self.config.initializer_range)


	@auto_docstring
	class ModernLayoutLmModel(ModernLayoutLmPreTrainedModel, ModernBertModel):
	"""
	The bare ModernLayoutLM encoder

	Identical to `ModernBertModel` except that `bbox` is threaded into the embedding module.
	"""

	def __init__(self, config: ModernLayoutLmConfig) -> None:
	super().__init__(config)
	self.embeddings = ModernLayoutLmEmbeddings(config)
	self.post_init()

	@merge_with_config_defaults
	@capture_outputs
	@auto_docstring
	def forward(
	self,
	input_ids: torch.LongTensor \| None = None,
	attention_mask: torch.Tensor \| None = None,
	position_ids: torch.Tensor \| None = None,
	inputs_embeds: torch.Tensor \| None = None,
	bbox: torch.Tensor \| None = None,
	**kwargs: Any,
	) -> BaseModelOutput:
	r"""
	bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, optional):
	Boxes as (x0, y0, x1, y1) on `[0, config.coordinate_scale]`, origin at the top left.
	Omitting it makes the model behave as plain ModernBERT.
	"""
	# Branched rather than written as an xor so that the non-None case is provable
	if inputs_embeds is not None:
	if input_ids is not None:
	raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
	seq_len, device = inputs_embeds.shape[1], inputs_embeds.device
	elif input_ids is not None:
	seq_len, device = input_ids.shape[1], input_ids.device
	else:
	raise ValueError("You must specify exactly one of input_ids or inputs_embeds")

	if position_ids is None:
	position_ids = torch.arange(seq_len, device=device).unsqueeze(0)

	hidden_states = self.embeddings(
	input_ids=input_ids,
	bbox=bbox,
	inputs_embeds=inputs_embeds,
	)

	if not isinstance(attention_mask_mapping := attention_mask, dict):
	mask_kwargs = {
	"config": self.config,
	"inputs_embeds": hidden_states,
	"attention_mask": attention_mask,
	}
	attention_mask_mapping = {
	"full_attention": create_bidirectional_mask(**mask_kwargs),
	"sliding_attention": create_bidirectional_sliding_window_mask(**mask_kwargs),
	}

	layer_types: list[str] = self.config.layer_types or []
	position_embeddings = {}
	for layer_type in set(layer_types):
	position_embeddings[layer_type] = self.rotary_emb(
	hidden_states, position_ids, layer_type
	)

	# Cast because nn.Module.__getattr__ erases the element type of self.layers
	for module in self.layers:
	encoder_layer = cast(ModernBertEncoderLayer, module)
	attention_type = encoder_layer.attention_type
	hidden_states = encoder_layer(
	hidden_states,
	attention_mask=attention_mask_mapping[attention_type],
	position_embeddings=position_embeddings[attention_type],
	**kwargs,
	)

	hidden_states = self.final_norm(hidden_states)

	return BaseModelOutput(last_hidden_state=hidden_states)


	@auto_docstring
	class ModernLayoutLmForMaskedLM(ModernLayoutLmPreTrainedModel, ModernBertForMaskedLM):
	"""
	ModernLayoutLM with a masked language modeling head, for the layout-aware MLM stage
	"""

	_tied_weights_keys = {"decoder.weight": "model.embeddings.tok_embeddings.weight"}

	def __init__(self, config: ModernLayoutLmConfig) -> None:
	super().__init__(config)
	self.model = ModernLayoutLmModel(config)
	self.post_init()

	def forward(
	self,
	input_ids: torch.LongTensor \| None = None,
	attention_mask: torch.Tensor \| None = None,
	position_ids: torch.Tensor \| None = None,
	inputs_embeds: torch.Tensor \| None = None,
	labels: torch.Tensor \| None = None,
	bbox: torch.Tensor \| None = None,
	**kwargs: Any,
	) -> tuple[torch.Tensor] \| MaskedLMOutput:
	"""
	Runs the encoder and the MLM head

	`bbox` is named rather than left to `**kwargs`, as `Trainer` prunes columns by signature.
	Were it implicit, the boxes would be dropped before they ever reached the model.

	Args:
	input_ids: Token ids of shape (batch, seq_len).
	attention_mask: Mask of shape (batch, seq_len).
	bbox: Boxes of shape (batch, seq_len, 4), normalized to the coordinate scale.
	position_ids: Optional explicit 1D positions.
	inputs_embeds: Precomputed token embeddings, used instead of `input_ids`.
	labels: MLM targets of shape (batch, seq_len), `-100` where unmasked.
	**kwargs: Forwarded to the backbone.

	Returns:
	A MaskedLMOutput, or a tuple when the config asks for one.
	"""
	return super().forward(
	input_ids=input_ids,
	attention_mask=attention_mask,
	position_ids=position_ids,
	inputs_embeds=inputs_embeds,
	labels=labels,
	# Ignore as the ModernBert head forwards bbox opaquely and pyright loses it
	bbox=bbox, # pyright: ignore[reportCallIssue]
	**kwargs,
	)


	@auto_docstring
	class ModernLayoutLmForTokenClassification(
	ModernLayoutLmPreTrainedModel, ModernBertForTokenClassification
	):
	"""
	ModernLayoutLM with a token classification head, for form and receipt entity extraction
	"""

	def __init__(self, config: ModernLayoutLmConfig) -> None:
	super().__init__(config)
	self.model = ModernLayoutLmModel(config)
	self.post_init()

	def forward(
	self,
	input_ids: torch.LongTensor \| None = None,
	attention_mask: torch.Tensor \| None = None,
	position_ids: torch.Tensor \| None = None,
	inputs_embeds: torch.Tensor \| None = None,
	labels: torch.Tensor \| None = None,
	bbox: torch.Tensor \| None = None,
	**kwargs: Any,
	) -> tuple[torch.Tensor] \| TokenClassifierOutput:
	"""
	Runs the encoder and the per-token classifier

	Args:
	input_ids: Token ids of shape (batch, seq_len).
	attention_mask: Mask of shape (batch, seq_len).
	bbox: Boxes of shape (batch, seq_len, 4), normalized to the coordinate scale.
	position_ids: Optional explicit 1D positions.
	inputs_embeds: Precomputed token embeddings, used instead of `input_ids`.
	labels: Class index per token, `-100` to ignore.
	**kwargs: Forwarded to the backbone.

	Returns:
	A TokenClassifierOutput, or a tuple when the config asks for one.
	"""
	return super().forward(
	input_ids=input_ids,
	attention_mask=attention_mask,
	position_ids=position_ids,
	inputs_embeds=inputs_embeds,
	labels=labels,
	# Ignore as the ModernBert head forwards bbox opaquely and pyright loses it
	bbox=bbox, # pyright: ignore[reportCallIssue]
	**kwargs,
	)


	@auto_docstring
	class ModernLayoutLmForSequenceClassification(
	ModernLayoutLmPreTrainedModel, ModernBertForSequenceClassification
	):
	"""
	ModernLayoutLM with a pooled sequence classification head, for document classification
	"""

	def __init__(self, config: ModernLayoutLmConfig) -> None:
	super().__init__(config)
	self.model = ModernLayoutLmModel(config)
	self.post_init()

	def forward(
	self,
	input_ids: torch.LongTensor \| None = None,
	attention_mask: torch.Tensor \| None = None,
	position_ids: torch.Tensor \| None = None,
	inputs_embeds: torch.Tensor \| None = None,
	labels: torch.Tensor \| None = None,
	bbox: torch.Tensor \| None = None,
	**kwargs: Any,
	) -> tuple[torch.Tensor] \| SequenceClassifierOutput:
	"""
	Runs the encoder, pools per the config, then classifies the document

	Args:
	input_ids: Token ids of shape (batch, seq_len).
	attention_mask: Mask of shape (batch, seq_len).
	bbox: Boxes of shape (batch, seq_len, 4), normalized to the coordinate scale.
	position_ids: Optional explicit 1D positions.
	inputs_embeds: Precomputed token embeddings, used instead of `input_ids`.
	labels: One class index per document.
	**kwargs: Forwarded to the backbone.

	Returns:
	A SequenceClassifierOutput, or a tuple when the config asks for one.
	"""
	return super().forward(
	input_ids=input_ids,
	attention_mask=attention_mask,
	position_ids=position_ids,
	inputs_embeds=inputs_embeds,
	labels=labels,
	# Ignore as the ModernBert head forwards bbox opaquely and pyright loses it
	bbox=bbox, # pyright: ignore[reportCallIssue]
	**kwargs,
	)


	__all__ = [
	"ModernLayoutLmEmbeddings",
	"ModernLayoutLmForMaskedLM",
	"ModernLayoutLmForSequenceClassification",
	"ModernLayoutLmForTokenClassification",
	"ModernLayoutLmModel",
	"ModernLayoutLmPreTrainedModel",
	]