Upload folder using huggingface_hub

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,153 @@
+---
+library_name: transformers
+license: apache-2.0
+language:
+- en
+base_model: answerdotai/ModernBERT-base
+pipeline_tag: fill-mask
+tags:
+- document-understanding
+- layoutlm
+- modernbert
+---
+
+# ModernLayoutLM
+
+ModernLayoutLM is an attempt to produce a long-context (like ModernBERT) bidirectional encoder that incorporates layout information into its training (like LayoutLM). ModernLayoutLM uses the same recipe as Xu et al. (2020) in the creation of LayoutLMv1, where the major difference is swapping BERT with ModernBERT (Warner et al., 2024).
+
+## Intended use
+
+The model is a pre-trained checkpoint that can be fine-tuned to layout understanding tasks, like named entity recognition or document classification.
+
+```python
+from transformers import AutoModelForMaskedLM, AutoProcessor
+
+path = "kkuroma/ModernLayoutLM"
+model = AutoModelForMaskedLM.from_pretrained(path, trust_remote_code=True)
+processor = AutoProcessor.from_pretrained(path, trust_remote_code=True)
+
+batch = processor(words=["Total", "42.00"], boxes=[[10, 10, 60, 20], [70, 10, 140, 20]], max_length=512, return_tensors="pt")
+model(**batch)
+```
+
+- `bbox` holds one `(x0, y0, x1, y1)` per word, with the origin at the top left of the page.
+- Coordinates are normalized to `[0, 1000]` by the page width and height, not given in pixels.
+- Every subword of a word takes the word's box, which the bundled processor handles.
+- `padding` defaults to `max_length`, which without an explicit length is the model's own 8192.
+- `bbox=None` skips the layout sum and the model runs as plain ModernBERT, while all-zero boxes put every word at the origin.
+
+## Training data
+
+ModernLayoutLM is pretrained on about 11M pages of English business and scientific documents, each read once, in two stages.
+
+| stage | source | pages |
+|---|---|---|
+| 1 | Industry Documents Library (IDL), words and boxes from Amazon Textract | 1.5M |
+| 1 | PubTables-1M | 461k |
+| 1 | PDFA (`pixparse/pdfa-eng-wds`) | 160k |
+| 1 | DocLayNet v1.2 | 68k |
+| 2 | IDL, from the shards stage 1 never read | 8.8M |
+
+Pages carrying a Bates number from the FUNSD or RVL-CDIP evaluation splits are dropped from both stages, since IDL and those benchmarks share IIT-CDIP.
+
+## Training procedure
+
+ModernLayoutLM is pretrained with the masked visual-language model of LayoutLMv1 (Xu et al., 2020). Four 2D position embedding tables, where `x0` and `x1` share table X and `y0` and `y1` share table Y, are summed into the token embeddings before the embedding LayerNorm. Text is masked and boxes are left intact, so recovering a masked token rewards reading its position on the page. The four tables start at zero, so the model starts numerically identical to ModernBERT. The number of iterations and the duration are set by hardware limitations, as every step ran on a single RTX 3090.
+
+| setting | value |
+|---|---|
+| sequence length | 512 tokens, packed |
+| masking rate | 30%, following ModernBERT |
+| batch | 64 sequences per optimizer step |
+| learning rate | 5e-5 for the backbone, 1e-3 for the four layout tables |
+| schedule | 5% warmup, weight decay 0.01, one epoch per stage |
+| precision | fp32 master weights under bf16 autocast |
+| steps | 29,899 in stage 1, 115,009 in stage 2 |
+
+| stage | data unpacking | training |
+|---|---|---|
+| 1 | not recorded | 10.1 hours at 52 sequences per second |
+| 2 | 0.9 hours to unpack the IDL shards, 2.3 hours to tokenize and pack | 17.9 hours at 114 sequences per second |
+
+## Evaluation
+
+### FUNSD
+
+- Source: Jaume et al. (2019), read from `nielsr/funsd-layoutlmv3`
+- Released: 2019
+- Size: 199 scanned forms, 149 train and 50 test
+- Task: entity labeling over header, question and answer, as 7 BIO labels
+
+### CORD
+
+- Source: Park et al. (2019), read from `naver-clova-ix/cord-v2`
+- Released: 2019
+- Size: 1,000 Indonesian receipts, 800 train, 100 validation and 100 test
+- Task: receipt field labeling over 29 categories such as menu name, price and total, as 59 BIO labels
+
+### SROIE
+
+- Source: the ICDAR 2019 Competition on Scanned Receipt OCR and Information Extraction (Huang et al., 2019), read from `Ssunbell/SROIE_layoutlmv2_sequence`
+- Released: 2019
+- Size: 973 receipts, 594 train, 32 validation and 347 test
+- Task: key information extraction over company, date, address and total, as 9 BIO labels
+
+### Results
+
+Each column is one model fine-tuned on identical data.
+
+- **ModernLayoutLM**: this model, fine-tuned with boxes.
+- **LayoutLMv1**: `microsoft/layoutlm-base-uncased`, fine-tuned with boxes.
+- **ModernLayoutLM (no-pretrain)**: the same architecture fine-tuned with boxes straight from ModernBERT, skipping the layout pretraining.
+- **ModernLayoutLM (no-bbox)**: this model fine-tuned with every box at the origin.
+- **ModernBERT**: `answerdotai/ModernBERT-base` fine-tuned on text alone.
+
+| setting | value |
+|---|---|
+| metric | entity F1, and field-level Hmean per ICDAR task 3 on SROIE |
+| seeds | mean ± standard deviation over three seeds |
+| split | FUNSD on test as it has no validation split, the rest on validation |
+| learning rate | 3e-4, chosen on FUNSD test, and 5e-5 for LayoutLMv1 as in its paper |
+| epochs | 30 |
+| window | 1024 tokens, 512 for LayoutLMv1 as its positions cap there |
+| long documents | chunks are stitched back into one document before scoring |
+
+| dataset | ModernLayoutLM | LayoutLMv1 | ModernLayoutLM (no-pretrain) | ModernLayoutLM (no-bbox) | ModernBERT |
+|---|---|---|---|---|---|
+| FUNSD | 0.7565 ± 0.0037 | 0.7785 ± 0.0104 | 0.5939 ± 0.0205 | 0.3000 ± 0.0247 | 0.5884 ± 0.0082 |
+| CORD | 0.9546 ± 0.0023 | 0.9685 ± 0.0037 | 0.9462 ± 0.0050 | 0.9312 ± 0.0042 | 0.9424 ± 0.0006 |
+| SROIE | 0.9337 ± 0.0060 | 0.9223 ± 0.0199 | 0.8883 ± 0.0121 | 0.8411 ± 0.0189 | 0.8955 ± 0.0072 |
+
+ModernLayoutLM performs comparably to LayoutLMv1, but natively supports long context.
+
+## Citation
+
+Xu, Y., Li, M., Cui, L., Huang, S., Wei, F., & Zhou, M. (2020). LayoutLM: Pre-training of Text and Layout for Document Image Understanding. In *Proceedings of the 26th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining* (pp. 1192-1200). https://doi.org/10.1145/3394486.3403172
+
+Warner, B., Chaffin, A., Clavié, B., Weller, O., Hallström, O., Taghadouini, S., Gallagher, A., Biswas, R., Ladhak, F., Aarsen, T., Cooper, N., Adams, G., Howard, J., & Poli, I. (2024). Smarter, Better, Faster, Longer: A Modern Bidirectional Encoder for Fast, Memory Efficient, and Long Context Finetuning and Inference. *arXiv preprint arXiv:2412.13663*. https://arxiv.org/abs/2412.13663
+
+```bibtex
+@inproceedings{xu2020layoutlm,
+  title={LayoutLM: Pre-training of Text and Layout for Document Image Understanding},
+  author={Xu, Yiheng and Li, Minghao and Cui, Lei and Huang, Shaohan and Wei, Furu and Zhou, Ming},
+  booktitle={Proceedings of the 26th ACM SIGKDD International Conference on Knowledge Discovery \& Data Mining},
+  pages={1192--1200},
+  year={2020},
+  publisher={ACM},
+  doi={10.1145/3394486.3403172}
+}
+
+@misc{warner2024smarterbetterfasterlonger,
+  title={Smarter, Better, Faster, Longer: A Modern Bidirectional Encoder for Fast, Memory Efficient, and Long Context Finetuning and Inference},
+  author={Benjamin Warner and Antoine Chaffin and Benjamin Clavi{\'e} and Orion Weller and Oskar Hallstr{\"o}m and Said Taghadouini and Alexis Gallagher and Raja Biswas and Faisal Ladhak and Tom Aarsen and Nathan Cooper and Griffin Adams and Jeremy Howard and Iacopo Poli},
+  year={2024},
+  eprint={2412.13663},
+  archivePrefix={arXiv},
+  primaryClass={cs.CL},
+  url={https://arxiv.org/abs/2412.13663}
+}
+```
+
+## License
+
+Apache 2.0, the license ModernBERT is released under, see `LICENSE`.

config.json

@@ -0,0 +1,87 @@
+{
+  "architectures": [
+    "ModernLayoutLmForMaskedLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_modernlayoutlm.ModernLayoutLmConfig",
+    "AutoModel": "modeling_modernlayoutlm.ModernLayoutLmModel",
+    "AutoModelForMaskedLM": "modeling_modernlayoutlm.ModernLayoutLmForMaskedLM",
+    "AutoModelForSequenceClassification": "modeling_modernlayoutlm.ModernLayoutLmForSequenceClassification",
+    "AutoModelForTokenClassification": "modeling_modernlayoutlm.ModernLayoutLmForTokenClassification"
+  },
+  "bos_token_id": 50281,
+  "classifier_activation": "gelu",
+  "classifier_bias": false,
+  "classifier_dropout": 0.0,
+  "classifier_pooling": "mean",
+  "cls_token_id": 50281,
+  "coordinate_scale": 1000,
+  "decoder_bias": true,
+  "deterministic_flash_attn": false,
+  "dtype": "float32",
+  "embedding_dropout": 0.0,
+  "eos_token_id": 50282,
+  "global_attn_every_n_layers": 3,
+  "gradient_checkpointing": false,
+  "hidden_activation": "gelu",
+  "hidden_size": 768,
+  "initializer_cutoff_factor": 2.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 1152,
+  "layer_norm_eps": 1e-05,
+  "layer_types": [
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "local_attention": 128,
+  "max_2d_position_embeddings": 1024,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "mlp_dropout": 0.0,
+  "model_type": "modernlayoutlm",
+  "norm_bias": false,
+  "norm_eps": 1e-05,
+  "num_attention_heads": 12,
+  "num_hidden_layers": 22,
+  "pad_token_id": 50283,
+  "position_embedding_type": "absolute",
+  "rope_parameters": {
+    "full_attention": {
+      "rope_theta": 160000.0,
+      "rope_type": "default"
+    },
+    "sliding_attention": {
+      "rope_theta": 10000.0,
+      "rope_type": "default"
+    }
+  },
+  "sep_token_id": 50282,
+  "sparse_pred_ignore_index": -100,
+  "sparse_prediction": false,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.1",
+  "vocab_size": 50368,
+  "zero_init_2d": true
+}

configuration_modernlayoutlm.py

@@ -0,0 +1,53 @@
+"""
+Configuration for ModernLayoutLM
+
+Extends ModernBertConfig with a coordinate vocabulary size and inherits the backbone untouched
+This module is copied into exported model directories and may import only from transformers
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from transformers.models.modernbert.configuration_modernbert import ModernBertConfig
+
+# Size to COORDINATE_SCALE + 1 rows minimum, 1024 matches LayoutLM
+DEFAULT_MAX_2D_POSITION_EMBEDDINGS = 1024
+DEFAULT_COORDINATE_SCALE = 1000
+
+
+class ModernLayoutLmConfig(ModernBertConfig):
+    """
+    Configuration for a ModernLayoutLM model
+
+    Args:
+        max_2d_position_embeddings: Rows per 2D table, which lookups clamp coordinates into.
+        coordinate_scale: The scale boxes are normalized to, kept here so the processor agrees.
+        zero_init_2d: Zero the layout tables, so a fresh graft matches its backbone exactly.
+        **kwargs: Forwarded to `ModernBertConfig`.
+    """
+
+    model_type = "modernlayoutlm"
+
+    def __init__(
+        self,
+        max_2d_position_embeddings: int = DEFAULT_MAX_2D_POSITION_EMBEDDINGS,
+        coordinate_scale: int = DEFAULT_COORDINATE_SCALE,
+        zero_init_2d: bool = True,
+        **kwargs: Any,
+    ) -> None:
+        # Set on the class as an instance "model_type" breaks Auto dispatch
+        kwargs.pop("model_type", None)
+        super().__init__(**kwargs)
+        if coordinate_scale >= max_2d_position_embeddings:
+            raise ValueError(
+                f"coordinate_scale {coordinate_scale} needs "
+                f"max_2d_position_embeddings > {coordinate_scale}, "
+                f"got {max_2d_position_embeddings}"
+            )
+        self.max_2d_position_embeddings = max_2d_position_embeddings
+        self.coordinate_scale = coordinate_scale
+        self.zero_init_2d = zero_init_2d
+
+
+__all__ = ["ModernLayoutLmConfig"]

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3934cf273741df0fae5d60e68093386c24b254c2e1bfd3c7acf2cc3ace4937b8
+size 611218424

modeling_modernlayoutlm.py

@@ -0,0 +1,425 @@
+"""
+ModernLayoutLM: LayoutLM's 2D layout embeddings summed into a ModernBERT backbone
+
+The graft is confined to the embedding module so attention keeps its fused kernels and RoPE
+This module is copied into exported model directories and may import only torch and transformers
+"""
+
+from __future__ import annotations
+
+from typing import Any, cast
+
+import torch
+from torch import nn
+from transformers.masking_utils import (
+    create_bidirectional_mask,
+    create_bidirectional_sliding_window_mask,
+)
+from transformers.modeling_outputs import (
+    BaseModelOutput,
+    MaskedLMOutput,
+    SequenceClassifierOutput,
+    TokenClassifierOutput,
+)
+from transformers.models.modernbert.modeling_modernbert import (
+    ModernBertEmbeddings,
+    ModernBertEncoderLayer,
+    ModernBertForMaskedLM,
+    ModernBertForSequenceClassification,
+    ModernBertForTokenClassification,
+    ModernBertModel,
+    ModernBertPreTrainedModel,
+)
+from transformers.utils.auto_docstring import auto_docstring
+from transformers.utils.generic import merge_with_config_defaults
+from transformers.utils.output_capturing import capture_outputs
+
+from .configuration_modernlayoutlm import ModernLayoutLmConfig
+
+# Bounding boxes arrive as (x0, y0, x1, y1) with the origin at the top left corner
+BBOX_COORDS = 4
+
+
+class LayoutEmbedding(nn.Embedding):
+    """
+    A coordinate embedding table, marked by its type so weight init can find it
+
+    `_init_weights` sees leaf modules, where a plain `nn.Embedding` would pass for the token table.
+    It would then take ModernBERT's default normal init instead of being zeroed.
+    """
+
+
+class ModernLayoutLmEmbeddings(ModernBertEmbeddings):
+    """
+    ModernBERT's embeddings plus LayoutLM's four 2D position embedding tables
+
+    Args:
+        config: The model config, supplying `max_2d_position_embeddings` and `zero_init_2d`.
+    """
+
+    def __init__(self, config: ModernLayoutLmConfig) -> None:
+        super().__init__(config)
+        max_2d = config.max_2d_position_embeddings
+        self.x_position_embeddings = LayoutEmbedding(max_2d, config.hidden_size)
+        self.y_position_embeddings = LayoutEmbedding(max_2d, config.hidden_size)
+        self.w_position_embeddings = LayoutEmbedding(max_2d, config.hidden_size)
+        self.h_position_embeddings = LayoutEmbedding(max_2d, config.hidden_size)
+        if getattr(config, "zero_init_2d", True):
+            self.reset_layout_parameters()
+
+    def reset_layout_parameters(self) -> None:
+        """
+        Zeroes the four layout tables so the model reduces exactly to its ModernBERT backbone
+        """
+        for table in self.layout_tables():
+            nn.init.zeros_(table.weight)
+
+    def layout_tables(self) -> list[LayoutEmbedding]:
+        """
+        Returns the four 2D position embedding tables
+
+        Used by the optimizer to give the freshly grafted parameters their own learning rate.
+
+        Returns:
+            The x, y, w and h embedding tables, in that order.
+        """
+        return [
+            self.x_position_embeddings,
+            self.y_position_embeddings,
+            self.w_position_embeddings,
+            self.h_position_embeddings,
+        ]
+
+    def layout_embeddings(self, bbox: torch.Tensor) -> torch.Tensor:
+        """
+        Looks up and sums the six layout embeddings for each box
+
+        Coordinates clamp into the table, and a negative width or height clamps to zero.
+        A malformed box therefore degrades rather than raising.
+        `x0` and `x1` share the X table and `y0` and `y1` the Y table, matching LayoutLM v1.
+
+        Args:
+            bbox: Long tensor of shape (batch, seq_len, 4) holding (x0, y0, x1, y1).
+
+        Returns:
+            A tensor of shape (batch, seq_len, hidden_size).
+
+        Raises:
+            ValueError: If the last dimension of `bbox` is not 4.
+        """
+        if bbox.shape[-1] != BBOX_COORDS:
+            raise ValueError(f"bbox must have last dimension {BBOX_COORDS}, got {bbox.shape[-1]}")
+
+        limit = self.x_position_embeddings.num_embeddings - 1
+        boxes = bbox.long().clamp(0, limit)
+        x0, y0, x1, y1 = boxes.unbind(dim=-1)
+        width = (x1 - x0).clamp(0, limit)
+        height = (y1 - y0).clamp(0, limit)
+
+        return (
+            self.x_position_embeddings(x0)
+            + self.y_position_embeddings(y0)
+            + self.x_position_embeddings(x1)
+            + self.y_position_embeddings(y1)
+            + self.w_position_embeddings(width)
+            + self.h_position_embeddings(height)
+        )
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        inputs_embeds: torch.Tensor | None = None,
+        bbox: torch.Tensor | None = None,
+    ) -> torch.Tensor:
+        """
+        Sums token and layout embeddings, then applies the inherited LayerNorm and dropout
+
+        Args:
+            input_ids: Token ids of shape (batch, seq_len).
+            bbox: Boxes of shape (batch, seq_len, 4), or None to behave as plain ModernBERT.
+            inputs_embeds: Precomputed token embeddings, used instead of `input_ids`.
+
+        Returns:
+            Normalized embeddings of shape (batch, seq_len, hidden_size).
+        """
+        if inputs_embeds is not None:
+            hidden_states = inputs_embeds
+        else:
+            hidden_states = self.tok_embeddings(input_ids)
+        if bbox is not None:
+            hidden_states = hidden_states + self.layout_embeddings(bbox)
+        return self.drop(self.norm(hidden_states))
+
+
+class ModernLayoutLmPreTrainedModel(ModernBertPreTrainedModel):
+    """
+    Shared base class, extending ModernBERT's weight initialization to the layout tables
+    """
+
+    config_class = ModernLayoutLmConfig
+    base_model_prefix = "model"
+    _no_split_modules = ["ModernLayoutLmEmbeddings", "ModernBertEncoderLayer"]
+
+    @torch.no_grad()
+    def _init_weights(self, module: nn.Module) -> None:
+        """
+        Initializes a module, adding layout table handling on top of ModernBERT's rules
+
+        ModernBERT's default init is far too large to add onto a pretrained token embedding.
+        `from_pretrained` calls this for keys a checkpoint lacks, so a graft starts zeroed.
+
+        Args:
+            module: The module to initialize.
+        """
+        super()._init_weights(module)
+        if isinstance(module, LayoutEmbedding):
+            if getattr(self.config, "zero_init_2d", True):
+                nn.init.zeros_(module.weight)
+            else:
+                nn.init.normal_(module.weight, mean=0.0, std=self.config.initializer_range)
+
+
+@auto_docstring
+class ModernLayoutLmModel(ModernLayoutLmPreTrainedModel, ModernBertModel):
+    """
+    The bare ModernLayoutLM encoder
+
+    Identical to `ModernBertModel` except that `bbox` is threaded into the embedding module.
+    """
+
+    def __init__(self, config: ModernLayoutLmConfig) -> None:
+        super().__init__(config)
+        self.embeddings = ModernLayoutLmEmbeddings(config)
+        self.post_init()
+
+    @merge_with_config_defaults
+    @capture_outputs
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.Tensor | None = None,
+        inputs_embeds: torch.Tensor | None = None,
+        bbox: torch.Tensor | None = None,
+        **kwargs: Any,
+    ) -> BaseModelOutput:
+        r"""
+        bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*):
+            Boxes as (x0, y0, x1, y1) on `[0, config.coordinate_scale]`, origin at the top left.
+            Omitting it makes the model behave as plain ModernBERT.
+        """
+        # Branched rather than written as an xor so that the non-None case is provable
+        if inputs_embeds is not None:
+            if input_ids is not None:
+                raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
+            seq_len, device = inputs_embeds.shape[1], inputs_embeds.device
+        elif input_ids is not None:
+            seq_len, device = input_ids.shape[1], input_ids.device
+        else:
+            raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
+
+        if position_ids is None:
+            position_ids = torch.arange(seq_len, device=device).unsqueeze(0)
+
+        hidden_states = self.embeddings(
+            input_ids=input_ids,
+            bbox=bbox,
+            inputs_embeds=inputs_embeds,
+        )
+
+        if not isinstance(attention_mask_mapping := attention_mask, dict):
+            mask_kwargs = {
+                "config": self.config,
+                "inputs_embeds": hidden_states,
+                "attention_mask": attention_mask,
+            }
+            attention_mask_mapping = {
+                "full_attention": create_bidirectional_mask(**mask_kwargs),
+                "sliding_attention": create_bidirectional_sliding_window_mask(**mask_kwargs),
+            }
+
+        layer_types: list[str] = self.config.layer_types or []
+        position_embeddings = {}
+        for layer_type in set(layer_types):
+            position_embeddings[layer_type] = self.rotary_emb(
+                hidden_states, position_ids, layer_type
+            )
+
+        # Cast because nn.Module.__getattr__ erases the element type of self.layers
+        for module in self.layers:
+            encoder_layer = cast(ModernBertEncoderLayer, module)
+            attention_type = encoder_layer.attention_type
+            hidden_states = encoder_layer(
+                hidden_states,
+                attention_mask=attention_mask_mapping[attention_type],
+                position_embeddings=position_embeddings[attention_type],
+                **kwargs,
+            )
+
+        hidden_states = self.final_norm(hidden_states)
+
+        return BaseModelOutput(last_hidden_state=hidden_states)
+
+
+@auto_docstring
+class ModernLayoutLmForMaskedLM(ModernLayoutLmPreTrainedModel, ModernBertForMaskedLM):
+    """
+    ModernLayoutLM with a masked language modeling head, for the layout-aware MLM stage
+    """
+
+    _tied_weights_keys = {"decoder.weight": "model.embeddings.tok_embeddings.weight"}
+
+    def __init__(self, config: ModernLayoutLmConfig) -> None:
+        super().__init__(config)
+        self.model = ModernLayoutLmModel(config)
+        self.post_init()
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.Tensor | None = None,
+        inputs_embeds: torch.Tensor | None = None,
+        labels: torch.Tensor | None = None,
+        bbox: torch.Tensor | None = None,
+        **kwargs: Any,
+    ) -> tuple[torch.Tensor] | MaskedLMOutput:
+        """
+        Runs the encoder and the MLM head
+
+        `bbox` is named rather than left to `**kwargs`, as `Trainer` prunes columns by signature.
+        Were it implicit, the boxes would be dropped before they ever reached the model.
+
+        Args:
+            input_ids: Token ids of shape (batch, seq_len).
+            attention_mask: Mask of shape (batch, seq_len).
+            bbox: Boxes of shape (batch, seq_len, 4), normalized to the coordinate scale.
+            position_ids: Optional explicit 1D positions.
+            inputs_embeds: Precomputed token embeddings, used instead of `input_ids`.
+            labels: MLM targets of shape (batch, seq_len), `-100` where unmasked.
+            **kwargs: Forwarded to the backbone.
+
+        Returns:
+            A MaskedLMOutput, or a tuple when the config asks for one.
+        """
+        return super().forward(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            inputs_embeds=inputs_embeds,
+            labels=labels,
+            # Ignore as the ModernBert head forwards bbox opaquely and pyright loses it
+            bbox=bbox,  # pyright: ignore[reportCallIssue]
+            **kwargs,
+        )
+
+
+@auto_docstring
+class ModernLayoutLmForTokenClassification(
+    ModernLayoutLmPreTrainedModel, ModernBertForTokenClassification
+):
+    """
+    ModernLayoutLM with a token classification head, for form and receipt entity extraction
+    """
+
+    def __init__(self, config: ModernLayoutLmConfig) -> None:
+        super().__init__(config)
+        self.model = ModernLayoutLmModel(config)
+        self.post_init()
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.Tensor | None = None,
+        inputs_embeds: torch.Tensor | None = None,
+        labels: torch.Tensor | None = None,
+        bbox: torch.Tensor | None = None,
+        **kwargs: Any,
+    ) -> tuple[torch.Tensor] | TokenClassifierOutput:
+        """
+        Runs the encoder and the per-token classifier
+
+        Args:
+            input_ids: Token ids of shape (batch, seq_len).
+            attention_mask: Mask of shape (batch, seq_len).
+            bbox: Boxes of shape (batch, seq_len, 4), normalized to the coordinate scale.
+            position_ids: Optional explicit 1D positions.
+            inputs_embeds: Precomputed token embeddings, used instead of `input_ids`.
+            labels: Class index per token, `-100` to ignore.
+            **kwargs: Forwarded to the backbone.
+
+        Returns:
+            A TokenClassifierOutput, or a tuple when the config asks for one.
+        """
+        return super().forward(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            inputs_embeds=inputs_embeds,
+            labels=labels,
+            # Ignore as the ModernBert head forwards bbox opaquely and pyright loses it
+            bbox=bbox,  # pyright: ignore[reportCallIssue]
+            **kwargs,
+        )
+
+
+@auto_docstring
+class ModernLayoutLmForSequenceClassification(
+    ModernLayoutLmPreTrainedModel, ModernBertForSequenceClassification
+):
+    """
+    ModernLayoutLM with a pooled sequence classification head, for document classification
+    """
+
+    def __init__(self, config: ModernLayoutLmConfig) -> None:
+        super().__init__(config)
+        self.model = ModernLayoutLmModel(config)
+        self.post_init()
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.Tensor | None = None,
+        inputs_embeds: torch.Tensor | None = None,
+        labels: torch.Tensor | None = None,
+        bbox: torch.Tensor | None = None,
+        **kwargs: Any,
+    ) -> tuple[torch.Tensor] | SequenceClassifierOutput:
+        """
+        Runs the encoder, pools per the config, then classifies the document
+
+        Args:
+            input_ids: Token ids of shape (batch, seq_len).
+            attention_mask: Mask of shape (batch, seq_len).
+            bbox: Boxes of shape (batch, seq_len, 4), normalized to the coordinate scale.
+            position_ids: Optional explicit 1D positions.
+            inputs_embeds: Precomputed token embeddings, used instead of `input_ids`.
+            labels: One class index per document.
+            **kwargs: Forwarded to the backbone.
+
+        Returns:
+            A SequenceClassifierOutput, or a tuple when the config asks for one.
+        """
+        return super().forward(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            inputs_embeds=inputs_embeds,
+            labels=labels,
+            # Ignore as the ModernBert head forwards bbox opaquely and pyright loses it
+            bbox=bbox,  # pyright: ignore[reportCallIssue]
+            **kwargs,
+        )
+
+
+__all__ = [
+    "ModernLayoutLmEmbeddings",
+    "ModernLayoutLmForMaskedLM",
+    "ModernLayoutLmForSequenceClassification",
+    "ModernLayoutLmForTokenClassification",
+    "ModernLayoutLmModel",
+    "ModernLayoutLmPreTrainedModel",
+]

processing_modernlayoutlm.py

@@ -0,0 +1,255 @@
+"""
+Turning words and boxes into model inputs
+
+There is no image tower, so this only tokenizes words and broadcasts each box across its subwords
+This module is copied into exported model directories and may import only torch and transformers
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from transformers.processing_utils import ProcessorMixin
+from transformers.tokenization_utils_base import BatchEncoding
+
+# Follow LayoutLM: CLS at the origin, SEP at the far corner, padding masked out
+CLS_BOX = (0, 0, 0, 0)
+SEP_BOX = (1000, 1000, 1000, 1000)
+PAD_BOX = (0, 0, 0, 0)
+
+# Label id meaning "do not compute loss here", the torch cross-entropy default
+IGNORE_INDEX = -100
+
+
+class ModernLayoutLmProcessor(ProcessorMixin):
+    """
+    Pairs a fast tokenizer with box and label expansion over subwords
+
+    Args:
+        tokenizer: Any fast tokenizer, so the LayoutLM baseline reuses this with its own one.
+        coordinate_scale: The scale boxes are expected to already be normalized to.
+        label_all_subwords: Label every subword, not only the first as published FUNSD and CORD do.
+    """
+
+    attributes = ["tokenizer"]
+    tokenizer_class = "AutoTokenizer"
+
+    def __init__(
+        self,
+        tokenizer: Any,
+        coordinate_scale: int = 1000,
+        label_all_subwords: bool = False,
+    ) -> None:
+        self.coordinate_scale = coordinate_scale
+        self.label_all_subwords = label_all_subwords
+        super().__init__(tokenizer)
+
+    # Diverge as ProcessorMixin.__call__ is typed for image and audio processors
+    def __call__(  # pyright: ignore[reportIncompatibleMethodOverride]
+        self,
+        words: list[str] | list[list[str]],
+        boxes: list[list[int]] | list[list[list[int]]],
+        word_labels: list[int] | list[list[int]] | None = None,
+        max_length: int | None = None,
+        padding: bool | str = "max_length",
+        truncation: bool = True,
+        return_overflowing_tokens: bool = False,
+        stride: int = 0,
+        return_tensors: str | None = None,
+        **kwargs: Any,
+    ) -> BatchEncoding:
+        """
+        Tokenizes words and aligns boxes, and optionally labels, to the resulting subwords
+
+        Accepts one document or a batch, told apart by whether `words` holds strings or lists.
+        Special tokens receive the sentinel boxes defined in this module.
+        With `return_overflowing_tokens` a long page becomes several chunks, each with its boxes.
+
+        Args:
+            words: One document's words, or a batch of documents' words.
+            boxes: One (x0, y0, x1, y1) box per word on `coordinate_scale`, batched like `words`.
+            word_labels: One class index per word, batched like `words`, or None for no labels.
+            max_length: Truncation length, by default the tokenizer's own.
+            padding: Passed to the tokenizer.
+            truncation: Passed to the tokenizer.
+            return_overflowing_tokens: Emit every chunk of an over-length page, not only the first.
+            stride: Overlap in tokens between consecutive chunks.
+            return_tensors: "pt" for torch tensors, None for lists.
+            **kwargs: Forwarded to the tokenizer.
+
+        Returns:
+            A BatchEncoding of `input_ids`, `attention_mask`, `bbox`, plus `labels` if given any.
+
+        Raises:
+            ValueError: If `words` and `boxes` differ in length, or a coordinate leaves the grid.
+        """
+        batched = bool(words) and isinstance(words[0], list)
+        batch_words: list[list[str]] = words if batched else [words]  # type: ignore[assignment]
+        batch_boxes: list[list[list[int]]] = boxes if batched else [boxes]  # type: ignore[assignment]
+        batch_labels: list[list[int]] | None = None
+        if word_labels is not None:
+            batch_labels = word_labels if batched else [word_labels]  # type: ignore[assignment]
+
+        self._validate(batch_words, batch_boxes, batch_labels)
+
+        encoding = self.tokenizer(
+            text=batch_words,
+            is_split_into_words=True,
+            max_length=max_length,
+            padding=padding,
+            truncation=truncation,
+            return_overflowing_tokens=return_overflowing_tokens,
+            stride=stride,
+            return_tensors=None,
+            **kwargs,
+        )
+
+        sample_map = encoding.get("overflow_to_sample_mapping")
+        if sample_map is None:
+            sample_map = list(range(len(encoding["input_ids"])))
+
+        aligned_boxes: list[list[list[int]]] = []
+        aligned_labels: list[list[int]] = []
+        for chunk_index, sample_index in enumerate(sample_map):
+            word_ids = encoding.word_ids(chunk_index)
+            source_boxes = batch_boxes[sample_index]
+            aligned_boxes.append(self._expand_boxes(word_ids, source_boxes))
+            if batch_labels is not None:
+                aligned_labels.append(self._expand_labels(word_ids, batch_labels[sample_index]))
+
+        encoding["bbox"] = aligned_boxes
+        if batch_labels is not None:
+            encoding["labels"] = aligned_labels
+
+        if return_tensors is not None:
+            encoding.convert_to_tensors(tensor_type=return_tensors)
+        return encoding
+
+    def _validate(
+        self,
+        batch_words: list[list[str]],
+        batch_boxes: list[list[list[int]]],
+        batch_labels: list[list[int]] | None,
+    ) -> None:
+        """
+        Checks that the parallel inputs line up and the coordinates are in range
+
+        A misalignment would otherwise train a model on wrong boxes without ever erroring.
+
+        Args:
+            batch_words: Words per document.
+            batch_boxes: Boxes per document.
+            batch_labels: Labels per document, or None.
+
+        Raises:
+            ValueError: On a length mismatch or an out of range coordinate.
+        """
+        if len(batch_words) != len(batch_boxes):
+            raise ValueError(f"Got {len(batch_words)} word lists but {len(batch_boxes)} box lists")
+        if batch_labels is not None and len(batch_labels) != len(batch_words):
+            raise ValueError(
+                f"Got {len(batch_words)} word lists but {len(batch_labels)} label lists"
+            )
+
+        for index, (doc_words, doc_boxes) in enumerate(zip(batch_words, batch_boxes, strict=True)):
+            if len(doc_words) != len(doc_boxes):
+                raise ValueError(
+                    f"Document {index} has {len(doc_words)} words but {len(doc_boxes)} boxes"
+                )
+            if batch_labels is not None and len(batch_labels[index]) != len(doc_words):
+                raise ValueError(
+                    f"Document {index} has {len(doc_words)} words "
+                    f"but {len(batch_labels[index])} labels"
+                )
+            for box in doc_boxes:
+                if len(box) != 4:
+                    raise ValueError(f"Document {index} has a box with {len(box)} coordinates")
+                if any(not 0 <= value <= self.coordinate_scale for value in box):
+                    raise ValueError(
+                        f"Document {index} has box {box} outside "
+                        f"[0, {self.coordinate_scale}]. Normalize before calling the processor."
+                    )
+
+    def _expand_boxes(
+        self,
+        word_ids: list[int | None],
+        source_boxes: list[list[int]],
+    ) -> list[list[int]]:
+        """
+        Assigns a box to every token position in one chunk
+
+        Args:
+            word_ids: Per-token index into `source_boxes`, None for special and padding tokens.
+            source_boxes: One box per word of the source document.
+
+        Returns:
+            One box per token position.
+        """
+        special_boxes = self._special_boxes(word_ids)
+        return [
+            list(source_boxes[word_id]) if word_id is not None else special_boxes[position]
+            for position, word_id in enumerate(word_ids)
+        ]
+
+    def _special_boxes(self, word_ids: list[int | None]) -> dict[int, list[int]]:
+        """
+        Chooses a sentinel box for each token that came from no word
+
+        Leading specials sit at the origin like CLS, and the first trailing one at the far corner.
+        Everything past it is padding at the origin.
+
+        Args:
+            word_ids: Per-token index into the source words, None for non-word tokens.
+
+        Returns:
+            A mapping from token position to sentinel box.
+        """
+        word_positions = [i for i, word_id in enumerate(word_ids) if word_id is not None]
+        last_word = word_positions[-1] if word_positions else -1
+
+        boxes: dict[int, list[int]] = {}
+        seen_separator = False
+        for position, word_id in enumerate(word_ids):
+            if word_id is not None:
+                continue
+            if position < last_word:
+                boxes[position] = list(CLS_BOX)
+            elif not seen_separator:
+                boxes[position] = list(SEP_BOX)
+                seen_separator = True
+            else:
+                boxes[position] = list(PAD_BOX)
+        return boxes
+
+    def _expand_labels(
+        self,
+        word_ids: list[int | None],
+        source_labels: list[int],
+    ) -> list[int]:
+        """
+        Assigns a label to every token position in one chunk
+
+        Special and padding tokens always get `IGNORE_INDEX`.
+        Whether the later subwords of a word inherit its label follows `label_all_subwords`.
+
+        Args:
+            word_ids: Per-token index into `source_labels`, None for non-word tokens.
+            source_labels: One class index per word of the source document.
+
+        Returns:
+            One label per token position.
+        """
+        labels: list[int] = []
+        previous = None
+        for word_id in word_ids:
+            if word_id is None:
+                labels.append(IGNORE_INDEX)
+            elif word_id != previous or self.label_all_subwords:
+                labels.append(source_labels[word_id])
+            else:
+                labels.append(IGNORE_INDEX)
+            previous = word_id
+        return labels
+
+
+__all__ = ["ModernLayoutLmProcessor"]

processor_config.json

@@ -0,0 +1,8 @@
+{
+  "auto_map": {
+    "AutoProcessor": "processing_modernlayoutlm.ModernLayoutLmProcessor"
+  },
+  "coordinate_scale": 1000,
+  "label_all_subwords": false,
+  "processor_class": "ModernLayoutLmProcessor"
+}

tokenizer_config.json

@@ -0,0 +1,21 @@
+{
+  "auto_map": {
+    "AutoProcessor": "processing_modernlayoutlm.ModernLayoutLmProcessor"
+  },
+  "backend": "tokenizers",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "is_local": true,
+  "local_files_only": false,
+  "mask_token": "[MASK]",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 8192,
+  "pad_token": "[PAD]",
+  "processor_class": "ModernLayoutLmProcessor",
+  "sep_token": "[SEP]",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "[UNK]"
+}