fp16 Core ML conversion: ruBert-base Collection3 NER (PER/ORG/LOC), parity 99.98% tokens

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

NERCollection3.mlpackage/Data/com.apple.CoreML/model.mlmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e991fe9ff66b4ff017064ad4e46477b253310715962fc53596b36528779bff7
+size 142069

NERCollection3.mlpackage/Data/com.apple.CoreML/weights/weight.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ae0df0bd689486e33d45caa4d4e54104a5c33e38858019ba443ee0426b2b885
+size 355453888

NERCollection3.mlpackage/Manifest.json

@@ -0,0 +1,18 @@
+{
+    "fileFormatVersion": "1.0.0",
+    "itemInfoEntries": {
+        "0DEC83E2-F4F4-4D9A-8169-6D2A054E1819": {
+            "author": "com.apple.CoreML",
+            "description": "CoreML Model Weights",
+            "name": "weights",
+            "path": "com.apple.CoreML/weights"
+        },
+        "8AC13A95-C044-4303-93E6-8F67B8AA8342": {
+            "author": "com.apple.CoreML",
+            "description": "CoreML Model Specification",
+            "name": "model.mlmodel",
+            "path": "com.apple.CoreML/model.mlmodel"
+        }
+    },
+    "rootModelIdentifier": "8AC13A95-C044-4303-93E6-8F67B8AA8342"
+}

README.md

@@ -0,0 +1,42 @@
+---
+license: apache-2.0
+language:
+  - ru
+tags:
+  - ner
+  - token-classification
+  - coreml
+  - russian
+base_model: ai-forever/ruBert-base
+---
+
+# rubert-base-collection3-ner-coreml
+
+Russian NER (PER / ORG / LOC) as a self-contained fp16 Core ML package for on-device
+inference on Apple Silicon. Converted for the [Letopis](https://smkrv.net/letopis) macOS app
+(on-device PII redaction of call transcripts); usable by any Core ML consumer.
+
+## Lineage
+
+- Base model: [ai-forever/ruBert-base](https://huggingface.co/ai-forever/ruBert-base)
+- Fine-tune (Collection3, PER/ORG/LOC): [viktor-shcherb/sberbank-rubert-base-collection3](https://huggingface.co/viktor-shcherb/sberbank-rubert-base-collection3) (Apache-2.0); reported Collection3 test F1 0.948
+- This repo: fp16 Core ML conversion of that fine-tune, plus the WordPiece vocab and label map
+
+## Files
+
+- `NERCollection3.mlpackage` - BertForTokenClassification, fp16 mlprogram, fixed shape
+  (1, 256): inputs `input_ids` + `attention_mask` (int32), output `logits` (1, 256, 7)
+- `vocab.txt` - WordPiece vocab (do_lower_case = true, accents stripped per BERT BasicTokenizer)
+- `tokenizer_config.json` - reference tokenizer settings
+- `config.json` - `id2label`: O, B-PER, I-PER, B-ORG, I-ORG, B-LOC, I-LOC; `seq_len: 256`
+- `tokenizer_fixtures.json` - reference tokenizations (ids + char offsets) for porting tokenizers
+
+## Conversion fidelity
+
+Verified against the PyTorch original on 1500 conversational RU sentences (cased and
+lowercase/unpunctuated): token-level label agreement 99.98%, entity-span agreement 99.8%.
+
+## License
+
+Apache-2.0, inherited from the fine-tune and base model. Conversion and packaging:
+© 2026 Sergey Makarov.

config.json

@@ -0,0 +1,14 @@
+{
+ "model_type": "bert",
+ "id2label": {
+  "0": "O",
+  "1": "B-PER",
+  "2": "I-PER",
+  "3": "B-ORG",
+  "4": "I-ORG",
+  "5": "B-LOC",
+  "6": "I-LOC"
+ },
+ "max_position_embeddings": 512,
+ "seq_len": 256
+}

tokenizer_config.json

@@ -0,0 +1,58 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": true,
+  "extra_special_tokens": {},
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

tokenizer_fixtures.json

@@ -0,0 +1,1409 @@
+[
+ {
+  "text": "Добрый день, меня зовут Колесникова Мария, чем могу помочь?",
+  "ids": [
+   48311,
+   378,
+   1336,
+   121,
+   1024,
+   10160,
+   23759,
+   9511,
+   14982,
+   390,
+   121,
+   1009,
+   1385,
+   4881,
+   161
+  ],
+  "tokens": [
+   "добры",
+   "##и",
+   "день",
+   ",",
+   "меня",
+   "зовут",
+   "колес",
+   "##никова",
+   "мари",
+   "##я",
+   ",",
+   "чем",
+   "могу",
+   "помочь",
+   "?"
+  ],
+  "offsets": [
+   [
+    0,
+    5
+   ],
+   [
+    5,
+    6
+   ],
+   [
+    7,
+    11
+   ],
+   [
+    11,
+    12
+   ],
+   [
+    13,
+    17
+   ],
+   [
+    18,
+    23
+   ],
+   [
+    24,
+    29
+   ],
+   [
+    29,
+    35
+   ],
+   [
+    36,
+    40
+   ],
+   [
+    40,
+    41
+   ],
+   [
+    41,
+    42
+   ],
+   [
+    43,
+    46
+   ],
+   [
+    47,
+    51
+   ],
+   [
+    52,
+    58
+   ],
+   [
+    58,
+    59
+   ]
+  ]
+ },
+ {
+  "text": "Счёт выставлен на ООО \"ПК МОРОШКА ДВ\", оплата ожидается до пятницы.",
+  "ids": [
+   3189,
+   65269,
+   660,
+   83963,
+   152,
+   117,
+   385,
+   10867,
+   1780,
+   116,
+   384,
+   152,
+   121,
+   45616,
+   11114,
+   708,
+   39716,
+   126
+  ],
+  "tokens": [
+   "счет",
+   "выставлен",
+   "на",
+   "ооо",
+   "\"",
+   "п",
+   "##к",
+   "моро",
+   "##шка",
+   "д",
+   "##в",
+   "\"",
+   ",",
+   "оплата",
+   "ожидается",
+   "до",
+   "пятницы",
+   "."
+  ],
+  "offsets": [
+   [
+    0,
+    4
+   ],
+   [
+    5,
+    14
+   ],
+   [
+    15,
+    17
+   ],
+   [
+    18,
+    21
+   ],
+   [
+    22,
+    23
+   ],
+   [
+    23,
+    24
+   ],
+   [
+    24,
+    25
+   ],
+   [
+    26,
+    30
+   ],
+   [
+    30,
+    33
+   ],
+   [
+    34,
+    35
+   ],
+   [
+    35,
+    36
+   ],
+   [
+    36,
+    37
+   ],
+   [
+    37,
+    38
+   ],
+   [
+    39,
+    45
+   ],
+   [
+    46,
+    55
+   ],
+   [
+    56,
+    58
+   ],
+   [
+    59,
+    66
+   ],
+   [
+    66,
+    67
+   ]
+  ]
+ },
+ {
+  "text": "Мы подключили Game Park (МагазинВидеоИгр ООО) к облачной кассе.",
+  "ids": [
+   945,
+   100208,
+   108282,
+   41337,
+   486,
+   160,
+   10001,
+   79511,
+   378,
+   12980,
+   83963,
+   158,
+   114,
+   87079,
+   378,
+   54650,
+   126
+  ],
+  "tokens": [
+   "мы",
+   "подключили",
+   "game",
+   "par",
+   "##k",
+   "(",
+   "магазин",
+   "##видео",
+   "##и",
+   "##гр",
+   "ооо",
+   ")",
+   "к",
+   "облачно",
+   "##и",
+   "кассе",
+   "."
+  ],
+  "offsets": [
+   [
+    0,
+    2
+   ],
+   [
+    3,
+    13
+   ],
+   [
+    14,
+    18
+   ],
+   [
+    19,
+    22
+   ],
+   [
+    22,
+    23
+   ],
+   [
+    24,
+    25
+   ],
+   [
+    25,
+    32
+   ],
+   [
+    32,
+    37
+   ],
+   [
+    37,
+    38
+   ],
+   [
+    38,
+    40
+   ],
+   [
+    41,
+    44
+   ],
+   [
+    44,
+    45
+   ],
+   [
+    46,
+    47
+   ],
+   [
+    48,
+    55
+   ],
+   [
+    55,
+    56
+   ],
+   [
+    57,
+    62
+   ],
+   [
+    62,
+    63
+   ]
+  ]
+ },
+ {
+  "text": "ИП Мкртчян Елена Тухаировна, ИНН 772456789012, телефон +7 (912) 345-67-89.",
+  "ids": [
+   90161,
+   119309,
+   1246,
+   398,
+   1731,
+   13475,
+   669,
+   92674,
+   377,
+   27009,
+   10239,
+   669,
+   121,
+   880,
+   379,
+   9444,
+   9968,
+   44295,
+   38383,
+   19359,
+   12626,
+   121,
+   5431,
+   250,
+   182,
+   160,
+   14013,
+   420,
+   158,
+   71951,
+   133,
+   9807,
+   133,
+   12447,
+   126
+  ],
+  "tokens": [
+   "ип",
+   "мк",
+   "##рт",
+   "##ч",
+   "##ян",
+   "еле",
+   "##на",
+   "тух",
+   "##а",
+   "##ир",
+   "##ов",
+   "##на",
+   ",",
+   "ин",
+   "##н",
+   "77",
+   "##24",
+   "##56",
+   "##78",
+   "##90",
+   "##12",
+   ",",
+   "телефон",
+   "+",
+   "7",
+   "(",
+   "91",
+   "##2",
+   ")",
+   "345",
+   "-",
+   "67",
+   "-",
+   "89",
+   "."
+  ],
+  "offsets": [
+   [
+    0,
+    2
+   ],
+   [
+    3,
+    5
+   ],
+   [
+    5,
+    7
+   ],
+   [
+    7,
+    8
+   ],
+   [
+    8,
+    10
+   ],
+   [
+    11,
+    14
+   ],
+   [
+    14,
+    16
+   ],
+   [
+    17,
+    20
+   ],
+   [
+    20,
+    21
+   ],
+   [
+    21,
+    23
+   ],
+   [
+    23,
+    25
+   ],
+   [
+    25,
+    27
+   ],
+   [
+    27,
+    28
+   ],
+   [
+    29,
+    31
+   ],
+   [
+    31,
+    32
+   ],
+   [
+    33,
+    35
+   ],
+   [
+    35,
+    37
+   ],
+   [
+    37,
+    39
+   ],
+   [
+    39,
+    41
+   ],
+   [
+    41,
+    43
+   ],
+   [
+    43,
+    45
+   ],
+   [
+    45,
+    46
+   ],
+   [
+    47,
+    54
+   ],
+   [
+    55,
+    56
+   ],
+   [
+    56,
+    57
+   ],
+   [
+    58,
+    59
+   ],
+   [
+    59,
+    61
+   ],
+   [
+    61,
+    62
+   ],
+   [
+    62,
+    63
+   ],
+   [
+    64,
+    67
+   ],
+   [
+    67,
+    68
+   ],
+   [
+    68,
+    70
+   ],
+   [
+    70,
+    71
+   ],
+   [
+    71,
+    73
+   ],
+   [
+    73,
+    74
+   ]
+  ]
+ },
+ {
+  "text": "Адрес: 193312, г. Санкт-Петербург, пр-кт Солидарности, д. 19, кв. 82.",
+  "ids": [
+   6546,
+   162,
+   14785,
+   12626,
+   121,
+   122,
+   126,
+   64100,
+   133,
+   9663,
+   121,
+   12417,
+   133,
+   114,
+   380,
+   36274,
+   121,
+   116,
+   126,
+   855,
+   121,
+   10650,
+   126,
+   11609,
+   126
+  ],
+  "tokens": [
+   "адрес",
+   ":",
+   "1933",
+   "##12",
+   ",",
+   "г",
+   ".",
+   "санкт",
+   "-",
+   "петербург",
+   ",",
+   "пр",
+   "-",
+   "к",
+   "##т",
+   "солидарности",
+   ",",
+   "д",
+   ".",
+   "19",
+   ",",
+   "кв",
+   ".",
+   "82",
+   "."
+  ],
+  "offsets": [
+   [
+    0,
+    5
+   ],
+   [
+    5,
+    6
+   ],
+   [
+    7,
+    11
+   ],
+   [
+    11,
+    13
+   ],
+   [
+    13,
+    14
+   ],
+   [
+    15,
+    16
+   ],
+   [
+    16,
+    17
+   ],
+   [
+    18,
+    23
+   ],
+   [
+    23,
+    24
+   ],
+   [
+    24,
+    33
+   ],
+   [
+    33,
+    34
+   ],
+   [
+    35,
+    37
+   ],
+   [
+    37,
+    38
+   ],
+   [
+    38,
+    39
+   ],
+   [
+    39,
+    40
+   ],
+   [
+    41,
+    53
+   ],
+   [
+    53,
+    54
+   ],
+   [
+    55,
+    56
+   ],
+   [
+    56,
+    57
+   ],
+   [
+    58,
+    60
+   ],
+   [
+    60,
+    61
+   ],
+   [
+    62,
+    64
+   ],
+   [
+    64,
+    65
+   ],
+   [
+    66,
+    68
+   ],
+   [
+    68,
+    69
+   ]
+  ]
+ },
+ {
+  "text": "ОБЩЕСТВО С ОГРАНИЧЕННОЙ ОТВЕТСТВЕННОСТЬЮ «СМАЙЛ ДЕНТ»",
+  "ids": [
+   7626,
+   110,
+   103859,
+   378,
+   49222,
+   151,
+   6798,
+   1181,
+   19871,
+   380,
+   150
+  ],
+  "tokens": [
+   "общество",
+   "с",
+   "ограниченно",
+   "##и",
+   "ответственностью",
+   "«",
+   "сма",
+   "##ил",
+   "ден",
+   "##т",
+   "»"
+  ],
+  "offsets": [
+   [
+    0,
+    8
+   ],
+   [
+    9,
+    10
+   ],
+   [
+    11,
+    22
+   ],
+   [
+    22,
+    23
+   ],
+   [
+    24,
+    40
+   ],
+   [
+    41,
+    42
+   ],
+   [
+    42,
+    45
+   ],
+   [
+    45,
+    47
+   ],
+   [
+    48,
+    51
+   ],
+   [
+    51,
+    52
+   ],
+   [
+    52,
+    53
+   ]
+  ]
+ },
+ {
+  "text": "передайте марии ивановне что договор готов",
+  "ids": [
+   1568,
+   42699,
+   14982,
+   378,
+   104691,
+   823,
+   691,
+   693,
+   7737,
+   3075
+  ],
+  "tokens": [
+   "переда",
+   "##ите",
+   "мари",
+   "##и",
+   "ива",
+   "##нов",
+   "##не",
+   "что",
+   "договор",
+   "готов"
+  ],
+  "offsets": [
+   [
+    0,
+    6
+   ],
+   [
+    6,
+    9
+   ],
+   [
+    10,
+    14
+   ],
+   [
+    14,
+    15
+   ],
+   [
+    16,
+    19
+   ],
+   [
+    19,
+    22
+   ],
+   [
+    22,
+    24
+   ],
+   [
+    25,
+    28
+   ],
+   [
+    29,
+    36
+   ],
+   [
+    37,
+    42
+   ]
+  ]
+ },
+ {
+  "text": "Сёмга, ёжик и Алёна Васильёва-Кузнецова приехали в Орехово-Зуево.",
+  "ids": [
+   1544,
+   741,
+   121,
+   76994,
+   385,
+   107,
+   29336,
+   669,
+   100295,
+   8983,
+   377,
+   133,
+   59637,
+   97208,
+   12166,
+   113,
+   64353,
+   375,
+   133,
+   4263,
+   19128,
+   126
+  ],
+  "tokens": [
+   "сем",
+   "##га",
+   ",",
+   "ежи",
+   "##к",
+   "и",
+   "але",
+   "##на",
+   "васи",
+   "##льев",
+   "##а",
+   "-",
+   "кузнец",
+   "##ова",
+   "приехали",
+   "в",
+   "орехов",
+   "##о",
+   "-",
+   "зу",
+   "##ево",
+   "."
+  ],
+  "offsets": [
+   [
+    0,
+    3
+   ],
+   [
+    3,
+    5
+   ],
+   [
+    5,
+    6
+   ],
+   [
+    7,
+    10
+   ],
+   [
+    10,
+    11
+   ],
+   [
+    12,
+    13
+   ],
+   [
+    14,
+    17
+   ],
+   [
+    17,
+    19
+   ],
+   [
+    20,
+    24
+   ],
+   [
+    24,
+    28
+   ],
+   [
+    28,
+    29
+   ],
+   [
+    29,
+    30
+   ],
+   [
+    30,
+    36
+   ],
+   [
+    36,
+    39
+   ],
+   [
+    40,
+    48
+   ],
+   [
+    49,
+    50
+   ],
+   [
+    51,
+    57
+   ],
+   [
+    57,
+    58
+   ],
+   [
+    58,
+    59
+   ],
+   [
+    59,
+    61
+   ],
+   [
+    61,
+    64
+   ],
+   [
+    64,
+    65
+   ]
+  ]
+ },
+ {
+  "text": "email info@servisspb.ru и сайт http://servisspb.ru",
+  "ids": [
+   43096,
+   7364,
+   27046,
+   270,
+   83848,
+   13645,
+   472,
+   483,
+   126,
+   4551,
+   107,
+   798,
+   1301,
+   10843,
+   162,
+   197,
+   197,
+   83848,
+   13645,
+   472,
+   483,
+   126,
+   4551
+  ],
+  "tokens": [
+   "em",
+   "##ail",
+   "info",
+   "@",
+   "serv",
+   "##iss",
+   "##p",
+   "##b",
+   ".",
+   "ru",
+   "и",
+   "са",
+   "##ит",
+   "http",
+   ":",
+   "/",
+   "/",
+   "serv",
+   "##iss",
+   "##p",
+   "##b",
+   ".",
+   "ru"
+  ],
+  "offsets": [
+   [
+    0,
+    2
+   ],
+   [
+    2,
+    5
+   ],
+   [
+    6,
+    10
+   ],
+   [
+    10,
+    11
+   ],
+   [
+    11,
+    15
+   ],
+   [
+    15,
+    18
+   ],
+   [
+    18,
+    19
+   ],
+   [
+    19,
+    20
+   ],
+   [
+    20,
+    21
+   ],
+   [
+    21,
+    23
+   ],
+   [
+    24,
+    25
+   ],
+   [
+    26,
+    28
+   ],
+   [
+    28,
+    30
+   ],
+   [
+    31,
+    35
+   ],
+   [
+    35,
+    36
+   ],
+   [
+    36,
+    37
+   ],
+   [
+    37,
+    38
+   ],
+   [
+    38,
+    42
+   ],
+   [
+    42,
+    45
+   ],
+   [
+    45,
+    46
+   ],
+   [
+    46,
+    47
+   ],
+   [
+    47,
+    48
+   ],
+   [
+    48,
+    50
+   ]
+  ]
+ },
+ {
+  "text": "Hello John Smith from New York, card 4276 1600 1234 5678.",
+  "ids": [
+   67124,
+   70471,
+   263,
+   30374,
+   31269,
+   10834,
+   18801,
+   52976,
+   207,
+   11267,
+   121,
+   81111,
+   90621,
+   451,
+   34196,
+   118430,
+   7889,
+   38383,
+   126
+  ],
+  "tokens": [
+   "hel",
+   "##lo",
+   "j",
+   "##ohn",
+   "sm",
+   "##ith",
+   "from",
+   "new",
+   "y",
+   "##ork",
+   ",",
+   "card",
+   "427",
+   "##6",
+   "1600",
+   "1234",
+   "56",
+   "##78",
+   "."
+  ],
+  "offsets": [
+   [
+    0,
+    3
+   ],
+   [
+    3,
+    5
+   ],
+   [
+    6,
+    7
+   ],
+   [
+    7,
+    10
+   ],
+   [
+    11,
+    13
+   ],
+   [
+    13,
+    16
+   ],
+   [
+    17,
+    21
+   ],
+   [
+    22,
+    25
+   ],
+   [
+    26,
+    27
+   ],
+   [
+    27,
+    30
+   ],
+   [
+    30,
+    31
+   ],
+   [
+    32,
+    36
+   ],
+   [
+    37,
+    40
+   ],
+   [
+    40,
+    41
+   ],
+   [
+    42,
+    46
+   ],
+   [
+    47,
+    51
+   ],
+   [
+    52,
+    54
+   ],
+   [
+    54,
+    56
+   ],
+   [
+    56,
+    57
+   ]
+  ]
+ },
+ {
+  "text": "строка с    несколькими   пробелами и\tтабом",
+  "ids": [
+   63594,
+   110,
+   14010,
+   32813,
+   14236,
+   107,
+   22360,
+   6536
+  ],
+  "tokens": [
+   "строка",
+   "с",
+   "несколькими",
+   "пробел",
+   "##ами",
+   "и",
+   "таб",
+   "##ом"
+  ],
+  "offsets": [
+   [
+    0,
+    6
+   ],
+   [
+    7,
+    8
+   ],
+   [
+    12,
+    23
+   ],
+   [
+    26,
+    32
+   ],
+   [
+    32,
+    35
+   ],
+   [
+    36,
+    37
+   ],
+   [
+    38,
+    41
+   ],
+   [
+    41,
+    43
+   ]
+  ]
+ },
+ {
+  "text": "深圳 unknown glyphs здесь",
+  "ids": [
+   100,
+   100,
+   16832,
+   93384,
+   20848,
+   45344,
+   71055,
+   466,
+   454,
+   1640
+  ],
+  "tokens": [
+   "[UNK]",
+   "[UNK]",
+   "un",
+   "##kn",
+   "##own",
+   "gl",
+   "##yp",
+   "##h",
+   "##s",
+   "здесь"
+  ],
+  "offsets": [
+   [
+    0,
+    1
+   ],
+   [
+    1,
+    2
+   ],
+   [
+    3,
+    5
+   ],
+   [
+    5,
+    7
+   ],
+   [
+    7,
+    10
+   ],
+   [
+    11,
+    13
+   ],
+   [
+    13,
+    15
+   ],
+   [
+    15,
+    16
+   ],
+   [
+    16,
+    17
+   ],
+   [
+    18,
+    23
+   ]
+  ]
+ },
+ {
+  "text": "",
+  "ids": [],
+  "tokens": [],
+  "offsets": []
+ },
+ {
+  "text": "а",
+  "ids": [
+   106
+  ],
+  "tokens": [
+   "а"
+  ],
+  "offsets": [
+   [
+    0,
+    1
+   ]
+  ]
+ }
+]