Release unilineal model

README.md

@@ -0,0 +1,60 @@
+---
+library_name: transformers
+pipeline_tag: token-classification
+tags:
+  - name-splitting
+  - ner
+  - modernbert
+  - names
+language:
+  - es
+  - en
+  - pt
+license: mit
+---
+
+# Tori v2 — Name Splitter
+
+ModernBERT-base (149M params) fine-tuned for splitting full name strings into
+**forenames** and **surnames** using BIO token classification.
+
+## Variants
+
+| Variant | Countries | Surname Pattern | Subfolder |
+|---------|-----------|-----------------|-----------|
+| **bilineal** (default) | MX, CO, ES, PE, CL | Double surname (paternal + maternal) | `/` (root) |
+| **unilineal** | AR, US, BR, PT | Single surname | `unilineal/` |
+
+## Usage
+
+```python
+from tori.inference import load_pipeline, split_name
+
+# Default: bilineal model (double-surname countries)
+pipe = load_pipeline("ittailup/tori2")
+result = split_name(pipe, "Juan Carlos García López")
+print(result.forenames)  # ['Juan', 'Carlos']
+print(result.surnames)   # ['García', 'López']
+
+# Unilineal model (single-surname countries)
+pipe = load_pipeline("ittailup/tori2", variant="unilineal")
+result = split_name(pipe, "John Michael Smith")
+print(result.forenames)  # ['John', 'Michael']
+print(result.surnames)   # ['Smith']
+```
+
+## Labels
+
+- `O` — Outside any name entity
+- `B-forenames` — Beginning of forename
+- `I-forenames` — Inside forename (continuation)
+- `B-surnames` — Beginning of surname
+- `I-surnames` — Inside surname (continuation)
+
+## Important: Custom Aggregation Required
+
+This model uses ModernBERT's GPT-style BPE tokenizer (Ġ prefix), which is
+**not compatible** with HuggingFace's built-in `aggregation_strategy="simple"`.
+Use the `tori.inference` module which handles subword aggregation correctly,
+or use `aggregation_strategy="none"` and aggregate tokens yourself using
+character offsets.

tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "backend": "tokenizers",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "is_local": false,
+  "mask_token": "[MASK]",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 8192,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "[UNK]"
+}

unilineal/config.json

@@ -0,0 +1,92 @@
+{
+  "architectures": [
+    "ModernBertForTokenClassification"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "classifier_activation": "gelu",
+  "classifier_bias": false,
+  "classifier_dropout": 0.0,
+  "classifier_pooling": "mean",
+  "cls_token_id": 50281,
+  "decoder_bias": true,
+  "deterministic_flash_attn": false,
+  "dtype": "float32",
+  "embedding_dropout": 0.0,
+  "eos_token_id": null,
+  "global_attn_every_n_layers": 3,
+  "gradient_checkpointing": false,
+  "hidden_activation": "gelu",
+  "hidden_size": 768,
+  "id2label": {
+    "0": "O",
+    "1": "B-forenames",
+    "2": "I-forenames",
+    "3": "B-surnames",
+    "4": "I-surnames"
+  },
+  "initializer_cutoff_factor": 2.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 1152,
+  "label2id": {
+    "B-forenames": 1,
+    "B-surnames": 3,
+    "I-forenames": 2,
+    "I-surnames": 4,
+    "O": 0
+  },
+  "layer_norm_eps": 1e-05,
+  "layer_types": [
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "local_attention": 128,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "mlp_dropout": 0.0,
+  "model_type": "modernbert",
+  "norm_bias": false,
+  "norm_eps": 1e-05,
+  "num_attention_heads": 12,
+  "num_hidden_layers": 22,
+  "pad_token_id": 50283,
+  "position_embedding_type": "absolute",
+  "rope_parameters": {
+    "full_attention": {
+      "rope_theta": 160000.0,
+      "rope_type": "default"
+    },
+    "sliding_attention": {
+      "rope_theta": 10000.0,
+      "rope_type": "default"
+    }
+  },
+  "sep_token_id": 50282,
+  "sparse_pred_ignore_index": -100,
+  "sparse_prediction": false,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.2.0",
+  "use_cache": false,
+  "vocab_size": 50368
+}

unilineal/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0660ea79c428e1e2991c25982ea283f727ff83bf4697d0a56dbb2365a5adbde6
+size 598449012