Upload 6 files

README.md

@@ -41,6 +41,32 @@ mELECTRA uses a **SentencePiece tokenizer** and requires a SentencePiece model f
 
 ### Example: Tokenization
 
+#### Using HuggingFace AutoTokenizer (Recommended)
+
+```python
+from transformers import AutoTokenizer
+
+# Load the tokenizer directly from HuggingFace Hub
+tokenizer = AutoTokenizer.from_pretrained("AILabTUL/mELECTRA")
+
+# Or load from local directory
+# tokenizer = AutoTokenizer.from_pretrained("./mELECTRA")
+
+# Tokenize input text
+sentence = "This is a multilingual model supporting multiple languages."
+tokens = tokenizer.tokenize(sentence)
+ids = tokenizer.encode(sentence)
+
+print(f"Tokens: {tokens}")
+print(f"IDs: {ids}")
+
+# Decode back to text
+decoded = tokenizer.decode(ids)
+print(f"Decoded: {decoded}")
+```
+
+#### Using SentencePiece directly
+
 ```python
 import sentencepiece as spm
 
@@ -48,7 +74,27 @@ import sentencepiece as spm
 sp = spm.SentencePieceProcessor()
 sp.load("m.model")
 
-# Tokenize input text
-sentence = "This is a multilingual model supporting multiple languages."
+# Tokenize input text (note: input should be lowercase)
+sentence = "this is a multilingual model supporting multiple languages."
 tokens = sp.encode(sentence, out_type=str)
 print(tokens)
+```
+
+---
+
+## Citation
+
+This model was published as part of the research paper:
+
+**"Study on Automatic Punctuation Restoration in Bilingual Broadcast Stream"**  
+*Martin Poláček, Petr Červa*  
+*RANLP Student Workshop 2025*
+
+Citation information will be provided after the conference publication.
+
+---
+
+## Related Models
+
+- **Czech-Slovak**: [AILabTUL/BiELECTRA-czech-slovak](https://huggingface.co/AILabTUL/BiELECTRA-czech-slovak)
+- **Norwegian-Swedish**: [AILabTUL/BiELECTRA-norwegian-swedish](https://huggingface.co/AILabTUL/BiELECTRA-norwegian-swedish)

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "unk_token": "<unk>",
+  "pad_token": "<unk>"
+}

tokenizer_config.json

@@ -0,0 +1,17 @@
+{
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "auto_map": {
+    "AutoTokenizer": [
+      "transformers",
+      "PreTrainedTokenizerFast"
+    ]
+  },
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "unk_token": "<unk>",
+  "pad_token": "<unk>",
+  "model_max_length": 512,
+  "special_tokens_map_file": null,
+  "name_or_path": "mELECTRA",
+  "tokenizer_type": "SentencePiece"
+}