Scandi+English tokenizer on OSCAR

.gitattributes

@@ -33,3 +33,10 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+texts/nn.opening.wiki.txt filter=lfs diff=lfs merge=lfs -text
+texts/no.opening.txt filter=lfs diff=lfs merge=lfs -text
+texts/sv.opening.txt filter=lfs diff=lfs merge=lfs -text
+texts/all.txt filter=lfs diff=lfs merge=lfs -text
+texts/da.opening.txt filter=lfs diff=lfs merge=lfs -text
+texts/en.opening.txt filter=lfs diff=lfs merge=lfs -text
+texts/nn.opening.txt filter=lfs diff=lfs merge=lfs -text

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

texts/all.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fef07b8bfdef0f9071b381deb4a0ac3627efa5e42f31f34d82f4e740d877a50c
+size 16552582106

texts/da.opening.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7525b7cf12415c0c2bbafd60f04a07b957d0b14dcad53a64c8fc59d1e25297e
+size 3258201809

texts/en.opening.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b64c3b85e582d4dbe3b16c900cb529ffce7118f56ce2b1620c1431921f0230d2
+size 6963630651

texts/nn.opening.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e94a2425df51215ceb5b4c64df20fe1e76c2aefa1a8fb093acb2ff6b16906ed
+size 87056553

texts/nn.opening.wiki.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00aa99b1c0aa097608c83a1d6244ada584e0cba27f9c4e08119fb2a40c142715
+size 113157261

texts/no.opening.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3edaf35fb8b2b66d5c54b345da22c9e2720709cd443ebca59cd62d97c75dfd8
+size 2399220841

texts/sv.opening.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8278001e6796c236c2301143bd1622e7346c70ebc4e82b3334698fb8e8398b01
+size 3731314991

tokenizer_config.json

@@ -0,0 +1,51 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "bos_token_id": 1,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "eos_token_id": 2,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "pad_token_id": 3,
+  "padding_side": "right",
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "unk_token_id": 0,
+  "use_default_system_prompt": false
+}

train_tokenizer.py

@@ -0,0 +1,168 @@
+import argparse
+import json
+import os
+import tempfile
+from pathlib import Path
+from tqdm import tqdm
+
+from datasets import load_dataset
+from tokenizers import SentencePieceBPETokenizer
+from transformers import LlamaTokenizerFast, TrainingArguments, AutoTokenizer
+
+def main(args):
+
+    # Load the dataset from the huggingface Hub and prepare it for training
+    if args.dataset_name is not None:
+        if args.dataset_type:
+            if os.path.isfile(args.dataset_name):
+                data_files = [args.dataset_name]
+            else:
+                data_files = os.listdir(args.dataset_name)
+                data_files = [Path(args.dataset_name) / f for f in data_files]
+            print(f"Training on {len(data_files)} files")
+            dataset = load_dataset(args.dataset_type,
+                data_files=data_files,
+                split=args.dataset_split,
+                token=args.hub_token if args.hub_token else None
+            )
+        else:
+            dataset = load_dataset(args.dataset_name,
+                split=args.dataset_split,
+                streaming=True,
+                token=args.hub_token if args.hub_token else None
+            )
+        print(dataset)
+    else:
+        raise ValueError("No dataset name provided or dataset is already tokenized") 
+
+    # Remove non text columns
+    dataset = dataset.remove_columns([col for col in dataset.column_names if col != "text"])
+
+    # Randomize docs
+    dataset = dataset.shuffle(seed=args.seed)
+    
+    # Select `num_samples` from the dataset
+    if args.num_samples:
+        dataset = dataset.select(range(args.num_samples))
+
+    # Create a SentencePieceBPETokenizer
+    tokenizer = SentencePieceBPETokenizer()
+
+    # Train the SentencePieceBPETokenizer on the dataset
+    tokenizer.train_from_iterator(
+        iterator=dataset['text'],
+        vocab_size=args.vocab_size,
+        show_progress=True,
+        special_tokens=["<unk>", "<s>", "</s>", "<pad>"],
+    )
+
+    # Save the tokenizer
+    new_tokenizer_file = tempfile.NamedTemporaryFile(prefix='tokenizer_', suffix='.json').name
+    tokenizer.save(new_tokenizer_file, pretty=True)
+
+    # Load reference tokenizer
+    if args.reference_tokenizer is not None and args.hub_token is not None:
+        reference_tokenizer = AutoTokenizer.from_pretrained(args.reference_tokenizer, token=args.hub_token if args.hub_token else None)
+        reference_tokenizer_path = tempfile.TemporaryDirectory().name
+        reference_tokenizer.save_pretrained(reference_tokenizer_path)
+    else:
+        raise ValueError("No tokenizer name provided or no hub token provided. Try using `--reference_tokenizer 'mistralai/Mistral-7B-Instruct-v0.2'")
+
+    # Read and dump the json file for the new tokenizer and the reference tokenizer
+    with open(new_tokenizer_file) as f:
+        new_tokenizer_json = json.load(f)
+
+    with open(Path(reference_tokenizer_path) / "tokenizer.json") as f:
+        reference_tokenizer_json = json.load(f)
+
+    # Add the reference tokenizer's config to the new tokenizer's config
+    new_tokenizer_json["normalizer"] = reference_tokenizer_json["normalizer"]
+    new_tokenizer_json["pre_tokenizer"] = reference_tokenizer_json["pre_tokenizer"]
+    new_tokenizer_json["post_processor"] = reference_tokenizer_json["post_processor"]
+    new_tokenizer_json["decoder"] = reference_tokenizer_json["decoder"]
+    new_tokenizer_json["model"]['fuse_unk'] = reference_tokenizer_json["model"]['fuse_unk']
+    new_tokenizer_json["model"]['byte_fallback'] = reference_tokenizer_json["model"]['byte_fallback']
+
+    # Dump the new tokenizer's config
+    with open(new_tokenizer_file, "w") as f:
+        json.dump(new_tokenizer_json, f, indent=2, ensure_ascii=False)
+
+    # Load the new tokenizer as a LlamaTokenizerFast
+    new_llama_tokenizer = LlamaTokenizerFast(
+        tokenizer_file=new_tokenizer_file,
+        name_or_path=args.reference_tokenizer + "-tokenizer",
+        unk_token="<unk>",
+        unk_token_id=0,
+        bos_token="<s>",
+        bos_token_id=1,
+        eos_token="</s>",
+        eos_token_id=2,
+        pad_token="<pad>",
+        pad_token_id=3,
+        padding_side="right",
+    )
+
+    # Save the new tokenizer
+    new_llama_tokenizer.save_pretrained(args.output)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Train a new Llama tokenizer")
+    parser.add_argument(
+        "--dataset_name",
+        type=str,
+        default=None,
+        help="The name of the dataset to be tokenized",
+    )
+    parser.add_argument(
+        "--dataset_type",
+        type=str,
+        default=None,
+        help="The type, 'text', 'json', or 'csv'. Leave blank for regular HF datasets",
+    )
+    parser.add_argument(
+        "--dataset_split",
+        type=str,
+        default=None,
+        help="The split of the dataset to be tokenized",
+    )
+    parser.add_argument(
+        "--hub_token",
+        type=str,
+        default=None,
+        help="The token to access the dataset on the hub",
+    )
+    parser.add_argument(
+        "--reference_tokenizer",
+        type=str,
+        default=None,
+        help="The name of the reference tokenizer to use",
+    )
+    parser.add_argument(
+        "--seed",
+        type=int,
+        default=123,
+        help="set random seed",
+    )
+    parser.add_argument(
+        "--num_samples",
+        type=int,
+        default=None,
+        help="Number of samples to use from the dataset",
+    )
+    parser.add_argument(
+        "--vocab_size",
+        type=int,
+        default=None,
+        help="Vocabulary size to use for the tokenizer",
+    )
+    parser.add_argument(
+        "--output",
+        type=str,
+        default="./",
+        help="Output path for the new tokenizer",
+    )
+    args = parser.parse_args()
+    main(args)
+
+# How to run:
+# python train_tokenizer.py --dataset_name texts/all.txt --dataset_type text --dataset_split train --reference_tokenizer mistralai/Mistral-7B-Instruct-v0.2 --vocab_size 32768 --hub_token True