Add files using upload-large-folder tool

tokenizers/gpt4o-balanced-unigram-tuned/README.md

@@ -25,7 +25,7 @@ The file is byte-identical to the one the models were trained with, which means
 | byte fallback | no | tokenizer file |
 | BOS token | <s> (id 0) | tokenizer file |
 | training data | balanced | training script |
-| trainer settings | `{"shrinking_factor": 0.7, "n_sub_iterations": 3, "max_piece_length": 64, "initial_alphabet": ["\u00d9", "e", "\u00f3", "\u0112", "\u0116", "\u00bf", "\u00ed", "\u011f", "\u0123", ">", "8", "u", "\u00ec", "s", "\u00e7", "\u013e", "\u0132", "\u010e", "\u0119", "F", "G", "\u00a8", "^", "j", "-", "\u0138", "\u0136", "\u012a", "#", "\u00f5", "r", "\u0137", "k", "\u0104", "m", "\u0143", "X", "3", "\u0101", "\u00a9", "\u00b7", "\u00c7", "b", ")", "'", "\u00ee", "\u012c", "\u00a3", "\u0117", "\u0135", "4", "\u00d1", "\u00ca", "g", "\u00f2", "W", "\u0111", "\u00bc", "h", "\u00fd", "\u00b2", "\u00bd", "\u00dc", "\u0133", "\u0121", "\u00e8", "\u011a", "\u00f0", "\u00e1", "o", "l", "y", "\u0113", "\u00c4", "6", "\u0108", "!", "\u0126", "i", "\u00f4", "\u00d0", "N", "\u0120", "\u00d7", "\u013c", "P", "\u012d", "\u00ae", "\u00c6", "/", "\u00cb", "\u011c", "A", "*", "\u00f6", "\u00f7", "\u0106", "$", "\u00e5", "p", "\u00ce", "\u0130", "\u00ba", "\u00fb", "\u0105", "\u00f9", "{", "O", "\u00d2", "Y", "Z", "Q", "\u013d", "\u00c2", "\u00e0", "H", "\u010a", "z", "\u00b3", "\u00be", "\u00d6", "\u00a7", "\u0134", "\u00af", "\u00c1", "\u00ef", "\u011b", "\u0127", "\u00b9", "\u013f", "\u00df", "\u010d", "D", "2", "\u00ea", ":", "\u010f", "K", "\u00ff", "a", "\\", "7", "\u0125", "\u00f8", "\"", "\u00a1", "\u00b5", "\u0110", "\u010c", "\u00da", ";", "L", "\u00fe", "c", "1", "+", "\u00ab", "\u012b", "E", "\u0131", "\u011e", "9", "\u0142", "<", "\u0118", "\u00a6", "f", "\u00c5", "M", "\u00aa", "[", "\u00cd", "\u00bb", "\u013a", "\u0115", "`", "(", "0", "v", ",", "]", "I", ".", "\u0129", "\u0141", "B", "\u00ac", "\u00a4", "}", "T", "\u00fa", "\u00dd", "\u00cc", "x", "\u00b1", "\u00c3", "\u00d3", "&", "t", "\u00e2", "q", "d", "\u0103", "\u00b4", "~", "\u00eb", "\u00b8", "%", "=", "U", "\u00b0", "\u00d5", "\u011d", "\u00e4", "|", "\u010b", "V", "\u00db", "\u0109", "\u00e3", "_", "\u0139", "\u012f", "@", "\u0102", "\u012e", "\u00fc", "\u00c8", "w", "n", "R", "\u00c0", "\u0122", "\u00d4", "\u00d8", "\u00a5", "\u00f1", "\u00e6", "\u00e9", "\u00cf", "C", "\u00c9", "\u0140", "?", "\u0100", "\u0114", "\u00b6", "J", "\u0128", "5", "\u0124", "\u00a2", "S", "\u00de", "\u013b", "\u0107"]}` | training script |
+| trainer settings | `{"shrinking_factor": 0.7, "n_sub_iterations": 3, "max_piece_length": 64, "initial_alphabet": ["\u0131", "\u00c5", "\u00b1", "}", "\u00cb", "_", "t", "\u00fa", "\u00be", "\u00d2", "\u00ea", "8", "\u012e", "\u00d4", "i", "\u00c2", "w", "r", "\u00b7", "\u013d", "\u00d0", "U", "p", "\u0121", "\u00c7", "R", "\u00ae", "\u0116", "\u00ac", "b", "\u00d7", "d", "?", "D", "\u00d1", "\u0143", "\u00e9", "\u00e4", "\u0129", "\u00b8", "W", "k", "\u00e8", "\u011b", "\u013b", "\u00c8", "\u0109", "\u00f8", "\u013e", "\u0103", "\u0112", "\u0111", "z", "L", "\u00e1", "\u00fb", "\u00a3", "\u00f6", "\u011c", "\u012d", "\u012c", "B", "C", "\u00df", "\u0122", "\u0120", "\u00b2", "\u00a7", "]", "5", "\u0139", ">", "`", "\u0108", "\u00ba", "\u0128", "(", "\u0115", "\u013a", "\u00f1", "f", "o", "\u00cd", "\u00da", "\u00a5", "\u012a", "\u010c", "\u00de", "x", "\u00e3", "\u00f7", "\u010a", "\u010e", "\u010f", "\u00c1", "\u00b0", "\"", "\u00ce", ".", "l", "J", "\u00ee", "F", "\u00a2", "\u00e0", "\u011d", "\u00bf", "Q", "\u00cc", "\u0117", "\u00cf", "\u0124", "Y", "$", "K", "\u00a9", "-", "\u00ef", "v", "^", "\u0105", "\u00fc", "\u0126", "\u00af", "\u0114", "\u0142", "I", "\u012b", "u", "<", "2", "\u00bb", "~", "\u0106", "n", "m", "\u0140", "\\", "1", "\u0127", "\u00e7", "6", "\u00db", "\u00d9", "\u00f5", "X", "\u00ab", "\u00eb", "'", "\u013c", "E", "\u00c0", "\u0135", "\u00dc", "\u00fd", "\u00c4", "\u0113", "\u0138", "\u00bd", "%", "@", "\u00ca", "3", "\u00ec", "\u00f2", "h", "y", "\u00b4", "\u00a4", "\u00b6", "&", "\u00ff", "\u011e", "g", "c", "\u00e6", "N", "\u0134", "4", "\u00f4", "\u00ed", "\u0118", "\u012f", "\u00b3", "V", "Z", "\u0141", "\u00b9", "\u0125", "\u0110", "O", "\u0100", "\u00f0", ")", "\u0136", "*", "P", "/", "\u00f9", "a", "\u0132", "\u011f", "\u0137", "!", "\u00d5", "\u00e5", "9", "A", "\u00fe", "q", "\u00d6", "e", "=", "\u0123", "\u00a8", "{", "\u00f3", "\u0133", "\u011a", "#", "\u00d3", "S", "G", "[", "\u010b", "\u0104", "s", "\u00c6", "\u0119", "\u0102", "\u010d", ":", "\u00a1", "\u00aa", "\u00d8", "\u0130", "j", "M", "7", "\u00a6", "\u0107", "\u013f", "\u0101", "0", "\u00c3", "\u00dd", ";", "H", "\u00b5", "+", "|", ",", "\u00c9", "T", "\u00bc", "\u00e2"]}` | training script |
 
 GPT-4o regex, balanced data, UnigramLM with tuned hyperparameters