Added files

README.md

@@ -0,0 +1,32 @@
+---
+language: tr
+tags:
+- roberta
+- language model
+datasets:
+- oscar
+---
+# TavBERT base model
+A Turkish BERT-style masked language model operating over characters, pre-trained by masking spans of characters, similarly to SpanBERT (Joshi et al., 2020).
+
+### How to use
+```python
+import numpy as np
+import torch
+from transformers import AutoModelForMaskedLM, AutoTokenizer
+
+model = AutoModelForMaskedLM.from_pretrained("tau/tavbert-tr")
+tokenizer = AutoTokenizer.from_pretrained("tau/tavbert-tr")
+
+def mask_sentence(sent, span_len=5):
+    start_pos = np.random.randint(0, len(sent) - span_len)
+    masked_sent = sent[:start_pos] + '[MASK]' * span_len + sent[start_pos + span_len:]
+    print("Masked sentence:", masked_sent)
+    output = model(**tokenizer.encode_plus(masked_sent, 
+                                           return_tensors='pt'))['logits'][0][1:-1]
+    preds = [int(x) for x in torch.argmax(torch.softmax(output, axis=1), axis=1)[start_pos:start_pos + span_len]]
+    pred_sent = sent[:start_pos] + ''.join(tokenizer.convert_ids_to_tokens(preds)) + sent[start_pos + span_len:]
+    print("Model's prediction:", pred_sent)
+```
+## Training data
+OSCAR (Ortiz, 2019) Turkish section (27 GB text, 77 million sentences).

config.json

@@ -0,0 +1,25 @@
+{
+  "architectures": [
+    "RobertaForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 2050,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "transformers_version": "4.6.0.dev0",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 250
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef2c5f9cc42662893a6b826ceb725f99cd0b89a71c6770d578b708f8bc91cd7e
+size 349766939

special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

tokenizer.json

@@ -0,0 +1 @@
+{"version":"1.0","truncation":null,"padding":null,"added_tokens":[],"normalizer":null,"pre_tokenizer":{"type":"Split","pattern":{"String":""},"behavior":"Isolated","invert":false},"post_processor":{"type":"TemplateProcessing","single":[{"SpecialToken":{"id":"[CLS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"[SEP]","type_id":0}}],"pair":[{"SpecialToken":{"id":"[CLS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"[SEP]","type_id":0}},{"Sequence":{"id":"B","type_id":1}},{"SpecialToken":{"id":"[SEP]","type_id":1}}],"special_tokens":{"[CLS]":{"id":"[CLS]","ids":[0],"tokens":["[CLS]"]},"[SEP]":{"id":"[SEP]","ids":[2],"tokens":["[SEP]"]}}},"decoder":null,"model":{"type":"WordLevel","vocab":{"[CLS]": 0, "[PAD]": 1, "[SEP]": 2, "[UNK]": 3, "unused00": 4, "unused01": 5, "unused02": 6, "unused03": 7, "unused04": 8, "unused05": 9, "unused06": 10, "unused07": 11, "unused08": 12, "unused09": 13, "unused10": 14, "unused11": 15, "unused12": 16, "unused13": 17, "unused14": 18, "unused15": 19, "unused16": 20, "unused17": 21, "unused18": 22, "unused19": 23, "unused20": 24, "unused21": 25, "unused22": 26, "unused23": 27, "unused24": 28, "unused25": 29, "unused26": 30, "unused27": 31, "unused28": 32, "unused29": 33, "unused30": 34, "unused31": 35, "unused32": 36, "unused33": 37, "unused34": 38, "unused35": 39, "unused36": 40, "unused37": 41, "unused38": 42, "unused39": 43, "unused40": 44, "unused41": 45, "unused42": 46, "unused43": 47, "unused44": 48, "unused45": 49, "unused46": 50, "unused47": 51, "unused48": 52, "unused49": 53, "unused50": 54, "unused51": 55, "unused52": 56, "unused53": 57, "unused54": 58, "unused55": 59, "unused56": 60, "unused57": 61, "unused58": 62, "unused59": 63, "unused60": 64, "unused61": 65, "unused62": 66, "unused63": 67, "unused64": 68, "unused65": 69, "unused66": 70, "unused67": 71, "unused68": 72, "unused69": 73, "unused70": 74, "unused71": 75, "unused72": 76, "unused73": 77, "unused74": 78, "unused75": 79, "unused76": 80, "unused77": 81, "unused78": 82, "unused79": 83, "unused80": 84, "unused81": 85, "unused82": 86, "unused83": 87, "unused84": 88, "unused85": 89, "unused86": 90, "unused87": 91, "unused88": 92, "unused89": 93, "unused90": 94, "unused91": 95, "unused92": 96, "unused93": 97, "unused94": 98, "unused95": 99, "unused96": 100, "unused97": 101, "unused98": 102, "unused99": 103, " ": 104, "!": 105, "\"": 106, "#": 107, "$": 108, "%": 109, "&": 110, "'": 111, "(": 112, ")": 113, "*": 114, "+": 115, ",": 116, "-": 117, ".": 118, "/": 119, "0": 120, "1": 121, "2": 122, "3": 123, "4": 124, "5": 125, "6": 126, "7": 127, "8": 128, "9": 129, ":": 130, ";": 131, "<": 132, "=": 133, ">": 134, "?": 135, "@": 136, "A": 137, "B": 138, "C": 139, "D": 140, "E": 141, "F": 142, "G": 143, "H": 144, "I": 145, "J": 146, "K": 147, "L": 148, "M": 149, "N": 150, "O": 151, "P": 152, "Q": 153, "R": 154, "S": 155, "T": 156, "U": 157, "V": 158, "W": 159, "X": 160, "Y": 161, "Z": 162, "[": 163, "\\": 164, "]": 165, "_": 166, "`": 167, "a": 168, "b": 169, "c": 170, "d": 171, "e": 172, "f": 173, "g": 174, "h": 175, "i": 176, "j": 177, "k": 178, "l": 179, "m": 180, "n": 181, "o": 182, "p": 183, "q": 184, "r": 185, "s": 186, "t": 187, "u": 188, "v": 189, "w": 190, "x": 191, "y": 192, "z": 193, "|": 194, "’": 195, "“": 196, "”": 197, "­": 198, "±": 199, "´": 200, "·": 201, "»": 202, "Â": 203, "Ã": 204, "Ä": 205, "Ç": 206, "Ö": 207, "Ü": 208, "â": 209, "ç": 210, "é": 211, "î": 212, "ð": 213, "ö": 214, "û": 215, "ü": 216, "ý": 217, "þ": 218, "Ğ": 219, "ğ": 220, "İ": 221, "ı": 222, "Ş": 223, "ş": 224, "̈": 225, "̧": 226, "а": 227, "в": 228, "е": 229, "и": 230, "к": 231, "л": 232, "н": 233, "о": 234, "р": 235, "с": 236, "т": 237, "у": 238, "ا": 239, "–": 240, "‘": 241, "’": 242, "“": 243, "”": 244, "•": 245, "…": 246, "あ": 247, "�": 248, "[MASK]": 249},"unk_token":"[UNK]"}}

tokenizer_config.json

@@ -0,0 +1 @@
+{"special_tokens_map_file": "./special_tokens_map.json", "name_or_path": ".", "tokenizer_class": "PreTrainedTokenizerFast"}

vocab.txt

@@ -0,0 +1,250 @@
+[CLS]
+[PAD]
+[SEP]
+[UNK]
+unused00
+unused01
+unused02
+unused03
+unused04
+unused05
+unused06
+unused07
+unused08
+unused09
+unused10
+unused11
+unused12
+unused13
+unused14
+unused15
+unused16
+unused17
+unused18
+unused19
+unused20
+unused21
+unused22
+unused23
+unused24
+unused25
+unused26
+unused27
+unused28
+unused29
+unused30
+unused31
+unused32
+unused33
+unused34
+unused35
+unused36
+unused37
+unused38
+unused39
+unused40
+unused41
+unused42
+unused43
+unused44
+unused45
+unused46
+unused47
+unused48
+unused49
+unused50
+unused51
+unused52
+unused53
+unused54
+unused55
+unused56
+unused57
+unused58
+unused59
+unused60
+unused61
+unused62
+unused63
+unused64
+unused65
+unused66
+unused67
+unused68
+unused69
+unused70
+unused71
+unused72
+unused73
+unused74
+unused75
+unused76
+unused77
+unused78
+unused79
+unused80
+unused81
+unused82
+unused83
+unused84
+unused85
+unused86
+unused87
+unused88
+unused89
+unused90
+unused91
+unused92
+unused93
+unused94
+unused95
+unused96
+unused97
+unused98
+unused99
+ 
+!
+"
+#
+$
+%
+&
+'
+(
+)
+*
++
+,
+-
+.
+/
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
+:
+;
+<
+=
+>
+?
+@
+A
+B
+C
+D
+E
+F
+G
+H
+I
+J
+K
+L
+M
+N
+O
+P
+Q
+R
+S
+T
+U
+V
+W
+X
+Y
+Z
+[
+\
+]
+_
+`
+a
+b
+c
+d
+e
+f
+g
+h
+i
+j
+k
+l
+m
+n
+o
+p
+q
+r
+s
+t
+u
+v
+w
+x
+y
+z
+|
+’
+“
+”
+­
+±
+´
+·
+»
+Â
+Ã
+Ä
+Ç
+Ö
+Ü
+â
+ç
+é
+î
+ð
+ö
+û
+ü
+ý
+þ
+Ğ
+ğ
+İ
+ı
+Ş
+ş
+̈
+̧
+а
+в
+е
+и
+к
+л
+н
+о
+р
+с
+т
+у
+ا
+–
+‘
+’
+“
+”
+•
+…
+あ
+�
+[MASK]