Update tokenizer.json
Browse files- tokenizer.json +12 -0
tokenizer.json
CHANGED
|
@@ -97,6 +97,18 @@
|
|
| 97 |
"INST=TANPURA": 44,
|
| 98 |
"INST=SAROD": 45,
|
| 99 |
"INST=BANSURI": 46
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 100 |
},
|
| 101 |
"unk_token": "[UNK]"
|
| 102 |
}
|
|
|
|
| 97 |
"INST=TANPURA": 44,
|
| 98 |
"INST=SAROD": 45,
|
| 99 |
"INST=BANSURI": 46
|
| 100 |
+
"TABLA_NA": 47,
|
| 101 |
+
"TABLA_TIN": 48,
|
| 102 |
+
"TABLA_TA": 49,
|
| 103 |
+
"TABLA_DHA": 50,
|
| 104 |
+
"TABLA_DHE": 51,
|
| 105 |
+
"TABLA_GE": 52,
|
| 106 |
+
"TAAL_START=TEENTAL": 53,
|
| 107 |
+
"TAAL_START=JHAPTAL": 54,
|
| 108 |
+
"TAAL_START=EKTAAL": 55,
|
| 109 |
+
"TAAL_END": 56,
|
| 110 |
+
"TABLA_THEKA_START": 57,
|
| 111 |
+
"TABLA_THEKA_END": 58
|
| 112 |
},
|
| 113 |
"unk_token": "[UNK]"
|
| 114 |
}
|