Upload folder using huggingface_hub

README.md

@@ -0,0 +1,17 @@
+---
+language:
+- sd
+license: mit
+tags:
+- sindhi
+- ner
+- token-classification
+---
+
+# Sindhi-BERT-NER
+
+Named Entity Recognition for Sindhi — 6 entity types.
+
+- **Dataset**: SiNER (25,262 sentences, 1M+ words)
+- **F1**: 77.85%
+- **Entities**: PERSON, LOCATION, ORG, TITLE, EVENT, DATE_TIME

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<mask>": 32000
+}

checkpoint-1424/config.json

@@ -0,0 +1,58 @@
+{
+  "add_cross_attention": false,
+  "architectures": [
+    "RobertaForTokenClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 1,
+  "classifier_dropout": null,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "O",
+    "1": "B-PERSON",
+    "2": "I-PERSON",
+    "3": "B-LOCATION",
+    "4": "I-LOCATION",
+    "5": "B-ORG",
+    "6": "I-ORG",
+    "7": "B-TITLE",
+    "8": "I-TITLE",
+    "9": "B-EVENT",
+    "10": "I-EVENT",
+    "11": "B-DATE_TIME",
+    "12": "I-DATE_TIME"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "is_decoder": false,
+  "label2id": {
+    "B-DATE_TIME": 11,
+    "B-EVENT": 9,
+    "B-LOCATION": 3,
+    "B-ORG": 5,
+    "B-PERSON": 1,
+    "B-TITLE": 7,
+    "I-DATE_TIME": 12,
+    "I-EVENT": 10,
+    "I-LOCATION": 4,
+    "I-ORG": 6,
+    "I-PERSON": 2,
+    "I-TITLE": 8,
+    "O": 0
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 514,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.0.0",
+  "type_vocab_size": 1,
+  "use_cache": false,
+  "vocab_size": 32001
+}

checkpoint-1424/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7cd818c3957e92fb0f42f6ff6951b08800aa73ca8e7e37f327fe6483b0c0c8b
+size 440177036

checkpoint-1424/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dcb7fdf681742f7291b0b007fe4d2f83eae23888201df21df38aef89aad9011e
+size 880475851

checkpoint-1424/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:693238eb4365ebe9209a7f8ca5b005d342dfac7fcc6bac3774317e5a0dec9780
+size 14645

checkpoint-1424/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:043500a0d6ffb06a78aab47d423a239f357644895b34276c68614b23ba32cde2
+size 1465

checkpoint-1424/trainer_state.json

@@ -0,0 +1,201 @@
+{
+  "best_global_step": 1424,
+  "best_metric": 0.7741483492424905,
+  "best_model_checkpoint": "sindhibert_ner_v2/checkpoint-1424",
+  "epoch": 4.0,
+  "eval_steps": 356,
+  "global_step": 1424,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.2812939521800281,
+      "grad_norm": 9.956046104431152,
+      "learning_rate": 1.6685393258426966e-05,
+      "loss": 2.9438055419921874,
+      "step": 100
+    },
+    {
+      "epoch": 0.5625879043600562,
+      "grad_norm": 9.802416801452637,
+      "learning_rate": 2.9987282164731155e-05,
+      "loss": 0.8266248321533203,
+      "step": 200
+    },
+    {
+      "epoch": 0.8438818565400844,
+      "grad_norm": 6.607327461242676,
+      "learning_rate": 2.9579691638256147e-05,
+      "loss": 0.622057991027832,
+      "step": 300
+    },
+    {
+      "epoch": 1.0,
+      "eval_f1": 0.5810973451327435,
+      "eval_f1_DATE_TIME": 0.8529801324503312,
+      "eval_f1_EVENT": 0.1316872427983539,
+      "eval_f1_LOCATION": 0.7365342432366562,
+      "eval_f1_ORG": 0.37763116556707504,
+      "eval_f1_PERSON": 0.605584642233857,
+      "eval_f1_TITLE": 0.6061554512258737,
+      "eval_loss": 0.24599596858024597,
+      "eval_runtime": 2.5923,
+      "eval_samples_per_second": 974.42,
+      "eval_steps_per_second": 30.475,
+      "step": 356
+    },
+    {
+      "epoch": 1.1237693389592125,
+      "grad_norm": 8.488602638244629,
+      "learning_rate": 2.8613205356500785e-05,
+      "loss": 0.4819778823852539,
+      "step": 400
+    },
+    {
+      "epoch": 1.4050632911392404,
+      "grad_norm": 6.543027400970459,
+      "learning_rate": 2.7124872462465625e-05,
+      "loss": 0.37242794036865234,
+      "step": 500
+    },
+    {
+      "epoch": 1.6863572433192686,
+      "grad_norm": 7.36555290222168,
+      "learning_rate": 2.517174649044749e-05,
+      "loss": 0.3637136459350586,
+      "step": 600
+    },
+    {
+      "epoch": 1.9676511954992968,
+      "grad_norm": 3.421555995941162,
+      "learning_rate": 2.2828698284245604e-05,
+      "loss": 0.3521112060546875,
+      "step": 700
+    },
+    {
+      "epoch": 2.0,
+      "eval_f1": 0.681080224914865,
+      "eval_f1_DATE_TIME": 0.7850241545893719,
+      "eval_f1_EVENT": 0.2574712643678161,
+      "eval_f1_LOCATION": 0.784957092377587,
+      "eval_f1_ORG": 0.5021991203518592,
+      "eval_f1_PERSON": 0.6830154405086286,
+      "eval_f1_TITLE": 0.7609511889862327,
+      "eval_loss": 0.20519763231277466,
+      "eval_runtime": 2.582,
+      "eval_samples_per_second": 978.319,
+      "eval_steps_per_second": 30.597,
+      "step": 712
+    },
+    {
+      "epoch": 2.247538677918425,
+      "grad_norm": 2.109884738922119,
+      "learning_rate": 2.0185545909245344e-05,
+      "loss": 0.24458488464355468,
+      "step": 800
+    },
+    {
+      "epoch": 2.528832630098453,
+      "grad_norm": 2.8797037601470947,
+      "learning_rate": 1.7343611579907725e-05,
+      "loss": 0.212690372467041,
+      "step": 900
+    },
+    {
+      "epoch": 2.810126582278481,
+      "grad_norm": 5.664893627166748,
+      "learning_rate": 1.4411837588899939e-05,
+      "loss": 0.18955570220947265,
+      "step": 1000
+    },
+    {
+      "epoch": 3.0,
+      "eval_f1": 0.6778448823903368,
+      "eval_f1_DATE_TIME": 0.7517401392111369,
+      "eval_f1_EVENT": 0.22174840085287847,
+      "eval_f1_LOCATION": 0.7845415508966911,
+      "eval_f1_ORG": 0.5135026199113262,
+      "eval_f1_PERSON": 0.7053455019556715,
+      "eval_f1_TITLE": 0.7071633237822349,
+      "eval_loss": 0.22204764187335968,
+      "eval_runtime": 2.5828,
+      "eval_samples_per_second": 978.001,
+      "eval_steps_per_second": 30.587,
+      "step": 1068
+    },
+    {
+      "epoch": 3.090014064697609,
+      "grad_norm": 3.924571990966797,
+      "learning_rate": 1.1502610129268493e-05,
+      "loss": 0.17344663619995118,
+      "step": 1100
+    },
+    {
+      "epoch": 3.371308016877637,
+      "grad_norm": 4.71477746963501,
+      "learning_rate": 8.727451098641628e-06,
+      "loss": 0.11389885902404785,
+      "step": 1200
+    },
+    {
+      "epoch": 3.652601969057665,
+      "grad_norm": 3.7797605991363525,
+      "learning_rate": 6.192743035205252e-06,
+      "loss": 0.13766860008239745,
+      "step": 1300
+    },
+    {
+      "epoch": 3.9338959212376934,
+      "grad_norm": 7.3217573165893555,
+      "learning_rate": 3.99565106512789e-06,
+      "loss": 0.11051109313964844,
+      "step": 1400
+    },
+    {
+      "epoch": 4.0,
+      "eval_f1": 0.7741483492424905,
+      "eval_f1_DATE_TIME": 0.8885869565217391,
+      "eval_f1_EVENT": 0.43939393939393934,
+      "eval_f1_LOCATION": 0.8579088471849866,
+      "eval_f1_ORG": 0.6117216117216118,
+      "eval_f1_PERSON": 0.7820689655172415,
+      "eval_f1_TITLE": 0.788785046728972,
+      "eval_loss": 0.2307787537574768,
+      "eval_runtime": 2.5777,
+      "eval_samples_per_second": 979.958,
+      "eval_steps_per_second": 30.648,
+      "step": 1424
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 1780,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 356,
+  "stateful_callbacks": {
+    "EarlyStoppingCallback": {
+      "args": {
+        "early_stopping_patience": 2,
+        "early_stopping_threshold": 0.0
+      },
+      "attributes": {
+        "early_stopping_patience_counter": 0
+      }
+    },
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.1882867759874048e+16,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1424/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e408cb707cffd121ea550570c3f95e1096bf110de0cd59e88ad83088ea4c5a45
+size 5137

checkpoint-1780/config.json

@@ -0,0 +1,58 @@
+{
+  "add_cross_attention": false,
+  "architectures": [
+    "RobertaForTokenClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 1,
+  "classifier_dropout": null,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "O",
+    "1": "B-PERSON",
+    "2": "I-PERSON",
+    "3": "B-LOCATION",
+    "4": "I-LOCATION",
+    "5": "B-ORG",
+    "6": "I-ORG",
+    "7": "B-TITLE",
+    "8": "I-TITLE",
+    "9": "B-EVENT",
+    "10": "I-EVENT",
+    "11": "B-DATE_TIME",
+    "12": "I-DATE_TIME"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "is_decoder": false,
+  "label2id": {
+    "B-DATE_TIME": 11,
+    "B-EVENT": 9,
+    "B-LOCATION": 3,
+    "B-ORG": 5,
+    "B-PERSON": 1,
+    "B-TITLE": 7,
+    "I-DATE_TIME": 12,
+    "I-EVENT": 10,
+    "I-LOCATION": 4,
+    "I-ORG": 6,
+    "I-PERSON": 2,
+    "I-TITLE": 8,
+    "O": 0
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 514,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.0.0",
+  "type_vocab_size": 1,
+  "use_cache": false,
+  "vocab_size": 32001
+}

checkpoint-1780/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af85b500e5863e29ac092288b8472a99661790776d5e5e3f8f9f1c19cc812909
+size 440177036

checkpoint-1780/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17af99095501e07c0c9068f5ebdcc3a9bc16e4fa9955bcd30c255901f5ddb92a
+size 880475851

checkpoint-1780/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5508252b62965557c8353f64441f420945ff5ad08c4213e0e213f79b64be0bf2
+size 14645

checkpoint-1780/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31db12016b59357ed984f1796e3d3160baeb4cfd211201cd83b3c7c62afa2747
+size 1465

checkpoint-1780/trainer_state.json

@@ -0,0 +1,237 @@
+{
+  "best_global_step": 1780,
+  "best_metric": 0.7784504694217776,
+  "best_model_checkpoint": "sindhibert_ner_v2/checkpoint-1780",
+  "epoch": 5.0,
+  "eval_steps": 356,
+  "global_step": 1780,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.2812939521800281,
+      "grad_norm": 9.956046104431152,
+      "learning_rate": 1.6685393258426966e-05,
+      "loss": 2.9438055419921874,
+      "step": 100
+    },
+    {
+      "epoch": 0.5625879043600562,
+      "grad_norm": 9.802416801452637,
+      "learning_rate": 2.9987282164731155e-05,
+      "loss": 0.8266248321533203,
+      "step": 200
+    },
+    {
+      "epoch": 0.8438818565400844,
+      "grad_norm": 6.607327461242676,
+      "learning_rate": 2.9579691638256147e-05,
+      "loss": 0.622057991027832,
+      "step": 300
+    },
+    {
+      "epoch": 1.0,
+      "eval_f1": 0.5810973451327435,
+      "eval_f1_DATE_TIME": 0.8529801324503312,
+      "eval_f1_EVENT": 0.1316872427983539,
+      "eval_f1_LOCATION": 0.7365342432366562,
+      "eval_f1_ORG": 0.37763116556707504,
+      "eval_f1_PERSON": 0.605584642233857,
+      "eval_f1_TITLE": 0.6061554512258737,
+      "eval_loss": 0.24599596858024597,
+      "eval_runtime": 2.5923,
+      "eval_samples_per_second": 974.42,
+      "eval_steps_per_second": 30.475,
+      "step": 356
+    },
+    {
+      "epoch": 1.1237693389592125,
+      "grad_norm": 8.488602638244629,
+      "learning_rate": 2.8613205356500785e-05,
+      "loss": 0.4819778823852539,
+      "step": 400
+    },
+    {
+      "epoch": 1.4050632911392404,
+      "grad_norm": 6.543027400970459,
+      "learning_rate": 2.7124872462465625e-05,
+      "loss": 0.37242794036865234,
+      "step": 500
+    },
+    {
+      "epoch": 1.6863572433192686,
+      "grad_norm": 7.36555290222168,
+      "learning_rate": 2.517174649044749e-05,
+      "loss": 0.3637136459350586,
+      "step": 600
+    },
+    {
+      "epoch": 1.9676511954992968,
+      "grad_norm": 3.421555995941162,
+      "learning_rate": 2.2828698284245604e-05,
+      "loss": 0.3521112060546875,
+      "step": 700
+    },
+    {
+      "epoch": 2.0,
+      "eval_f1": 0.681080224914865,
+      "eval_f1_DATE_TIME": 0.7850241545893719,
+      "eval_f1_EVENT": 0.2574712643678161,
+      "eval_f1_LOCATION": 0.784957092377587,
+      "eval_f1_ORG": 0.5021991203518592,
+      "eval_f1_PERSON": 0.6830154405086286,
+      "eval_f1_TITLE": 0.7609511889862327,
+      "eval_loss": 0.20519763231277466,
+      "eval_runtime": 2.582,
+      "eval_samples_per_second": 978.319,
+      "eval_steps_per_second": 30.597,
+      "step": 712
+    },
+    {
+      "epoch": 2.247538677918425,
+      "grad_norm": 2.109884738922119,
+      "learning_rate": 2.0185545909245344e-05,
+      "loss": 0.24458488464355468,
+      "step": 800
+    },
+    {
+      "epoch": 2.528832630098453,
+      "grad_norm": 2.8797037601470947,
+      "learning_rate": 1.7343611579907725e-05,
+      "loss": 0.212690372467041,
+      "step": 900
+    },
+    {
+      "epoch": 2.810126582278481,
+      "grad_norm": 5.664893627166748,
+      "learning_rate": 1.4411837588899939e-05,
+      "loss": 0.18955570220947265,
+      "step": 1000
+    },
+    {
+      "epoch": 3.0,
+      "eval_f1": 0.6778448823903368,
+      "eval_f1_DATE_TIME": 0.7517401392111369,
+      "eval_f1_EVENT": 0.22174840085287847,
+      "eval_f1_LOCATION": 0.7845415508966911,
+      "eval_f1_ORG": 0.5135026199113262,
+      "eval_f1_PERSON": 0.7053455019556715,
+      "eval_f1_TITLE": 0.7071633237822349,
+      "eval_loss": 0.22204764187335968,
+      "eval_runtime": 2.5828,
+      "eval_samples_per_second": 978.001,
+      "eval_steps_per_second": 30.587,
+      "step": 1068
+    },
+    {
+      "epoch": 3.090014064697609,
+      "grad_norm": 3.924571990966797,
+      "learning_rate": 1.1502610129268493e-05,
+      "loss": 0.17344663619995118,
+      "step": 1100
+    },
+    {
+      "epoch": 3.371308016877637,
+      "grad_norm": 4.71477746963501,
+      "learning_rate": 8.727451098641628e-06,
+      "loss": 0.11389885902404785,
+      "step": 1200
+    },
+    {
+      "epoch": 3.652601969057665,
+      "grad_norm": 3.7797605991363525,
+      "learning_rate": 6.192743035205252e-06,
+      "loss": 0.13766860008239745,
+      "step": 1300
+    },
+    {
+      "epoch": 3.9338959212376934,
+      "grad_norm": 7.3217573165893555,
+      "learning_rate": 3.99565106512789e-06,
+      "loss": 0.11051109313964844,
+      "step": 1400
+    },
+    {
+      "epoch": 4.0,
+      "eval_f1": 0.7741483492424905,
+      "eval_f1_DATE_TIME": 0.8885869565217391,
+      "eval_f1_EVENT": 0.43939393939393934,
+      "eval_f1_LOCATION": 0.8579088471849866,
+      "eval_f1_ORG": 0.6117216117216118,
+      "eval_f1_PERSON": 0.7820689655172415,
+      "eval_f1_TITLE": 0.788785046728972,
+      "eval_loss": 0.2307787537574768,
+      "eval_runtime": 2.5777,
+      "eval_samples_per_second": 979.958,
+      "eval_steps_per_second": 30.648,
+      "step": 1424
+    },
+    {
+      "epoch": 4.213783403656821,
+      "grad_norm": 5.604534149169922,
+      "learning_rate": 2.2203981889025476e-06,
+      "loss": 0.09554487228393554,
+      "step": 1500
+    },
+    {
+      "epoch": 4.49507735583685,
+      "grad_norm": 2.9002649784088135,
+      "learning_rate": 9.350366892310286e-07,
+      "loss": 0.09470049858093262,
+      "step": 1600
+    },
+    {
+      "epoch": 4.776371308016878,
+      "grad_norm": 5.062108993530273,
+      "learning_rate": 1.888394248260733e-07,
+      "loss": 0.08586543083190917,
+      "step": 1700
+    },
+    {
+      "epoch": 5.0,
+      "eval_f1": 0.7784504694217776,
+      "eval_f1_DATE_TIME": 0.874331550802139,
+      "eval_f1_EVENT": 0.4296296296296296,
+      "eval_f1_LOCATION": 0.8630211153221439,
+      "eval_f1_ORG": 0.6381176470588235,
+      "eval_f1_PERSON": 0.771977096665544,
+      "eval_f1_TITLE": 0.7957259585166562,
+      "eval_loss": 0.23349934816360474,
+      "eval_runtime": 2.5751,
+      "eval_samples_per_second": 980.932,
+      "eval_steps_per_second": 30.678,
+      "step": 1780
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 1780,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 356,
+  "stateful_callbacks": {
+    "EarlyStoppingCallback": {
+      "args": {
+        "early_stopping_patience": 2,
+        "early_stopping_threshold": 0.0
+      },
+      "attributes": {
+        "early_stopping_patience_counter": 0
+      }
+    },
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.485358469984256e+16,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1780/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e408cb707cffd121ea550570c3f95e1096bf110de0cd59e88ad83088ea4c5a45
+size 5137

config.json

@@ -0,0 +1,58 @@
+{
+  "add_cross_attention": false,
+  "architectures": [
+    "RobertaForTokenClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 1,
+  "classifier_dropout": null,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "O",
+    "1": "B-PERSON",
+    "2": "I-PERSON",
+    "3": "B-LOCATION",
+    "4": "I-LOCATION",
+    "5": "B-ORG",
+    "6": "I-ORG",
+    "7": "B-TITLE",
+    "8": "I-TITLE",
+    "9": "B-EVENT",
+    "10": "I-EVENT",
+    "11": "B-DATE_TIME",
+    "12": "I-DATE_TIME"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "is_decoder": false,
+  "label2id": {
+    "B-DATE_TIME": 11,
+    "B-EVENT": 9,
+    "B-LOCATION": 3,
+    "B-ORG": 5,
+    "B-PERSON": 1,
+    "B-TITLE": 7,
+    "I-DATE_TIME": 12,
+    "I-EVENT": 10,
+    "I-LOCATION": 4,
+    "I-ORG": 6,
+    "I-PERSON": 2,
+    "I-TITLE": 8,
+    "O": 0
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 514,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.0.0",
+  "type_vocab_size": 1,
+  "use_cache": false,
+  "vocab_size": 32001
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af85b500e5863e29ac092288b8472a99661790776d5e5e3f8f9f1c19cc812909
+size 440177036

sindhi_bpe_32k.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4b0bb0c4dfcc9dac594b288c5eb6bb103388fc39f75d40003d4d6a2ddf8cf46
+size 644934

tokenizer_config.json

@@ -0,0 +1,53 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32000": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": null,
+  "backend": "custom",
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "tokenizer_class": "SindhiTokenizer",
+  "unk_token": "<unk>"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e408cb707cffd121ea550570c3f95e1096bf110de0cd59e88ad83088ea4c5a45
+size 5137