diff --git a/Government/10k/en-si_final_model/config.json b/Government/10k/en-si_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/10k/en-si_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/10k/en-si_final_model/generation_config.json b/Government/10k/en-si_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/10k/en-si_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/10k/en-si_final_model/pytorch_model.bin b/Government/10k/en-si_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..474c649700842be8c77e2dcb1e4ace8e1baf5b8f --- /dev/null +++ b/Government/10k/en-si_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e30119a318abf1038957a71b5bfcabb495656eeda1e99a3f6fdaff42fd702be5 +size 2468870723 diff --git a/Government/10k/en-si_final_model/training_args.bin b/Government/10k/en-si_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..e9d29fa05fe41c81512c15014fdbb90ad771ad4a --- /dev/null +++ b/Government/10k/en-si_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4712d7c12c0ef3878d4996ea8548dfa42237c138285149fb0f26b05f0b3e7b2b +size 4143 diff --git a/Government/10k/en-ta/runs/Oct02_21-05-18_NLPC-Sita-MLM/events.out.tfevents.1759419318.NLPC-Sita-MLM.2581313.3 b/Government/10k/en-ta/runs/Oct02_21-05-18_NLPC-Sita-MLM/events.out.tfevents.1759419318.NLPC-Sita-MLM.2581313.3 new file mode 100644 index 0000000000000000000000000000000000000000..a761595b9150e5303b4ac41400cbb4684a4e3d87 --- /dev/null +++ b/Government/10k/en-ta/runs/Oct02_21-05-18_NLPC-Sita-MLM/events.out.tfevents.1759419318.NLPC-Sita-MLM.2581313.3 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a58afb508f23020e92bf74d8c0c79efda381747205e84f94310bb3d1da8f418 +size 6601 diff --git a/Government/10k/en-ta_final_model/config.json b/Government/10k/en-ta_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/10k/en-ta_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/10k/en-ta_final_model/generation_config.json b/Government/10k/en-ta_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/10k/en-ta_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/10k/en-ta_final_model/pytorch_model.bin b/Government/10k/en-ta_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..4051d4d27fb6e6bf55e5bbea6d8fec392d4802be --- /dev/null +++ b/Government/10k/en-ta_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b0b002987a5effbb2d8549c38c695dc444511ae926d31c3d462a2adea4ffd51 +size 2468870723 diff --git a/Government/10k/en-ta_final_model/training_args.bin b/Government/10k/en-ta_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..9452f9664e211e73d684edda1d258750fa775863 --- /dev/null +++ b/Government/10k/en-ta_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a85f4d0ad2096fdd87467285085b54ed5ebd25b8f8b06c9f5185ae5946e10ee6 +size 4143 diff --git a/Government/10k/nllb_bible_eval_government_en-si_trained_10k.csv b/Government/10k/nllb_bible_eval_government_en-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..2d9e07ac5aa6fea05068cc7476a668a117829629 --- /dev/null +++ b/Government/10k/nllb_bible_eval_government_en-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,9.883167308842296,8.08164852593724,0.1224908082681523,1.6782909910263053,1.5168849986089101,0.4502922486960888 diff --git a/Government/10k/nllb_bible_eval_government_en-ta_trained_10k.csv b/Government/10k/nllb_bible_eval_government_en-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..5fd4c80dc950a285dfff7b67c8504e625612557f --- /dev/null +++ b/Government/10k/nllb_bible_eval_government_en-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,39.04016452320649,33.15214888174151,2.8184624261362243,13.751787694216905,12.653757012332289,0.7956526825428009 diff --git a/Government/10k/nllb_bible_eval_government_si-en_trained_10k.csv b/Government/10k/nllb_bible_eval_government_si-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..f7662251fca081ff2a93b21c895e98a81ed579bd --- /dev/null +++ b/Government/10k/nllb_bible_eval_government_si-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,45.07330718302318,44.13769071062262,21.65309928276407,22.884942868586275,22.55774563190717,0.7763391099870205 diff --git a/Government/10k/nllb_bible_eval_government_ta-en_trained_10k.csv b/Government/10k/nllb_bible_eval_government_ta-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..74802513c6f20c2fffca6b68aa4ebcb7a0514db0 --- /dev/null +++ b/Government/10k/nllb_bible_eval_government_ta-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,39.66184981994951,38.70144410559059,16.635038725941627,18.45114127947077,17.833644803220327,0.7544466648101806 diff --git a/Government/10k/nllb_floresplus_eval_government_en-si_trained_10k.csv b/Government/10k/nllb_floresplus_eval_government_en-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..bcda6e1f9a8007cecfcafa679de25ae30b8b7783 --- /dev/null +++ b/Government/10k/nllb_floresplus_eval_government_en-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,7.101437099437691,5.433916452424211,0.006096076575261776,0.06517614911605754,0.12039641693926925,0.4025747983646487 diff --git a/Government/10k/nllb_floresplus_eval_government_en-ta_trained_10k.csv b/Government/10k/nllb_floresplus_eval_government_en-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..6e019dbb8437c0eaf24d2382b28c661087145ae3 --- /dev/null +++ b/Government/10k/nllb_floresplus_eval_government_en-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,51.33106002932208,45.11555555810329,10.204760623961306,24.73892903928464,24.451190328689275,0.8639418881519039 diff --git a/Government/10k/nllb_floresplus_eval_government_si-en_trained_10k.csv b/Government/10k/nllb_floresplus_eval_government_si-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..57cfa457550191d7b7350e3d4c9f4a2a4904819a --- /dev/null +++ b/Government/10k/nllb_floresplus_eval_government_si-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,56.99668379741255,54.809779080789056,28.151436901598544,29.99779167953794,30.877600166083894,0.8655534790085239 diff --git a/Government/10k/nllb_floresplus_eval_government_si-ta_trained_10k.csv b/Government/10k/nllb_floresplus_eval_government_si-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..52fb6ca2606b5571d8776ec1d413a899921a186e --- /dev/null +++ b/Government/10k/nllb_floresplus_eval_government_si-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,46.836618790209656,40.6531192195075,7.401129420771832,19.285564207292083,19.20115461267066,0.8505266855651211 diff --git a/Government/10k/nllb_floresplus_eval_government_ta-en_trained_10k.csv b/Government/10k/nllb_floresplus_eval_government_ta-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..fe71ee76d23de8a9f23782b858ff357b5440091a --- /dev/null +++ b/Government/10k/nllb_floresplus_eval_government_ta-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,55.43089314826211,53.2716331352053,26.94205797497817,28.534227665681914,29.344144666730266,0.8467663534780736 diff --git a/Government/10k/nllb_floresplus_eval_government_ta-si_trained_10k.csv b/Government/10k/nllb_floresplus_eval_government_ta-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..5f281dbd344c753d8cc0f5d930b50ad50910ae6d --- /dev/null +++ b/Government/10k/nllb_floresplus_eval_government_ta-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,38.55509522752336,34.58953090015751,7.345789897949188,20.669067893370794,24.236483807724387,0.8423951678567018 diff --git a/Government/10k/nllb_government_eval_bible_en-si_trained_10k.csv b/Government/10k/nllb_government_eval_bible_en-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..2d9e07ac5aa6fea05068cc7476a668a117829629 --- /dev/null +++ b/Government/10k/nllb_government_eval_bible_en-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,9.883167308842296,8.08164852593724,0.1224908082681523,1.6782909910263053,1.5168849986089101,0.4502922486960888 diff --git a/Government/10k/nllb_government_eval_bible_en-ta_trained_10k.csv b/Government/10k/nllb_government_eval_bible_en-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..5fd4c80dc950a285dfff7b67c8504e625612557f --- /dev/null +++ b/Government/10k/nllb_government_eval_bible_en-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,39.04016452320649,33.15214888174151,2.8184624261362243,13.751787694216905,12.653757012332289,0.7956526825428009 diff --git a/Government/10k/nllb_government_eval_bible_si-en_trained_10k.csv b/Government/10k/nllb_government_eval_bible_si-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..f7662251fca081ff2a93b21c895e98a81ed579bd --- /dev/null +++ b/Government/10k/nllb_government_eval_bible_si-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,45.07330718302318,44.13769071062262,21.65309928276407,22.884942868586275,22.55774563190717,0.7763391099870205 diff --git a/Government/10k/nllb_government_eval_bible_ta-en_trained_10k.csv b/Government/10k/nllb_government_eval_bible_ta-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..74802513c6f20c2fffca6b68aa4ebcb7a0514db0 --- /dev/null +++ b/Government/10k/nllb_government_eval_bible_ta-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,39.66184981994951,38.70144410559059,16.635038725941627,18.45114127947077,17.833644803220327,0.7544466648101806 diff --git a/Government/10k/nllb_government_eval_en-si_trained_10k.csv b/Government/10k/nllb_government_eval_en-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..475a2f0e3d22a387f2fce4c54e4a6dffe1a06e34 --- /dev/null +++ b/Government/10k/nllb_government_eval_en-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,7.902449368038944,6.167021841193213,0.12913685919379658,0.6381179265527395,1.0437367875692796,0.4717434812504041 diff --git a/Government/10k/nllb_government_eval_en-ta_trained_10k.csv b/Government/10k/nllb_government_eval_en-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..52c165fa81c7fda8d5688d9d0c41ee042d0d0070 --- /dev/null +++ b/Government/10k/nllb_government_eval_en-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,58.5163631920332,52.47541722887641,18.705159304284056,34.31014350673382,34.46902785490381,0.894309900259454 diff --git a/Government/10k/nllb_government_eval_government_en-si_trained_10k.csv b/Government/10k/nllb_government_eval_government_en-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..8c6e0989f28710592438ef07c8ac6bc754fb6dbe --- /dev/null +++ b/Government/10k/nllb_government_eval_government_en-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,7.931510211422728,6.177940517062402,0.14382494181276936,0.6302031145633212,1.0312510931564078,0.4728857491314411 diff --git a/Government/10k/nllb_government_eval_government_en-ta_trained_10k.csv b/Government/10k/nllb_government_eval_government_en-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..8fc06293d55d0485bc0b2c404ac8d83dcb09f5b2 --- /dev/null +++ b/Government/10k/nllb_government_eval_government_en-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,59.06764975646131,52.913556093013966,18.823324236734802,34.91316090158198,34.99057592315347,0.8964461959302426 diff --git a/Government/10k/nllb_government_eval_government_si-en_trained_10k.csv b/Government/10k/nllb_government_eval_government_si-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..719ebc9a968042afcc552e4802bfdfb5292ecee4 --- /dev/null +++ b/Government/10k/nllb_government_eval_government_si-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,62.73844567044389,59.981579805973475,34.628357862719746,36.654224341305095,37.66515942097653,0.8708139849305153 diff --git a/Government/10k/nllb_government_eval_government_si-ta_trained_10k.csv b/Government/10k/nllb_government_eval_government_si-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b06c97dba135814bbc170026377225cdd7729877 --- /dev/null +++ b/Government/10k/nllb_government_eval_government_si-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,58.14078422125692,51.75968329106598,18.738581491878676,33.452466220048635,33.60204293320304,0.8897950460314751 diff --git a/Government/10k/nllb_government_eval_government_ta-en_trained_10k.csv b/Government/10k/nllb_government_eval_government_ta-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..f60dbf766d2304b6a99087cef271880864516a3e --- /dev/null +++ b/Government/10k/nllb_government_eval_government_ta-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,61.75798035193796,59.12903021473232,34.39499240839612,35.92470564198692,36.87249579225605,0.8535116041898727 diff --git a/Government/10k/nllb_government_eval_government_ta-si_trained_10k.csv b/Government/10k/nllb_government_eval_government_ta-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..24eeec70be2b99382a4956fd0646da49a3853d32 --- /dev/null +++ b/Government/10k/nllb_government_eval_government_ta-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,53.003758933588315,48.821356943977854,19.31417763140684,37.860609529253814,44.2256042868842,0.8984571205079556 diff --git a/Government/10k/nllb_government_eval_si-en_trained_10k.csv b/Government/10k/nllb_government_eval_si-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..dbfe77a3d69aebd52c5ee35e6c80882782584f71 --- /dev/null +++ b/Government/10k/nllb_government_eval_si-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,62.77163616395269,60.088768360210246,34.996756733529814,36.878655165742614,37.87601722462576,0.8708597361774464 diff --git a/Government/10k/nllb_government_eval_si-ta_trained_10k.csv b/Government/10k/nllb_government_eval_si-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..aa006c0dcb35c5adbde53b71d3cbc81210377c9d --- /dev/null +++ b/Government/10k/nllb_government_eval_si-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,57.709379937212915,51.49821637879505,18.361711830471652,33.027753690726136,33.252341828411595,0.8876093401426615 diff --git a/Government/10k/nllb_government_eval_ta-en_trained_10k.csv b/Government/10k/nllb_government_eval_ta-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b70c1279f4a2d1982ae62b86b509416ca7ff425b --- /dev/null +++ b/Government/10k/nllb_government_eval_ta-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,61.514353664271894,58.91580691940435,34.20673276601652,35.4759236821559,36.414960010511194,0.8512381771200694 diff --git a/Government/10k/nllb_government_eval_ta-si_trained_10k.csv b/Government/10k/nllb_government_eval_ta-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..a29999f835bf5254e2fb8c819bc38ebb67bd93d9 --- /dev/null +++ b/Government/10k/nllb_government_eval_ta-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,52.52915163414007,48.39236805377741,18.89862517601417,37.437326547884425,43.68221773382746,0.8974438560298604 diff --git a/Government/10k/nllb_health_eval_government_en-si_trained_10k.csv b/Government/10k/nllb_health_eval_government_en-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..1975b03df0f5cab7934e971fda08da13fc7da80b --- /dev/null +++ b/Government/10k/nllb_health_eval_government_en-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,5.863895093672576,4.60535828879025,0.009505375970596416,0.031490697571333916,0.07411878218389448,0.3987277927693356 diff --git a/Government/10k/nllb_health_eval_government_en-ta_trained_10k.csv b/Government/10k/nllb_health_eval_government_en-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..98f326d39dfc41e11b5aadc56a78ece7543e11ae --- /dev/null +++ b/Government/10k/nllb_health_eval_government_en-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,58.33440706989346,51.67512736768906,13.453431672446712,31.752384138597638,30.682964623161613,0.8420447525680065 diff --git a/Government/10k/nllb_health_eval_government_si-en_trained_10k.csv b/Government/10k/nllb_health_eval_government_si-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..d5ba79395eb5583bb4c61ef24adc8675d4afe583 --- /dev/null +++ b/Government/10k/nllb_health_eval_government_si-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,57.48438223269277,54.351559869063756,23.155730792464308,29.116973964187466,29.53171280944268,0.8489664266953307 diff --git a/Government/10k/nllb_health_eval_government_si-ta_trained_10k.csv b/Government/10k/nllb_health_eval_government_si-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..9e8e3581f778526f75585e066426bb5e1a8f9b49 --- /dev/null +++ b/Government/10k/nllb_health_eval_government_si-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,51.05445562150454,44.1282584335198,7.43661377510804,22.47372435594028,21.762093358602204,0.8153357454480565 diff --git a/Government/10k/nllb_health_eval_government_ta-en_trained_10k.csv b/Government/10k/nllb_health_eval_government_ta-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..66a3f62fa1abf9ab03e103ac13aa80788a64cadc --- /dev/null +++ b/Government/10k/nllb_health_eval_government_ta-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,58.86040077530651,56.06605342114281,28.22128878884421,31.735624558849164,32.58044952164305,0.831830090880394 diff --git a/Government/10k/nllb_health_eval_government_ta-si_trained_10k.csv b/Government/10k/nllb_health_eval_government_ta-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..5893de37421e81afbd6e1363bf55ea44eb25c202 --- /dev/null +++ b/Government/10k/nllb_health_eval_government_ta-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,37.03803416828515,33.12868511463382,8.343706010206716,20.31863699567055,22.899490207344897,0.7829769134749563 diff --git a/Government/10k/nllb_jailbreak_eval_government_en-si_trained_10k.csv b/Government/10k/nllb_jailbreak_eval_government_en-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..92ed34d934ae2fb3cdcf2df385561b5794de01cb --- /dev/null +++ b/Government/10k/nllb_jailbreak_eval_government_en-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,6.071189768294846,4.833782795198674,0.04147568853163803,0.11702441115988822,0.1268903339098027,0.4328604847192764 diff --git a/Government/10k/nllb_jailbreak_eval_government_en-ta_trained_10k.csv b/Government/10k/nllb_jailbreak_eval_government_en-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..d9a3088128193fa3a6358878767708b7cd0267c6 --- /dev/null +++ b/Government/10k/nllb_jailbreak_eval_government_en-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,56.21916360382115,51.056636792979745,18.72396709640088,30.6603151104876,29.946766608740884,0.8520219075381756 diff --git a/Government/10k/nllb_jailbreak_eval_government_si-en_trained_10k.csv b/Government/10k/nllb_jailbreak_eval_government_si-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..189bf64ed31e9fba9fa3ef44485219482607d04e --- /dev/null +++ b/Government/10k/nllb_jailbreak_eval_government_si-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,53.9237987618724,52.13672088503674,28.302042608214858,29.449375428703405,30.324157627931157,0.8554237298518419 diff --git a/Government/10k/nllb_jailbreak_eval_government_si-ta_trained_10k.csv b/Government/10k/nllb_jailbreak_eval_government_si-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..28126f14111392e338349b63076117efd3d88c22 --- /dev/null +++ b/Government/10k/nllb_jailbreak_eval_government_si-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,51.49816428271053,46.14675934942884,14.048096092055278,23.686153244821465,22.950627944652986,0.8389946821331978 diff --git a/Government/10k/nllb_jailbreak_eval_government_ta-en_trained_10k.csv b/Government/10k/nllb_jailbreak_eval_government_ta-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..57da2c4c9f5d31276a1f0ff10e075f063466cb7c --- /dev/null +++ b/Government/10k/nllb_jailbreak_eval_government_ta-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,54.54161727059267,53.27600517263037,30.879667285745647,29.763568275179793,30.61953628467286,0.8417990252375602 diff --git a/Government/10k/nllb_jailbreak_eval_government_ta-si_trained_10k.csv b/Government/10k/nllb_jailbreak_eval_government_ta-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..54f6ecc659e9e580191c9269bbb6fcb9c6ac5514 --- /dev/null +++ b/Government/10k/nllb_jailbreak_eval_government_ta-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,38.24081710579901,35.136434247784166,11.725754619794358,20.08262685955402,22.570575210159387,0.8129719280898571 diff --git a/Government/10k/nllb_literature-children_eval_government_en-si_trained_10k.csv b/Government/10k/nllb_literature-children_eval_government_en-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..6da349633bb5b462672fcab63b7ba87ecfb6fa51 --- /dev/null +++ b/Government/10k/nllb_literature-children_eval_government_en-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,6.376891953361408,4.851865039223764,0.01805068331672696,0.044259882485194114,0.06432299953162877,0.42568679946661 diff --git a/Government/10k/nllb_literature-children_eval_government_en-ta_trained_10k.csv b/Government/10k/nllb_literature-children_eval_government_en-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..324df62543b90da8e6792afac1be2b8d0a9a9390 --- /dev/null +++ b/Government/10k/nllb_literature-children_eval_government_en-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,52.80011373193968,47.35622145271364,12.741313369670017,26.446102922675102,25.894374314433485,0.8346316877890516 diff --git a/Government/10k/nllb_literature-children_eval_government_si-en_trained_10k.csv b/Government/10k/nllb_literature-children_eval_government_si-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..3751d66521baba0d2cca7d0b1c341c7a52bfda84 --- /dev/null +++ b/Government/10k/nllb_literature-children_eval_government_si-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,43.69342098316437,42.546686819581005,19.19400950178123,18.807084984445588,18.98680461063105,0.8059120564162732 diff --git a/Government/10k/nllb_literature-children_eval_government_si-ta_trained_10k.csv b/Government/10k/nllb_literature-children_eval_government_si-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..294ed03ebacabce23c381f32f70a6ffba4cd5fd2 --- /dev/null +++ b/Government/10k/nllb_literature-children_eval_government_si-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,44.03344124736107,38.56380996498546,6.175457135375574,15.423270454345603,14.782094050787254,0.8010131987495399 diff --git a/Government/10k/nllb_literature-children_eval_government_ta-en_trained_10k.csv b/Government/10k/nllb_literature-children_eval_government_ta-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..60a2459458fa851e6f4fb561e8111e63908ce0ea --- /dev/null +++ b/Government/10k/nllb_literature-children_eval_government_ta-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,50.17644250494791,49.375202104535795,26.14306719178659,25.516428252417697,26.009296617616663,0.8162300244870561 diff --git a/Government/10k/nllb_literature-children_eval_government_ta-si_trained_10k.csv b/Government/10k/nllb_literature-children_eval_government_ta-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..e55f97377725876ef2f26b4f15a622219cda01cb --- /dev/null +++ b/Government/10k/nllb_literature-children_eval_government_ta-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,30.75703384311096,27.58530232637968,4.349458365486436,10.994093247594385,12.700897148810842,0.7969094352745121 diff --git a/Government/10k/nllb_literature-kathlyn_eval_government_en-si_trained_10k.csv b/Government/10k/nllb_literature-kathlyn_eval_government_en-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..6da349633bb5b462672fcab63b7ba87ecfb6fa51 --- /dev/null +++ b/Government/10k/nllb_literature-kathlyn_eval_government_en-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,6.376891953361408,4.851865039223764,0.01805068331672696,0.044259882485194114,0.06432299953162877,0.42568679946661 diff --git a/Government/10k/nllb_literature-kathlyn_eval_government_en-ta_trained_10k.csv b/Government/10k/nllb_literature-kathlyn_eval_government_en-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..324df62543b90da8e6792afac1be2b8d0a9a9390 --- /dev/null +++ b/Government/10k/nllb_literature-kathlyn_eval_government_en-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,52.80011373193968,47.35622145271364,12.741313369670017,26.446102922675102,25.894374314433485,0.8346316877890516 diff --git a/Government/10k/nllb_literature-kathlyn_eval_government_si-en_trained_10k.csv b/Government/10k/nllb_literature-kathlyn_eval_government_si-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..3751d66521baba0d2cca7d0b1c341c7a52bfda84 --- /dev/null +++ b/Government/10k/nllb_literature-kathlyn_eval_government_si-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,43.69342098316437,42.546686819581005,19.19400950178123,18.807084984445588,18.98680461063105,0.8059120564162732 diff --git a/Government/10k/nllb_literature-kathlyn_eval_government_si-ta_trained_10k.csv b/Government/10k/nllb_literature-kathlyn_eval_government_si-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..294ed03ebacabce23c381f32f70a6ffba4cd5fd2 --- /dev/null +++ b/Government/10k/nllb_literature-kathlyn_eval_government_si-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,44.03344124736107,38.56380996498546,6.175457135375574,15.423270454345603,14.782094050787254,0.8010131987495399 diff --git a/Government/10k/nllb_literature-kathlyn_eval_government_ta-en_trained_10k.csv b/Government/10k/nllb_literature-kathlyn_eval_government_ta-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..60a2459458fa851e6f4fb561e8111e63908ce0ea --- /dev/null +++ b/Government/10k/nllb_literature-kathlyn_eval_government_ta-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,50.17644250494791,49.375202104535795,26.14306719178659,25.516428252417697,26.009296617616663,0.8162300244870561 diff --git a/Government/10k/nllb_literature-kathlyn_eval_government_ta-si_trained_10k.csv b/Government/10k/nllb_literature-kathlyn_eval_government_ta-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..e55f97377725876ef2f26b4f15a622219cda01cb --- /dev/null +++ b/Government/10k/nllb_literature-kathlyn_eval_government_ta-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,30.75703384311096,27.58530232637968,4.349458365486436,10.994093247594385,12.700897148810842,0.7969094352745121 diff --git a/Government/10k/nllb_literature-old_eval_government_en-si_trained_10k.csv b/Government/10k/nllb_literature-old_eval_government_en-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..6da349633bb5b462672fcab63b7ba87ecfb6fa51 --- /dev/null +++ b/Government/10k/nllb_literature-old_eval_government_en-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,6.376891953361408,4.851865039223764,0.01805068331672696,0.044259882485194114,0.06432299953162877,0.42568679946661 diff --git a/Government/10k/nllb_literature-old_eval_government_en-ta_trained_10k.csv b/Government/10k/nllb_literature-old_eval_government_en-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..324df62543b90da8e6792afac1be2b8d0a9a9390 --- /dev/null +++ b/Government/10k/nllb_literature-old_eval_government_en-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,52.80011373193968,47.35622145271364,12.741313369670017,26.446102922675102,25.894374314433485,0.8346316877890516 diff --git a/Government/10k/nllb_literature-old_eval_government_si-en_trained_10k.csv b/Government/10k/nllb_literature-old_eval_government_si-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..3751d66521baba0d2cca7d0b1c341c7a52bfda84 --- /dev/null +++ b/Government/10k/nllb_literature-old_eval_government_si-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,43.69342098316437,42.546686819581005,19.19400950178123,18.807084984445588,18.98680461063105,0.8059120564162732 diff --git a/Government/10k/nllb_literature-old_eval_government_si-ta_trained_10k.csv b/Government/10k/nllb_literature-old_eval_government_si-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..294ed03ebacabce23c381f32f70a6ffba4cd5fd2 --- /dev/null +++ b/Government/10k/nllb_literature-old_eval_government_si-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,44.03344124736107,38.56380996498546,6.175457135375574,15.423270454345603,14.782094050787254,0.8010131987495399 diff --git a/Government/10k/nllb_literature-old_eval_government_ta-en_trained_10k.csv b/Government/10k/nllb_literature-old_eval_government_ta-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..60a2459458fa851e6f4fb561e8111e63908ce0ea --- /dev/null +++ b/Government/10k/nllb_literature-old_eval_government_ta-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,50.17644250494791,49.375202104535795,26.14306719178659,25.516428252417697,26.009296617616663,0.8162300244870561 diff --git a/Government/10k/nllb_literature-old_eval_government_ta-si_trained_10k.csv b/Government/10k/nllb_literature-old_eval_government_ta-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..e55f97377725876ef2f26b4f15a622219cda01cb --- /dev/null +++ b/Government/10k/nllb_literature-old_eval_government_ta-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,30.75703384311096,27.58530232637968,4.349458365486436,10.994093247594385,12.700897148810842,0.7969094352745121 diff --git a/Government/10k/nllb_literature_eval_government_en-si_trained_10k.csv b/Government/10k/nllb_literature_eval_government_en-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..6da349633bb5b462672fcab63b7ba87ecfb6fa51 --- /dev/null +++ b/Government/10k/nllb_literature_eval_government_en-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,6.376891953361408,4.851865039223764,0.01805068331672696,0.044259882485194114,0.06432299953162877,0.42568679946661 diff --git a/Government/10k/nllb_literature_eval_government_en-ta_trained_10k.csv b/Government/10k/nllb_literature_eval_government_en-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..324df62543b90da8e6792afac1be2b8d0a9a9390 --- /dev/null +++ b/Government/10k/nllb_literature_eval_government_en-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,52.80011373193968,47.35622145271364,12.741313369670017,26.446102922675102,25.894374314433485,0.8346316877890516 diff --git a/Government/10k/nllb_literature_eval_government_si-en_trained_10k.csv b/Government/10k/nllb_literature_eval_government_si-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..3751d66521baba0d2cca7d0b1c341c7a52bfda84 --- /dev/null +++ b/Government/10k/nllb_literature_eval_government_si-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,43.69342098316437,42.546686819581005,19.19400950178123,18.807084984445588,18.98680461063105,0.8059120564162732 diff --git a/Government/10k/nllb_literature_eval_government_si-ta_trained_10k.csv b/Government/10k/nllb_literature_eval_government_si-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..294ed03ebacabce23c381f32f70a6ffba4cd5fd2 --- /dev/null +++ b/Government/10k/nllb_literature_eval_government_si-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,44.03344124736107,38.56380996498546,6.175457135375574,15.423270454345603,14.782094050787254,0.8010131987495399 diff --git a/Government/10k/nllb_literature_eval_government_ta-en_trained_10k.csv b/Government/10k/nllb_literature_eval_government_ta-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..60a2459458fa851e6f4fb561e8111e63908ce0ea --- /dev/null +++ b/Government/10k/nllb_literature_eval_government_ta-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,50.17644250494791,49.375202104535795,26.14306719178659,25.516428252417697,26.009296617616663,0.8162300244870561 diff --git a/Government/10k/nllb_literature_eval_government_ta-si_trained_10k.csv b/Government/10k/nllb_literature_eval_government_ta-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..e55f97377725876ef2f26b4f15a622219cda01cb --- /dev/null +++ b/Government/10k/nllb_literature_eval_government_ta-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,30.75703384311096,27.58530232637968,4.349458365486436,10.994093247594385,12.700897148810842,0.7969094352745121 diff --git a/Government/10k/nllb_maths_eval_government_en-si_trained_10k.csv b/Government/10k/nllb_maths_eval_government_en-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..5537ddeac8a35ec9a81023126ef4214eb444b6bc --- /dev/null +++ b/Government/10k/nllb_maths_eval_government_en-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,6.786315824711309,5.199195420389427,0.015374364795371227,0.02212559402322474,0.04426990822951497,0.42914291797578336 diff --git a/Government/10k/nllb_maths_eval_government_en-ta_trained_10k.csv b/Government/10k/nllb_maths_eval_government_en-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..85e33795f8e06c830c61eceab12ddbd0bdfec36f --- /dev/null +++ b/Government/10k/nllb_maths_eval_government_en-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,50.452883635523364,46.509719373423394,20.235962579515416,26.069489072072418,25.576828778799833,0.8329620827734471 diff --git a/Government/10k/nllb_maths_eval_government_si-en_trained_10k.csv b/Government/10k/nllb_maths_eval_government_si-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..01a6201edc3b84d20f7de1a3481c2fc5ac9ee161 --- /dev/null +++ b/Government/10k/nllb_maths_eval_government_si-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,57.016858143481265,56.42857018784528,37.39970757061199,37.218307322272175,37.74691390025642,0.8421808927953244 diff --git a/Government/10k/nllb_maths_eval_government_ta-en_trained_10k.csv b/Government/10k/nllb_maths_eval_government_ta-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..86dbea78da7ffbb9eb53639a0722f913c2bb3cca --- /dev/null +++ b/Government/10k/nllb_maths_eval_government_ta-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,53.84290157280306,53.064500963953186,33.947287247661755,33.50291595801113,34.28204636472693,0.8322957644760609 diff --git a/Government/10k/nllb_news_eval_government_en-si_trained_10k.csv b/Government/10k/nllb_news_eval_government_en-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..51f763c8775f7349e9e954cc20146e4ce4c5895b --- /dev/null +++ b/Government/10k/nllb_news_eval_government_en-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,6.783172487448554,5.22918019278509,0.014067933486200796,0.1930584845597379,0.3257971104827631,0.4174291704893112 diff --git a/Government/10k/nllb_news_eval_government_en-ta_trained_10k.csv b/Government/10k/nllb_news_eval_government_en-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..42aebb8aba5a244f1011454fb3c3ad93de1a1edd --- /dev/null +++ b/Government/10k/nllb_news_eval_government_en-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,58.12530632693921,52.442792795931496,16.683589966304222,33.32834972613715,33.03293022848524,0.8653240330876356 diff --git a/Government/10k/nllb_news_eval_government_si-en_trained_10k.csv b/Government/10k/nllb_news_eval_government_si-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..c578e37dcecb0d249a744f5fed8553625d6b879e --- /dev/null +++ b/Government/10k/nllb_news_eval_government_si-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,55.16912437882716,52.41452548941682,23.225707785715844,25.69047415193137,26.629105316212765,0.8533352153301239 diff --git a/Government/10k/nllb_news_eval_government_si-ta_trained_10k.csv b/Government/10k/nllb_news_eval_government_si-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..80cac8d3f9a49b3cc86335f4c5e2efc3b9f55286 --- /dev/null +++ b/Government/10k/nllb_news_eval_government_si-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,50.5543703039938,44.53653984259175,8.793252120251214,21.94804983747638,22.03497418894363,0.8499003647255802 diff --git a/Government/10k/nllb_news_eval_government_ta-en_trained_10k.csv b/Government/10k/nllb_news_eval_government_ta-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..66678bae4f8833e11b145af23a4cdbf1eb051e24 --- /dev/null +++ b/Government/10k/nllb_news_eval_government_ta-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,59.44241851014225,57.543245973599454,32.02068197304198,32.989606765740824,34.080175890389235,0.8565062239287369 diff --git a/Government/10k/nllb_news_eval_government_ta-si_trained_10k.csv b/Government/10k/nllb_news_eval_government_ta-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..996d5a175723a38735371e807584f578c8b8105e --- /dev/null +++ b/Government/10k/nllb_news_eval_government_ta-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,37.45298774264474,33.35210096895649,6.719988257674699,19.863863864849968,23.370670517369696,0.825703306333913 diff --git a/Government/10k/nllb_nllb_eval_government_en-si_trained_10k.csv b/Government/10k/nllb_nllb_eval_government_en-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..00d3df2157398180319badc27ca0c79dafcae75b --- /dev/null +++ b/Government/10k/nllb_nllb_eval_government_en-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,7.869460496102654,6.09056504503559,0.10536894394498288,0.49852360413969166,0.5363278084194655,0.46018390784517144 diff --git a/Government/10k/nllb_nllb_eval_government_en-ta_trained_10k.csv b/Government/10k/nllb_nllb_eval_government_en-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..a31a39a55c45924e0acc60cfb68f8ffbba8002a4 --- /dev/null +++ b/Government/10k/nllb_nllb_eval_government_en-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,55.701610270706645,50.770739016004974,16.868370570975276,29.73773902223746,29.018614315651423,0.9026897527385341 diff --git a/Government/10k/nllb_nllb_eval_government_si-en_trained_10k.csv b/Government/10k/nllb_nllb_eval_government_si-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..c467ecd8c0fbf11bc3545d0c737c26a57ffaa925 --- /dev/null +++ b/Government/10k/nllb_nllb_eval_government_si-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,62.2963419600587,61.51956040246825,41.345476841818595,41.07059044087541,41.2644213431573,0.8979451639677248 diff --git a/Government/10k/nllb_nllb_eval_government_si-ta_trained_10k.csv b/Government/10k/nllb_nllb_eval_government_si-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..055ed5658c8a4f46fef1c7e3d0b31ab9fb3a35f5 --- /dev/null +++ b/Government/10k/nllb_nllb_eval_government_si-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,53.303848750785654,48.157417447088996,13.72785644479278,26.326082651119886,25.30305922974426,0.89893731930169 diff --git a/Government/10k/nllb_nllb_eval_government_ta-en_trained_10k.csv b/Government/10k/nllb_nllb_eval_government_ta-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..a5d915c044dab6302ea1dcd722859afb809530af --- /dev/null +++ b/Government/10k/nllb_nllb_eval_government_ta-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,62.68994539190883,62.004399618856276,42.1481485980801,41.777717693770626,41.99095634978417,0.8899459793635505 diff --git a/Government/10k/nllb_nllb_eval_government_ta-si_trained_10k.csv b/Government/10k/nllb_nllb_eval_government_ta-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..606a9ee5051159617d530bababec01b89805a513 --- /dev/null +++ b/Government/10k/nllb_nllb_eval_government_ta-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,41.14962374342426,37.83151456057326,10.654342943989796,21.77550293245442,25.623770716708265,0.8870724192334603 diff --git a/Government/10k/nllb_opensubtitles_eval_government_en-si_trained_10k.csv b/Government/10k/nllb_opensubtitles_eval_government_en-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..70ce1a080b8e9268706e9b389b67355d41670cf3 --- /dev/null +++ b/Government/10k/nllb_opensubtitles_eval_government_en-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,9.800694865178825,8.375756621578411,0.5730870903662663,1.4827913789729208,1.4035582820664598,0.5102573437988758 diff --git a/Government/10k/nllb_opensubtitles_eval_government_en-ta_trained_10k.csv b/Government/10k/nllb_opensubtitles_eval_government_en-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..f2e59c8d042c833fd7f17b3090c61883418797b2 --- /dev/null +++ b/Government/10k/nllb_opensubtitles_eval_government_en-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,44.84399057895997,41.46688671504414,14.827865536541932,25.05746381065306,25.058395301870654,0.826336010709405 diff --git a/Government/10k/nllb_opensubtitles_eval_government_si-en_trained_10k.csv b/Government/10k/nllb_opensubtitles_eval_government_si-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..68e85b7bada637b1965fb90a14b40203299e9fe2 --- /dev/null +++ b/Government/10k/nllb_opensubtitles_eval_government_si-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,62.570689566558116,61.76668331962446,40.21677890085725,41.20475903534689,41.44644800048119,0.8913131302297115 diff --git a/Government/10k/nllb_opensubtitles_eval_government_si-ta_trained_10k.csv b/Government/10k/nllb_opensubtitles_eval_government_si-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..a4efbe85289a292ac62ab7225a49384e953ec3e4 --- /dev/null +++ b/Government/10k/nllb_opensubtitles_eval_government_si-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,53.154454788185866,48.712022861518555,15.884186660493718,28.40768736031095,27.62017506581964,0.8876451987326145 diff --git a/Government/10k/nllb_opensubtitles_eval_government_ta-en_trained_10k.csv b/Government/10k/nllb_opensubtitles_eval_government_ta-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..647ad11ea1b7affb78978de7c4cdce40fbf05fce --- /dev/null +++ b/Government/10k/nllb_opensubtitles_eval_government_ta-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,59.37888756243647,58.611080366930466,36.011471758769666,36.921736269966054,37.186287821937476,0.8704207756072283 diff --git a/Government/10k/nllb_opensubtitles_eval_government_ta-si_trained_10k.csv b/Government/10k/nllb_opensubtitles_eval_government_ta-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..30942d49db563e355af84742ce2503cb2163187b --- /dev/null +++ b/Government/10k/nllb_opensubtitles_eval_government_ta-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,32.779991853044535,29.769163253223102,6.336988867612174,17.075376725173953,18.95436559347672,0.8551582052856683 diff --git a/Government/10k/nllb_wikipedia_eval_government_en-si_trained_10k.csv b/Government/10k/nllb_wikipedia_eval_government_en-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..e6ce37b31d04dd47720218678c0df7109c0b491b --- /dev/null +++ b/Government/10k/nllb_wikipedia_eval_government_en-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,7.275620508536446,5.622158052913327,0.02142326080191755,0.1101172691185725,0.15954709264385347,0.42036394535005095 diff --git a/Government/10k/nllb_wikipedia_eval_government_en-ta_trained_10k.csv b/Government/10k/nllb_wikipedia_eval_government_en-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b59a3e814593b2c90e7676d901200925f263571a --- /dev/null +++ b/Government/10k/nllb_wikipedia_eval_government_en-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,53.76769626572674,47.42992356678965,11.011035596320127,27.818719548232682,27.688381316325337,0.878192524343729 diff --git a/Government/10k/nllb_wikipedia_eval_government_si-en_trained_10k.csv b/Government/10k/nllb_wikipedia_eval_government_si-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..8524eb7a04c32c443bc828bdfa7ca2361016b65f --- /dev/null +++ b/Government/10k/nllb_wikipedia_eval_government_si-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,54.564501797650244,52.47018091510572,25.835169532912634,28.99903907364859,29.498197853102567,0.8424103186130524 diff --git a/Government/10k/nllb_wikipedia_eval_government_si-ta_trained_10k.csv b/Government/10k/nllb_wikipedia_eval_government_si-ta_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ff73d4ccd47e254e0ad8b889da0bd5053475f732 --- /dev/null +++ b/Government/10k/nllb_wikipedia_eval_government_si-ta_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,47.526336143455275,41.23071711558105,6.9203346734527695,20.920052776866168,20.688363725891694,0.850786651223898 diff --git a/Government/10k/nllb_wikipedia_eval_government_ta-en_trained_10k.csv b/Government/10k/nllb_wikipedia_eval_government_ta-en_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..aaec186ee017b6cc8fae927614a5a35e7695aeed --- /dev/null +++ b/Government/10k/nllb_wikipedia_eval_government_ta-en_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,58.71705205081543,56.79024100743819,30.6912228651309,33.38481512896798,33.970360670851335,0.844701763510704 diff --git a/Government/10k/nllb_wikipedia_eval_government_ta-si_trained_10k.csv b/Government/10k/nllb_wikipedia_eval_government_ta-si_trained_10k.csv new file mode 100644 index 0000000000000000000000000000000000000000..3b601289ba7c27a2cc8797a70e353fd6eec79425 --- /dev/null +++ b/Government/10k/nllb_wikipedia_eval_government_ta-si_trained_10k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,38.4952395569435,34.30218847145008,6.345584298933479,19.66453846144426,22.87273195578639,0.8488303147852421 diff --git a/Government/10k/si-en/runs/Oct02_23-28-25_NLPC-Sita-MLM/events.out.tfevents.1759427905.NLPC-Sita-MLM.2581313.9 b/Government/10k/si-en/runs/Oct02_23-28-25_NLPC-Sita-MLM/events.out.tfevents.1759427905.NLPC-Sita-MLM.2581313.9 new file mode 100644 index 0000000000000000000000000000000000000000..bdb87e339a9dd84abe905dd5fdd82f3994c20918 --- /dev/null +++ b/Government/10k/si-en/runs/Oct02_23-28-25_NLPC-Sita-MLM/events.out.tfevents.1759427905.NLPC-Sita-MLM.2581313.9 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0ebfafbbd288ceb9842b89f1259c9159413b0cf0f4aa817b76c95182e8fdb63 +size 6601 diff --git a/Government/10k/si-en_final_model/config.json b/Government/10k/si-en_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/10k/si-en_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/10k/si-en_final_model/generation_config.json b/Government/10k/si-en_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/10k/si-en_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/10k/si-en_final_model/pytorch_model.bin b/Government/10k/si-en_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..1f2b6152f33ad07bb5c693dd2b797d4298bd51d2 --- /dev/null +++ b/Government/10k/si-en_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cb78b9ac7c24968a49f398e5e52020363101c3b561f99c57151cc5ef01a3d12 +size 2468870723 diff --git a/Government/10k/si-en_final_model/training_args.bin b/Government/10k/si-en_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..4323c1654844ae9ce6a9469ff6e8f534fd41bc98 --- /dev/null +++ b/Government/10k/si-en_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fe1abfb902e06aa805d315131f2b005df384e32a3e6e72aad72a9eee21f9319 +size 4143 diff --git a/Government/10k/si-ta/runs/Oct03_06-18-48_NLPC-Sita-MLM/events.out.tfevents.1759452528.NLPC-Sita-MLM.2581313.21 b/Government/10k/si-ta/runs/Oct03_06-18-48_NLPC-Sita-MLM/events.out.tfevents.1759452528.NLPC-Sita-MLM.2581313.21 new file mode 100644 index 0000000000000000000000000000000000000000..36bbeb2cf61247c021ad32d751f522781de30722 --- /dev/null +++ b/Government/10k/si-ta/runs/Oct03_06-18-48_NLPC-Sita-MLM/events.out.tfevents.1759452528.NLPC-Sita-MLM.2581313.21 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19e7d47dfaa6189e35faa555f6f14402564500250679d0eacc909f0c6a0fdf37 +size 6601 diff --git a/Government/10k/si-ta_final_model/config.json b/Government/10k/si-ta_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/10k/si-ta_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/10k/si-ta_final_model/generation_config.json b/Government/10k/si-ta_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/10k/si-ta_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/10k/si-ta_final_model/pytorch_model.bin b/Government/10k/si-ta_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..2b74c565ddf750aa4ebd0c00499e0d68726e255d --- /dev/null +++ b/Government/10k/si-ta_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3829d3dbcd386a31557995b21dd30a93a980da7cec8e9ea5912c007fb1ff665 +size 2468870723 diff --git a/Government/10k/si-ta_final_model/training_args.bin b/Government/10k/si-ta_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..16115fa3c6f1d2841af1b9422d7e6f9dedac12f2 --- /dev/null +++ b/Government/10k/si-ta_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9aec49ae20b2b366d61f1bffa94b91db859a989dbb413531c92b4e0fec4e844b +size 4143 diff --git a/Government/10k/ta-en/runs/Oct03_01-49-56_NLPC-Sita-MLM/events.out.tfevents.1759436396.NLPC-Sita-MLM.2581313.15 b/Government/10k/ta-en/runs/Oct03_01-49-56_NLPC-Sita-MLM/events.out.tfevents.1759436396.NLPC-Sita-MLM.2581313.15 new file mode 100644 index 0000000000000000000000000000000000000000..8daf086a747ae62f2895cf01cdb847e65daaf3c4 --- /dev/null +++ b/Government/10k/ta-en/runs/Oct03_01-49-56_NLPC-Sita-MLM/events.out.tfevents.1759436396.NLPC-Sita-MLM.2581313.15 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efb74009f00b9a500e891e65482817180fedcab942ced012b556bb4e86478c3a +size 6601 diff --git a/Government/10k/ta-en_final_model/config.json b/Government/10k/ta-en_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/10k/ta-en_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/10k/ta-en_final_model/generation_config.json b/Government/10k/ta-en_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/10k/ta-en_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/10k/ta-en_final_model/pytorch_model.bin b/Government/10k/ta-en_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..ee2fc47e8f64de7d42bd5a42b756c1411ba2f693 --- /dev/null +++ b/Government/10k/ta-en_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0e9ec47b6889748759ebcbeaf8351d502cb87841919fc6508e47bb08ac54082 +size 2468870723 diff --git a/Government/10k/ta-en_final_model/training_args.bin b/Government/10k/ta-en_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..681f3198f81c2def22d473b8173f50e95b7c38ac --- /dev/null +++ b/Government/10k/ta-en_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6849566467d7291aefa8741d8ad4c29c84d967d77822dd99a8dae62dd956af91 +size 4143 diff --git a/Government/10k/ta-si/runs/Oct03_08-49-38_NLPC-Sita-MLM/events.out.tfevents.1759461578.NLPC-Sita-MLM.2581313.27 b/Government/10k/ta-si/runs/Oct03_08-49-38_NLPC-Sita-MLM/events.out.tfevents.1759461578.NLPC-Sita-MLM.2581313.27 new file mode 100644 index 0000000000000000000000000000000000000000..88a01886d54b66f22c6ef91f00d80214a4aefb7d --- /dev/null +++ b/Government/10k/ta-si/runs/Oct03_08-49-38_NLPC-Sita-MLM/events.out.tfevents.1759461578.NLPC-Sita-MLM.2581313.27 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca898c4568289e16628caf91a80d6143f457eaf5fc8342bf6108b9d0c4cb2ee6 +size 6601 diff --git a/Government/10k/ta-si_final_model/config.json b/Government/10k/ta-si_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/10k/ta-si_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/10k/ta-si_final_model/generation_config.json b/Government/10k/ta-si_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/10k/ta-si_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/10k/ta-si_final_model/pytorch_model.bin b/Government/10k/ta-si_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..088269741a3aaa449b4504764fbea561fcd8dd57 --- /dev/null +++ b/Government/10k/ta-si_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5c310e8ce77572dce19a8831619e44f3c16043cf60e73c6144506f88f0a4e6c +size 2468870723 diff --git a/Government/10k/ta-si_final_model/training_args.bin b/Government/10k/ta-si_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..02f01d6989476234bf5794d6700ebb27673ea60b --- /dev/null +++ b/Government/10k/ta-si_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af4d4eae3cf6f86b3167dd2398f54f2cb0202d2735c833c088bc01d96bb66f28 +size 4143 diff --git a/Government/10k/tok-en-si/sentencepiece.bpe.model b/Government/10k/tok-en-si/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/10k/tok-en-si/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/10k/tok-en-si/special_tokens_map.json b/Government/10k/tok-en-si/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/10k/tok-en-si/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/10k/tok-en-si/tokenizer_config.json b/Government/10k/tok-en-si/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fd1139979a7c7c4188b26d922af637f4bfb23e1f --- /dev/null +++ b/Government/10k/tok-en-si/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "eng_Latn", + "tgt_lang": "sin_Sinh", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/10k/tok-en-ta/sentencepiece.bpe.model b/Government/10k/tok-en-ta/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/10k/tok-en-ta/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/10k/tok-en-ta/special_tokens_map.json b/Government/10k/tok-en-ta/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/10k/tok-en-ta/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/10k/tok-en-ta/tokenizer_config.json b/Government/10k/tok-en-ta/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ef8f263366a29cf3dd0c97df4b45c41fbbf160b5 --- /dev/null +++ b/Government/10k/tok-en-ta/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "eng_Latn", + "tgt_lang": "tam_Taml", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/10k/tok-si-en/sentencepiece.bpe.model b/Government/10k/tok-si-en/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/10k/tok-si-en/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/10k/tok-si-en/special_tokens_map.json b/Government/10k/tok-si-en/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/10k/tok-si-en/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/10k/tok-si-en/tokenizer_config.json b/Government/10k/tok-si-en/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f80b41ef163bc0a47ebef66aa6c1210dfe0b9792 --- /dev/null +++ b/Government/10k/tok-si-en/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "sin_Sinh", + "tgt_lang": "eng_Latn", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/10k/tok-si-ta/sentencepiece.bpe.model b/Government/10k/tok-si-ta/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/10k/tok-si-ta/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/10k/tok-si-ta/special_tokens_map.json b/Government/10k/tok-si-ta/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/10k/tok-si-ta/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/10k/tok-si-ta/tokenizer_config.json b/Government/10k/tok-si-ta/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..386158eef85de789e966861e91bebf69e9a4f32b --- /dev/null +++ b/Government/10k/tok-si-ta/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "sin_Sinh", + "tgt_lang": "tam_Taml", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/10k/tok-ta-en/sentencepiece.bpe.model b/Government/10k/tok-ta-en/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/10k/tok-ta-en/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/10k/tok-ta-en/special_tokens_map.json b/Government/10k/tok-ta-en/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/10k/tok-ta-en/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/10k/tok-ta-en/tokenizer_config.json b/Government/10k/tok-ta-en/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6deaa799386cb7438bf8a0f96250f961c73dec30 --- /dev/null +++ b/Government/10k/tok-ta-en/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "tam_Taml", + "tgt_lang": "eng_Latn", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/10k/tok-ta-si/sentencepiece.bpe.model b/Government/10k/tok-ta-si/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/10k/tok-ta-si/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/10k/tok-ta-si/special_tokens_map.json b/Government/10k/tok-ta-si/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/10k/tok-ta-si/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/10k/tok-ta-si/tokenizer_config.json b/Government/10k/tok-ta-si/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9526d1cacd39f5ee55eece61fdacbbc6ce50136d --- /dev/null +++ b/Government/10k/tok-ta-si/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "tam_Taml", + "tgt_lang": "sin_Sinh", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/15k/en-si_final_model/config.json b/Government/15k/en-si_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/15k/en-si_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/15k/en-si_final_model/generation_config.json b/Government/15k/en-si_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/15k/en-si_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/15k/en-si_final_model/pytorch_model.bin b/Government/15k/en-si_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..44fabb7f04ffa64845b5c913c97460ee6e804f14 --- /dev/null +++ b/Government/15k/en-si_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b22977288910016709e4b73f4b2928ef867d345070ff2ed400c1da9be0d094dc +size 2468870723 diff --git a/Government/15k/en-si_final_model/training_args.bin b/Government/15k/en-si_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..90ea1b6d25d426a30c00fd0ad6af43e6645a64e0 --- /dev/null +++ b/Government/15k/en-si_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:028d725c7a1a2700319378862c832b3231c5c9abcbed224afbcdf5e90c69348f +size 4143 diff --git a/Government/15k/en-ta/runs/Oct02_20-36-32_NLPC-Sita-MLM/events.out.tfevents.1759417592.NLPC-Sita-MLM.2581313.2 b/Government/15k/en-ta/runs/Oct02_20-36-32_NLPC-Sita-MLM/events.out.tfevents.1759417592.NLPC-Sita-MLM.2581313.2 new file mode 100644 index 0000000000000000000000000000000000000000..176c428062802dd7922075b202f55337d22915fb --- /dev/null +++ b/Government/15k/en-ta/runs/Oct02_20-36-32_NLPC-Sita-MLM/events.out.tfevents.1759417592.NLPC-Sita-MLM.2581313.2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:046e5f81d31cb6c45e951f8f163149ea693f9e0721675aad0a72452c308fc7b4 +size 7072 diff --git a/Government/15k/en-ta_final_model/config.json b/Government/15k/en-ta_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/15k/en-ta_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/15k/en-ta_final_model/generation_config.json b/Government/15k/en-ta_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/15k/en-ta_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/15k/en-ta_final_model/pytorch_model.bin b/Government/15k/en-ta_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..c5ea86ee246b8b866597b389b5d83449217488bd --- /dev/null +++ b/Government/15k/en-ta_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb134083c7f3aef3bae991bd2bc6c7208a3f9f9b8288f5d0bb1b55e95ae24084 +size 2468870723 diff --git a/Government/15k/en-ta_final_model/training_args.bin b/Government/15k/en-ta_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..4617c68fa152abdeab39d83c7815247f3ee26fdd --- /dev/null +++ b/Government/15k/en-ta_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2aff9fe3c1664363899a272255c9490f2bf65ce8bc2ed94fe490e9adab86b23 +size 4143 diff --git a/Government/15k/nllb_bible_eval_government_en-si_trained_15k.csv b/Government/15k/nllb_bible_eval_government_en-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..4ddc3ee11549212cbc6b550bd47f4a6fe8e35e17 --- /dev/null +++ b/Government/15k/nllb_bible_eval_government_en-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,10.333521780820627,8.453661202177699,0.06656251784264193,1.5401320162594445,1.3016910398397168,0.45090242648124695 diff --git a/Government/15k/nllb_bible_eval_government_en-ta_trained_15k.csv b/Government/15k/nllb_bible_eval_government_en-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..878ec19d947fef19fd1272a63a854aa2176b31e8 --- /dev/null +++ b/Government/15k/nllb_bible_eval_government_en-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,39.35146130939026,33.3893609747557,2.6919493432304913,14.209230774547496,12.92935046957307,0.798079612314701 diff --git a/Government/15k/nllb_bible_eval_government_si-en_trained_15k.csv b/Government/15k/nllb_bible_eval_government_si-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..6e7676db1c389f0573b31b276e41ffeb58b52894 --- /dev/null +++ b/Government/15k/nllb_bible_eval_government_si-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,45.43490212145097,44.461716905870546,21.836615165017935,23.113368684205426,22.79028610179947,0.7760936993658543 diff --git a/Government/15k/nllb_bible_eval_government_ta-en_trained_15k.csv b/Government/15k/nllb_bible_eval_government_ta-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..964733562fb560bca2ebb789074cf1bacc5bd056 --- /dev/null +++ b/Government/15k/nllb_bible_eval_government_ta-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,38.93958526618946,37.9423666899648,15.899713952027419,17.724165475640607,17.08788519688558,0.7514738410115241 diff --git a/Government/15k/nllb_floresplus_eval_government_en-si_trained_15k.csv b/Government/15k/nllb_floresplus_eval_government_en-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..5cb10dc5357d21c56ca976ce6beb18a801215422 --- /dev/null +++ b/Government/15k/nllb_floresplus_eval_government_en-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,8.110053421354792,6.406640947709287,0.04680080036208984,0.13415601552398707,0.14781206574773043,0.3853278315880082 diff --git a/Government/15k/nllb_floresplus_eval_government_en-ta_trained_15k.csv b/Government/15k/nllb_floresplus_eval_government_en-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..001a3e46b1b80047ce698896675b7c56ba94f807 --- /dev/null +++ b/Government/15k/nllb_floresplus_eval_government_en-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,51.603214168549805,45.46102073550337,10.678829148020034,24.980858657267614,24.55441496505266,0.8630498663886733 diff --git a/Government/15k/nllb_floresplus_eval_government_si-en_trained_15k.csv b/Government/15k/nllb_floresplus_eval_government_si-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..415796fe39c5ddb1ee28d9e35b152e336585de3a --- /dev/null +++ b/Government/15k/nllb_floresplus_eval_government_si-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,56.66471171297911,54.55280182410841,28.147612954372494,29.996577761750906,30.809015410443397,0.8639266135782121 diff --git a/Government/15k/nllb_floresplus_eval_government_si-ta_trained_15k.csv b/Government/15k/nllb_floresplus_eval_government_si-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..5e83bc9b953947e0373772ec83067fb2708f8709 --- /dev/null +++ b/Government/15k/nllb_floresplus_eval_government_si-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,46.62147952798226,40.48548461848681,7.200193895231319,19.35224094154367,19.205403285057585,0.8485663407463802 diff --git a/Government/15k/nllb_floresplus_eval_government_ta-en_trained_15k.csv b/Government/15k/nllb_floresplus_eval_government_ta-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..c1273dbd9eb8660d99cfb6a9b8d1eefe23f17d92 --- /dev/null +++ b/Government/15k/nllb_floresplus_eval_government_ta-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,55.68135527981136,53.55910697731873,27.455037344067588,29.32706115143437,30.20855458604444,0.8470760650432156 diff --git a/Government/15k/nllb_floresplus_eval_government_ta-si_trained_15k.csv b/Government/15k/nllb_floresplus_eval_government_ta-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ed2f3ec2c73c749cd9c05c99fff014b7204e9fdc --- /dev/null +++ b/Government/15k/nllb_floresplus_eval_government_ta-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,38.45302325056041,34.450685923333786,7.005155614085869,20.396857388683156,23.95739109565358,0.8433839754595351 diff --git a/Government/15k/nllb_government_eval_bible_en-si_trained_15k.csv b/Government/15k/nllb_government_eval_bible_en-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..4ddc3ee11549212cbc6b550bd47f4a6fe8e35e17 --- /dev/null +++ b/Government/15k/nllb_government_eval_bible_en-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,10.333521780820627,8.453661202177699,0.06656251784264193,1.5401320162594445,1.3016910398397168,0.45090242648124695 diff --git a/Government/15k/nllb_government_eval_bible_en-ta_trained_15k.csv b/Government/15k/nllb_government_eval_bible_en-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..878ec19d947fef19fd1272a63a854aa2176b31e8 --- /dev/null +++ b/Government/15k/nllb_government_eval_bible_en-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,39.35146130939026,33.3893609747557,2.6919493432304913,14.209230774547496,12.92935046957307,0.798079612314701 diff --git a/Government/15k/nllb_government_eval_bible_si-en_trained_15k.csv b/Government/15k/nllb_government_eval_bible_si-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..6e7676db1c389f0573b31b276e41ffeb58b52894 --- /dev/null +++ b/Government/15k/nllb_government_eval_bible_si-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,45.43490212145097,44.461716905870546,21.836615165017935,23.113368684205426,22.79028610179947,0.7760936993658543 diff --git a/Government/15k/nllb_government_eval_bible_ta-en_trained_15k.csv b/Government/15k/nllb_government_eval_bible_ta-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..964733562fb560bca2ebb789074cf1bacc5bd056 --- /dev/null +++ b/Government/15k/nllb_government_eval_bible_ta-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,38.93958526618946,37.9423666899648,15.899713952027419,17.724165475640607,17.08788519688558,0.7514738410115241 diff --git a/Government/15k/nllb_government_eval_en-si_trained_15k.csv b/Government/15k/nllb_government_eval_en-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..86525db58327984536246320ade7adf6a30e48e6 --- /dev/null +++ b/Government/15k/nllb_government_eval_en-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,9.654245870815185,7.730708090493862,0.1164836727252356,0.38989761608785434,0.5182228231390497,0.3989743923192021 diff --git a/Government/15k/nllb_government_eval_en-ta_trained_15k.csv b/Government/15k/nllb_government_eval_en-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..19b0ed18079a4860f0b05749240dad73894d4559 --- /dev/null +++ b/Government/15k/nllb_government_eval_en-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,59.525815651870836,53.39892235035869,19.99448127892838,35.422931165772006,35.58181399915633,0.8965862399181525 diff --git a/Government/15k/nllb_government_eval_government_en-si_trained_15k.csv b/Government/15k/nllb_government_eval_government_en-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..fd551ed0dd24aa5f9d5367b759a1ab4dadb53018 --- /dev/null +++ b/Government/15k/nllb_government_eval_government_en-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,9.731813599282967,7.8013772560248285,0.13145813863544065,0.36572279592324386,0.4982705083406589,0.3988617490530014 diff --git a/Government/15k/nllb_government_eval_government_en-ta_trained_15k.csv b/Government/15k/nllb_government_eval_government_en-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..7863ec53fbcb61bf6be0ea0647e76405ebd34ee0 --- /dev/null +++ b/Government/15k/nllb_government_eval_government_en-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,60.327104755281844,54.11043139111106,20.734959128906038,36.29463437793894,36.41417193218862,0.9002067874670029 diff --git a/Government/15k/nllb_government_eval_government_si-en_trained_15k.csv b/Government/15k/nllb_government_eval_government_si-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..596173fe4864c1d30665121f2a29d38719503635 --- /dev/null +++ b/Government/15k/nllb_government_eval_government_si-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,63.66743657611551,60.907282231187885,35.963847478597884,38.062048194354155,39.14898865638359,0.8755780427753925 diff --git a/Government/15k/nllb_government_eval_government_si-ta_trained_15k.csv b/Government/15k/nllb_government_eval_government_si-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b2abe06a0d48fc792be447f53c3961ddf5c04425 --- /dev/null +++ b/Government/15k/nllb_government_eval_government_si-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,58.51502500815897,52.10340591896176,18.71465430045571,33.94814988996791,34.16824189172424,0.892300804167986 diff --git a/Government/15k/nllb_government_eval_government_ta-en_trained_15k.csv b/Government/15k/nllb_government_eval_government_ta-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..a7dba1b9ec13d2b41b0e5fd2105e70e57d4b95cd --- /dev/null +++ b/Government/15k/nllb_government_eval_government_ta-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,63.04912140181778,60.405523019129504,35.798060148238854,37.48408588690264,38.46184102871189,0.8601772767901421 diff --git a/Government/15k/nllb_government_eval_government_ta-si_trained_15k.csv b/Government/15k/nllb_government_eval_government_ta-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..9e79a8f0e5dc4d7527df7972c064173bb3172aa7 --- /dev/null +++ b/Government/15k/nllb_government_eval_government_ta-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,53.96876394544066,49.68875362882517,19.57129556993449,39.16798179399488,45.52770375688494,0.9014797831177711 diff --git a/Government/15k/nllb_government_eval_si-en_trained_15k.csv b/Government/15k/nllb_government_eval_si-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..1cdea83032d40ecb84c9fa7257744ab3ea0b419d --- /dev/null +++ b/Government/15k/nllb_government_eval_si-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,63.59478404532841,60.876178601448295,35.90397524489571,37.801994611746665,38.84636636350326,0.8742459429637224 diff --git a/Government/15k/nllb_government_eval_si-ta_trained_15k.csv b/Government/15k/nllb_government_eval_si-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..c729301458742a36e8b7d08bea452918ce1aa60c --- /dev/null +++ b/Government/15k/nllb_government_eval_si-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,58.16275068335038,51.9160372817746,18.795765249653105,33.76508640251838,34.03779648875346,0.8901587198575064 diff --git a/Government/15k/nllb_government_eval_ta-en_trained_15k.csv b/Government/15k/nllb_government_eval_ta-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b695d2abd509b9eea3e6567b61f506db64e0275c --- /dev/null +++ b/Government/15k/nllb_government_eval_ta-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,62.7418753591663,60.135397094241824,35.62022929635181,37.04730164160621,37.98939045653956,0.8567265743308853 diff --git a/Government/15k/nllb_government_eval_ta-si_trained_15k.csv b/Government/15k/nllb_government_eval_ta-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..aeb4bf4946687d89730e36ff31ae5a1c48e812a3 --- /dev/null +++ b/Government/15k/nllb_government_eval_ta-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,53.53591484550607,49.37108999065305,19.673913656825604,38.9293396378409,45.1283309556417,0.9004071638203752 diff --git a/Government/15k/nllb_health_eval_government_en-si_trained_15k.csv b/Government/15k/nllb_health_eval_government_en-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..2e28d1f7e9565ef747a771ffd73de64ede77bf9b --- /dev/null +++ b/Government/15k/nllb_health_eval_government_en-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,9.423835346437757,7.569232714787168,0.016680110408727196,0.10450402224309237,0.10668848027028774,0.3727596676751469 diff --git a/Government/15k/nllb_health_eval_government_en-ta_trained_15k.csv b/Government/15k/nllb_health_eval_government_en-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..80b3e291c6a8f159677d6ad5ac8e29d9fc4fa860 --- /dev/null +++ b/Government/15k/nllb_health_eval_government_en-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,58.100641671237,51.51679561098018,13.729242014213668,30.92716962659006,29.867780694387307,0.8373897092342377 diff --git a/Government/15k/nllb_health_eval_government_si-en_trained_15k.csv b/Government/15k/nllb_health_eval_government_si-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..e5de26503e56b384e03f3bb97a7a5275052f47bd --- /dev/null +++ b/Government/15k/nllb_health_eval_government_si-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,57.449860202434664,54.331743280233425,23.193473569032133,28.956658292795414,29.306935414676797,0.8493524404054278 diff --git a/Government/15k/nllb_health_eval_government_si-ta_trained_15k.csv b/Government/15k/nllb_health_eval_government_si-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..c0d3f504eaeb112a9af0701494ade6bd84299206 --- /dev/null +++ b/Government/15k/nllb_health_eval_government_si-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,50.641585133776026,43.91872731729796,7.56702385722543,21.66294710805543,20.83484070879369,0.8097094015731495 diff --git a/Government/15k/nllb_health_eval_government_ta-en_trained_15k.csv b/Government/15k/nllb_health_eval_government_ta-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..379a4b0d33cc2c3fa80a25b53ce5840ee74ef622 --- /dev/null +++ b/Government/15k/nllb_health_eval_government_ta-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,59.067049911385126,56.3283334193523,28.487133101658152,31.97818434390073,32.75584316497059,0.8321624001860619 diff --git a/Government/15k/nllb_health_eval_government_ta-si_trained_15k.csv b/Government/15k/nllb_health_eval_government_ta-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..3b490997636bb1a44a6ed599bc891f7160a905c0 --- /dev/null +++ b/Government/15k/nllb_health_eval_government_ta-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,36.94074923703173,33.023939801820724,7.766407385787166,19.567224746683117,22.092532643274968,0.7912866747652998 diff --git a/Government/15k/nllb_jailbreak_eval_government_en-si_trained_15k.csv b/Government/15k/nllb_jailbreak_eval_government_en-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..fac317790a8879ba56a7a3d07cc81242fa3b0a64 --- /dev/null +++ b/Government/15k/nllb_jailbreak_eval_government_en-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,5.374077312754306,4.392153280587568,0.06092086280113295,0.08853089245215734,0.11605574790245451,0.43360042932629583 diff --git a/Government/15k/nllb_jailbreak_eval_government_en-ta_trained_15k.csv b/Government/15k/nllb_jailbreak_eval_government_en-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..f4bcb4cbb1a462f34198b41b12b89393a778c619 --- /dev/null +++ b/Government/15k/nllb_jailbreak_eval_government_en-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,57.19502017847633,52.106862659615615,20.146883816765218,32.03345035713335,31.151251845438047,0.8521424018740654 diff --git a/Government/15k/nllb_jailbreak_eval_government_si-en_trained_15k.csv b/Government/15k/nllb_jailbreak_eval_government_si-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..7f1ae20a0c7c5947e0630a043dbe4da7e8d88501 --- /dev/null +++ b/Government/15k/nllb_jailbreak_eval_government_si-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,54.27109503581603,52.4599949261298,29.295788397067184,29.68282584631609,30.324293926971475,0.8559750660508871 diff --git a/Government/15k/nllb_jailbreak_eval_government_si-ta_trained_15k.csv b/Government/15k/nllb_jailbreak_eval_government_si-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..f26d47cc36e47631b632595ff4bacb9075757f3b --- /dev/null +++ b/Government/15k/nllb_jailbreak_eval_government_si-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,51.00509178274327,45.801115742436075,13.660645461567995,23.874330078442313,23.090086318772954,0.8354445858299733 diff --git a/Government/15k/nllb_jailbreak_eval_government_ta-en_trained_15k.csv b/Government/15k/nllb_jailbreak_eval_government_ta-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..d5fc09cb73feb5edec62dc9e434da1af2ae873e7 --- /dev/null +++ b/Government/15k/nllb_jailbreak_eval_government_ta-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,55.045920882965426,53.797912454365495,32.82592941773117,30.46379047561909,31.30141057598678,0.8432262063622474 diff --git a/Government/15k/nllb_jailbreak_eval_government_ta-si_trained_15k.csv b/Government/15k/nllb_jailbreak_eval_government_ta-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..1c633b7f5c76b768dc3b9b254a9612f14b63dfde --- /dev/null +++ b/Government/15k/nllb_jailbreak_eval_government_ta-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,37.70430986761613,34.67567508096997,11.514139456675753,19.386135763634236,21.834840330552364,0.8098987563550473 diff --git a/Government/15k/nllb_literature-children_eval_government_en-si_trained_15k.csv b/Government/15k/nllb_literature-children_eval_government_en-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..697d62681745f19313035316c3b00a53d2779e13 --- /dev/null +++ b/Government/15k/nllb_literature-children_eval_government_en-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,5.80325024079758,4.518580409004698,0.026286496780860515,0.02709034848087214,0.04351758619159029,0.43771635806560516 diff --git a/Government/15k/nllb_literature-children_eval_government_en-ta_trained_15k.csv b/Government/15k/nllb_literature-children_eval_government_en-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..f49b4970d5de02340e9c884e33519b15d40c44df --- /dev/null +++ b/Government/15k/nllb_literature-children_eval_government_en-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,53.330662605200885,47.977488579298054,13.743036092488099,27.0984453930422,26.13129626962484,0.8352127524989622 diff --git a/Government/15k/nllb_literature-children_eval_government_si-en_trained_15k.csv b/Government/15k/nllb_literature-children_eval_government_si-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..407521e17c15fc2c944e0109fad155dc9f319df4 --- /dev/null +++ b/Government/15k/nllb_literature-children_eval_government_si-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,43.36910379797703,42.289734486998555,19.171156780144635,18.83634896985832,19.03263358047044,0.8026190398931503 diff --git a/Government/15k/nllb_literature-children_eval_government_si-ta_trained_15k.csv b/Government/15k/nllb_literature-children_eval_government_si-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..03c7e0e1d370129b5ec3bbfad7ec81339a84d831 --- /dev/null +++ b/Government/15k/nllb_literature-children_eval_government_si-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,44.42504437023244,39.027253448053614,6.704736604280777,16.12462739360897,15.357443866101152,0.80511938022936 diff --git a/Government/15k/nllb_literature-children_eval_government_ta-en_trained_15k.csv b/Government/15k/nllb_literature-children_eval_government_ta-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ab397a4e50a3e8872649eb06d0d89f9a010e7782 --- /dev/null +++ b/Government/15k/nllb_literature-children_eval_government_ta-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,50.190185691728914,49.424318295275064,26.457919601300983,25.555387915175853,25.992960120207965,0.8157399566844106 diff --git a/Government/15k/nllb_literature-children_eval_government_ta-si_trained_15k.csv b/Government/15k/nllb_literature-children_eval_government_ta-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..39904dc72e1eb92598c6059c2edb40f9d8a4cdf9 --- /dev/null +++ b/Government/15k/nllb_literature-children_eval_government_ta-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,30.847155564452667,27.666299764563313,4.727389743288037,11.183768052909961,12.876095051785802,0.799267442432982 diff --git a/Government/15k/nllb_literature-kathlyn_eval_government_en-si_trained_15k.csv b/Government/15k/nllb_literature-kathlyn_eval_government_en-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..697d62681745f19313035316c3b00a53d2779e13 --- /dev/null +++ b/Government/15k/nllb_literature-kathlyn_eval_government_en-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,5.80325024079758,4.518580409004698,0.026286496780860515,0.02709034848087214,0.04351758619159029,0.43771635806560516 diff --git a/Government/15k/nllb_literature-kathlyn_eval_government_en-ta_trained_15k.csv b/Government/15k/nllb_literature-kathlyn_eval_government_en-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..f49b4970d5de02340e9c884e33519b15d40c44df --- /dev/null +++ b/Government/15k/nllb_literature-kathlyn_eval_government_en-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,53.330662605200885,47.977488579298054,13.743036092488099,27.0984453930422,26.13129626962484,0.8352127524989622 diff --git a/Government/15k/nllb_literature-kathlyn_eval_government_si-en_trained_15k.csv b/Government/15k/nllb_literature-kathlyn_eval_government_si-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..407521e17c15fc2c944e0109fad155dc9f319df4 --- /dev/null +++ b/Government/15k/nllb_literature-kathlyn_eval_government_si-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,43.36910379797703,42.289734486998555,19.171156780144635,18.83634896985832,19.03263358047044,0.8026190398931503 diff --git a/Government/15k/nllb_literature-kathlyn_eval_government_si-ta_trained_15k.csv b/Government/15k/nllb_literature-kathlyn_eval_government_si-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..03c7e0e1d370129b5ec3bbfad7ec81339a84d831 --- /dev/null +++ b/Government/15k/nllb_literature-kathlyn_eval_government_si-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,44.42504437023244,39.027253448053614,6.704736604280777,16.12462739360897,15.357443866101152,0.80511938022936 diff --git a/Government/15k/nllb_literature-kathlyn_eval_government_ta-en_trained_15k.csv b/Government/15k/nllb_literature-kathlyn_eval_government_ta-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ab397a4e50a3e8872649eb06d0d89f9a010e7782 --- /dev/null +++ b/Government/15k/nllb_literature-kathlyn_eval_government_ta-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,50.190185691728914,49.424318295275064,26.457919601300983,25.555387915175853,25.992960120207965,0.8157399566844106 diff --git a/Government/15k/nllb_literature-kathlyn_eval_government_ta-si_trained_15k.csv b/Government/15k/nllb_literature-kathlyn_eval_government_ta-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..39904dc72e1eb92598c6059c2edb40f9d8a4cdf9 --- /dev/null +++ b/Government/15k/nllb_literature-kathlyn_eval_government_ta-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,30.847155564452667,27.666299764563313,4.727389743288037,11.183768052909961,12.876095051785802,0.799267442432982 diff --git a/Government/15k/nllb_literature-old_eval_government_en-si_trained_15k.csv b/Government/15k/nllb_literature-old_eval_government_en-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..697d62681745f19313035316c3b00a53d2779e13 --- /dev/null +++ b/Government/15k/nllb_literature-old_eval_government_en-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,5.80325024079758,4.518580409004698,0.026286496780860515,0.02709034848087214,0.04351758619159029,0.43771635806560516 diff --git a/Government/15k/nllb_literature-old_eval_government_en-ta_trained_15k.csv b/Government/15k/nllb_literature-old_eval_government_en-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..f49b4970d5de02340e9c884e33519b15d40c44df --- /dev/null +++ b/Government/15k/nllb_literature-old_eval_government_en-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,53.330662605200885,47.977488579298054,13.743036092488099,27.0984453930422,26.13129626962484,0.8352127524989622 diff --git a/Government/15k/nllb_literature-old_eval_government_si-en_trained_15k.csv b/Government/15k/nllb_literature-old_eval_government_si-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..407521e17c15fc2c944e0109fad155dc9f319df4 --- /dev/null +++ b/Government/15k/nllb_literature-old_eval_government_si-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,43.36910379797703,42.289734486998555,19.171156780144635,18.83634896985832,19.03263358047044,0.8026190398931503 diff --git a/Government/15k/nllb_literature-old_eval_government_si-ta_trained_15k.csv b/Government/15k/nllb_literature-old_eval_government_si-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..03c7e0e1d370129b5ec3bbfad7ec81339a84d831 --- /dev/null +++ b/Government/15k/nllb_literature-old_eval_government_si-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,44.42504437023244,39.027253448053614,6.704736604280777,16.12462739360897,15.357443866101152,0.80511938022936 diff --git a/Government/15k/nllb_literature-old_eval_government_ta-en_trained_15k.csv b/Government/15k/nllb_literature-old_eval_government_ta-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ab397a4e50a3e8872649eb06d0d89f9a010e7782 --- /dev/null +++ b/Government/15k/nllb_literature-old_eval_government_ta-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,50.190185691728914,49.424318295275064,26.457919601300983,25.555387915175853,25.992960120207965,0.8157399566844106 diff --git a/Government/15k/nllb_literature-old_eval_government_ta-si_trained_15k.csv b/Government/15k/nllb_literature-old_eval_government_ta-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..39904dc72e1eb92598c6059c2edb40f9d8a4cdf9 --- /dev/null +++ b/Government/15k/nllb_literature-old_eval_government_ta-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,30.847155564452667,27.666299764563313,4.727389743288037,11.183768052909961,12.876095051785802,0.799267442432982 diff --git a/Government/15k/nllb_literature_eval_government_en-si_trained_15k.csv b/Government/15k/nllb_literature_eval_government_en-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..697d62681745f19313035316c3b00a53d2779e13 --- /dev/null +++ b/Government/15k/nllb_literature_eval_government_en-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,5.80325024079758,4.518580409004698,0.026286496780860515,0.02709034848087214,0.04351758619159029,0.43771635806560516 diff --git a/Government/15k/nllb_literature_eval_government_en-ta_trained_15k.csv b/Government/15k/nllb_literature_eval_government_en-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..f49b4970d5de02340e9c884e33519b15d40c44df --- /dev/null +++ b/Government/15k/nllb_literature_eval_government_en-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,53.330662605200885,47.977488579298054,13.743036092488099,27.0984453930422,26.13129626962484,0.8352127524989622 diff --git a/Government/15k/nllb_literature_eval_government_si-en_trained_15k.csv b/Government/15k/nllb_literature_eval_government_si-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..407521e17c15fc2c944e0109fad155dc9f319df4 --- /dev/null +++ b/Government/15k/nllb_literature_eval_government_si-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,43.36910379797703,42.289734486998555,19.171156780144635,18.83634896985832,19.03263358047044,0.8026190398931503 diff --git a/Government/15k/nllb_literature_eval_government_si-ta_trained_15k.csv b/Government/15k/nllb_literature_eval_government_si-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..03c7e0e1d370129b5ec3bbfad7ec81339a84d831 --- /dev/null +++ b/Government/15k/nllb_literature_eval_government_si-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,44.42504437023244,39.027253448053614,6.704736604280777,16.12462739360897,15.357443866101152,0.80511938022936 diff --git a/Government/15k/nllb_literature_eval_government_ta-en_trained_15k.csv b/Government/15k/nllb_literature_eval_government_ta-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ab397a4e50a3e8872649eb06d0d89f9a010e7782 --- /dev/null +++ b/Government/15k/nllb_literature_eval_government_ta-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,50.190185691728914,49.424318295275064,26.457919601300983,25.555387915175853,25.992960120207965,0.8157399566844106 diff --git a/Government/15k/nllb_literature_eval_government_ta-si_trained_15k.csv b/Government/15k/nllb_literature_eval_government_ta-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..39904dc72e1eb92598c6059c2edb40f9d8a4cdf9 --- /dev/null +++ b/Government/15k/nllb_literature_eval_government_ta-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,30.847155564452667,27.666299764563313,4.727389743288037,11.183768052909961,12.876095051785802,0.799267442432982 diff --git a/Government/15k/nllb_maths_eval_government_en-si_trained_15k.csv b/Government/15k/nllb_maths_eval_government_en-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..438d9e3aa66da8e3acb846d00b089ac48c8e6ca6 --- /dev/null +++ b/Government/15k/nllb_maths_eval_government_en-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,6.837063431902609,5.473992424430646,0.016974339404795976,0.02406231000542957,0.016119390622961006,0.4090483323037624 diff --git a/Government/15k/nllb_maths_eval_government_en-ta_trained_15k.csv b/Government/15k/nllb_maths_eval_government_en-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b3df444267d68243d6b23b2d180c4baaa2f21584 --- /dev/null +++ b/Government/15k/nllb_maths_eval_government_en-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,50.41074236607987,46.39616223416936,19.92487952308717,26.499396868002357,25.886182013385874,0.8344578335285187 diff --git a/Government/15k/nllb_maths_eval_government_si-en_trained_15k.csv b/Government/15k/nllb_maths_eval_government_si-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..c0024c1d819dab872159adc2ebf6a2c66e8eaf6b --- /dev/null +++ b/Government/15k/nllb_maths_eval_government_si-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,57.310069958060105,56.76185730321125,38.43488474377299,38.07576167493571,38.69220960171209,0.8414851614534855 diff --git a/Government/15k/nllb_maths_eval_government_ta-en_trained_15k.csv b/Government/15k/nllb_maths_eval_government_ta-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..6aa29c5f70b1d0ab325e1ae4c1f97c1fcac629d6 --- /dev/null +++ b/Government/15k/nllb_maths_eval_government_ta-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,53.84658011107133,53.045838570945094,33.67318492445719,33.17528266750687,33.94058941032087,0.8335989883840084 diff --git a/Government/15k/nllb_news_eval_government_en-si_trained_15k.csv b/Government/15k/nllb_news_eval_government_en-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..4e8cb5963cbfb63fd01fe03d501a19764bf8ccf6 --- /dev/null +++ b/Government/15k/nllb_news_eval_government_en-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,7.475587226092283,5.872985418064176,0.013261650304781982,0.09391607214255804,0.1147211509808602,0.396491152331233 diff --git a/Government/15k/nllb_news_eval_government_en-ta_trained_15k.csv b/Government/15k/nllb_news_eval_government_en-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..4e6c96d260e0515fa46bd653653b6b31f615e506 --- /dev/null +++ b/Government/15k/nllb_news_eval_government_en-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,58.18240480822938,52.583530136995606,16.883673473235667,33.380973106643324,33.202182667126806,0.8685271259711449 diff --git a/Government/15k/nllb_news_eval_government_si-en_trained_15k.csv b/Government/15k/nllb_news_eval_government_si-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..8249a3e1d080a2a6056febc98421bc552d73dac9 --- /dev/null +++ b/Government/15k/nllb_news_eval_government_si-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,54.79661168252055,52.118872979463774,23.385029031710246,25.404756406612243,26.521867470756305,0.8521652072072029 diff --git a/Government/15k/nllb_news_eval_government_si-ta_trained_15k.csv b/Government/15k/nllb_news_eval_government_si-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..0a8d918bb458525f1eb3aa784fb56b3613d55361 --- /dev/null +++ b/Government/15k/nllb_news_eval_government_si-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,50.44887726763213,44.51688307506956,9.174720208715458,22.10982496645205,22.061340830915096,0.849138046314399 diff --git a/Government/15k/nllb_news_eval_government_ta-en_trained_15k.csv b/Government/15k/nllb_news_eval_government_ta-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..e356c59a5ab4e311e8094b6435ee787c1abdc939 --- /dev/null +++ b/Government/15k/nllb_news_eval_government_ta-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,59.30171072542917,57.48852277318919,32.08427537674978,33.049149792465755,34.08395652798429,0.8557109833661333 diff --git a/Government/15k/nllb_news_eval_government_ta-si_trained_15k.csv b/Government/15k/nllb_news_eval_government_ta-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..af74dfa5c4007a668f8c5e1b71b57324837c6c39 --- /dev/null +++ b/Government/15k/nllb_news_eval_government_ta-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,37.39422991381425,33.25335478871849,6.7076017167597035,19.666608785867624,23.247194308801653,0.8248296706306357 diff --git a/Government/15k/nllb_nllb_eval_government_en-si_trained_15k.csv b/Government/15k/nllb_nllb_eval_government_en-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..8fd015ec1a3d1d78904e28a48d799f0aaeea409d --- /dev/null +++ b/Government/15k/nllb_nllb_eval_government_en-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,6.324312311847597,4.99984908358139,0.06385138956503969,0.2349258100220535,0.23784025867144348,0.45816841441812484 diff --git a/Government/15k/nllb_nllb_eval_government_en-ta_trained_15k.csv b/Government/15k/nllb_nllb_eval_government_en-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ab1ca3286745a41703504650c5f0d1e7f4f6d76d --- /dev/null +++ b/Government/15k/nllb_nllb_eval_government_en-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,55.95602263572816,50.96604446578231,16.97995308261609,29.940204717375405,29.06211337216374,0.9060601958381017 diff --git a/Government/15k/nllb_nllb_eval_government_si-en_trained_15k.csv b/Government/15k/nllb_nllb_eval_government_si-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..fe3270b8f9f5916e365ff19a6819eea382808113 --- /dev/null +++ b/Government/15k/nllb_nllb_eval_government_si-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,62.13359523376011,61.36838945744591,41.138863850499035,40.83376787968107,41.02992900037648,0.8981701793280233 diff --git a/Government/15k/nllb_nllb_eval_government_si-ta_trained_15k.csv b/Government/15k/nllb_nllb_eval_government_si-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..8b06aa98e230202be8368d55d3a999987ddb26bb --- /dev/null +++ b/Government/15k/nllb_nllb_eval_government_si-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,53.51653697424845,48.41602536871634,14.103769750493683,26.635338195111228,25.807689005505196,0.8983319042863336 diff --git a/Government/15k/nllb_nllb_eval_government_ta-en_trained_15k.csv b/Government/15k/nllb_nllb_eval_government_ta-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..17edc7f45d83bbca0f29736b9bebc028129acaf5 --- /dev/null +++ b/Government/15k/nllb_nllb_eval_government_ta-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,62.122531248298095,61.423738860976584,41.175402204866195,40.927687728202656,41.18346186472626,0.8877022689095867 diff --git a/Government/15k/nllb_nllb_eval_government_ta-si_trained_15k.csv b/Government/15k/nllb_nllb_eval_government_ta-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..43583ad11e18e8ff3e96124ab6bf8d2d5ad62be8 --- /dev/null +++ b/Government/15k/nllb_nllb_eval_government_ta-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,40.78065473084989,37.457620865710304,10.342019107119588,21.709779587946734,25.558254074279883,0.8867994582004328 diff --git a/Government/15k/nllb_opensubtitles_eval_government_en-si_trained_15k.csv b/Government/15k/nllb_opensubtitles_eval_government_en-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..397a38947135ac092d4326df7eafbfea311947fc --- /dev/null +++ b/Government/15k/nllb_opensubtitles_eval_government_en-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,6.577743041669929,5.300362655175006,0.10032284851116866,0.35165695021323184,0.3133280497660109,0.4609458845481276 diff --git a/Government/15k/nllb_opensubtitles_eval_government_en-ta_trained_15k.csv b/Government/15k/nllb_opensubtitles_eval_government_en-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..8cec39c0143305501e9d1e629976f286664fd638 --- /dev/null +++ b/Government/15k/nllb_opensubtitles_eval_government_en-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,51.053005804755614,47.20893438933282,17.80236231837347,29.0229461422151,28.7448018857421,0.8637542776614427 diff --git a/Government/15k/nllb_opensubtitles_eval_government_si-en_trained_15k.csv b/Government/15k/nllb_opensubtitles_eval_government_si-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..dbd55bec7263e7bb210b6df04c1eae1d7185356d --- /dev/null +++ b/Government/15k/nllb_opensubtitles_eval_government_si-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,62.309569067666494,61.51447385810135,39.85246405730881,40.53007397285569,40.78251697321025,0.8903046472668648 diff --git a/Government/15k/nllb_opensubtitles_eval_government_si-ta_trained_15k.csv b/Government/15k/nllb_opensubtitles_eval_government_si-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..a05a2dd72cede1a60a354f7c501f6d81b6fa0e14 --- /dev/null +++ b/Government/15k/nllb_opensubtitles_eval_government_si-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,54.127066085395946,49.69329220502086,17.16696700590091,29.26894214159221,28.652638825484704,0.8884524757713079 diff --git a/Government/15k/nllb_opensubtitles_eval_government_ta-en_trained_15k.csv b/Government/15k/nllb_opensubtitles_eval_government_ta-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..39965485dcc9d0b5f88313a108f5a79beba8841c --- /dev/null +++ b/Government/15k/nllb_opensubtitles_eval_government_ta-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,58.971334689650476,58.25553140519777,35.8433465718555,36.49129606679034,36.78553642188729,0.868731607362628 diff --git a/Government/15k/nllb_opensubtitles_eval_government_ta-si_trained_15k.csv b/Government/15k/nllb_opensubtitles_eval_government_ta-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..85e27113580ac60abd853253ae4f36b4d287a110 --- /dev/null +++ b/Government/15k/nllb_opensubtitles_eval_government_ta-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,32.52286357531791,29.487865443464617,6.248544947911525,17.09394300575521,18.70591304753473,0.8551399844735861 diff --git a/Government/15k/nllb_wikipedia_eval_government_en-si_trained_15k.csv b/Government/15k/nllb_wikipedia_eval_government_en-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..947010a260c56e47bcf0864d15d5e55c1bbb2be0 --- /dev/null +++ b/Government/15k/nllb_wikipedia_eval_government_en-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,9.591036118587715,7.478943564953437,0.014733951862163606,0.09345199612962554,0.08773712446315937,0.374054478764534 diff --git a/Government/15k/nllb_wikipedia_eval_government_en-ta_trained_15k.csv b/Government/15k/nllb_wikipedia_eval_government_en-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..2d75bc453643e4bf906cb6da907a384fa7330fad --- /dev/null +++ b/Government/15k/nllb_wikipedia_eval_government_en-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,54.147304763835265,47.77457877302795,11.134996124969733,27.911489519257888,27.664231157728317,0.8771339770406484 diff --git a/Government/15k/nllb_wikipedia_eval_government_si-en_trained_15k.csv b/Government/15k/nllb_wikipedia_eval_government_si-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..6db25fef165bf893937d06cd77c8aeae5ac8b033 --- /dev/null +++ b/Government/15k/nllb_wikipedia_eval_government_si-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,54.596528187164026,52.52762694790201,25.839339896954755,29.114118958651478,29.585157945833636,0.8438672468662262 diff --git a/Government/15k/nllb_wikipedia_eval_government_si-ta_trained_15k.csv b/Government/15k/nllb_wikipedia_eval_government_si-ta_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..75809cd7ce58366a4b8259dcbb0d9224e9b01db7 --- /dev/null +++ b/Government/15k/nllb_wikipedia_eval_government_si-ta_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,47.63386011934315,41.36829753464779,6.992413270637968,21.42097035492968,21.228218568403623,0.8501114219129086 diff --git a/Government/15k/nllb_wikipedia_eval_government_ta-en_trained_15k.csv b/Government/15k/nllb_wikipedia_eval_government_ta-en_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b60bdfe0792e5b1e4a40d5047ce613ef43e38c95 --- /dev/null +++ b/Government/15k/nllb_wikipedia_eval_government_ta-en_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,58.659834653131284,56.6802895135369,30.142395620650245,33.11914588724891,33.70708416313059,0.8469915004372597 diff --git a/Government/15k/nllb_wikipedia_eval_government_ta-si_trained_15k.csv b/Government/15k/nllb_wikipedia_eval_government_ta-si_trained_15k.csv new file mode 100644 index 0000000000000000000000000000000000000000..dfa001ae325ff58f77776535f6ceb29c5b2b0d3f --- /dev/null +++ b/Government/15k/nllb_wikipedia_eval_government_ta-si_trained_15k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,38.446678519102036,34.258587708481436,6.186473063416697,19.75736103493816,23.017708074232512,0.8492549396157265 diff --git a/Government/15k/si-en/runs/Oct02_23-00-45_NLPC-Sita-MLM/events.out.tfevents.1759426245.NLPC-Sita-MLM.2581313.8 b/Government/15k/si-en/runs/Oct02_23-00-45_NLPC-Sita-MLM/events.out.tfevents.1759426245.NLPC-Sita-MLM.2581313.8 new file mode 100644 index 0000000000000000000000000000000000000000..d52add21bacf9af3abc22c95bd5c5ee062e1195c --- /dev/null +++ b/Government/15k/si-en/runs/Oct02_23-00-45_NLPC-Sita-MLM/events.out.tfevents.1759426245.NLPC-Sita-MLM.2581313.8 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29a2d22ac199cd7ae02ff887717f8d64cb08759a200e8594bdd1f2951591623a +size 7072 diff --git a/Government/15k/si-en_final_model/config.json b/Government/15k/si-en_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/15k/si-en_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/15k/si-en_final_model/generation_config.json b/Government/15k/si-en_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/15k/si-en_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/15k/si-en_final_model/pytorch_model.bin b/Government/15k/si-en_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..35e1b2aebe30b59cdc6145e93f5e5e15e80d3e9e --- /dev/null +++ b/Government/15k/si-en_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdbcfca47a86fc269eda27f08113a91b85f483442548e66ce56fb5749a65d23e +size 2468870723 diff --git a/Government/15k/si-en_final_model/training_args.bin b/Government/15k/si-en_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..4eadc1571b6c0f24687967327286073ed18a2304 --- /dev/null +++ b/Government/15k/si-en_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f38d1035c5082ab873ef12a4989e7974691f28d47980e7be0678e3d5c48c9806 +size 4143 diff --git a/Government/15k/si-ta/runs/Oct03_05-49-10_NLPC-Sita-MLM/events.out.tfevents.1759450750.NLPC-Sita-MLM.2581313.20 b/Government/15k/si-ta/runs/Oct03_05-49-10_NLPC-Sita-MLM/events.out.tfevents.1759450750.NLPC-Sita-MLM.2581313.20 new file mode 100644 index 0000000000000000000000000000000000000000..15637ca2b065ff548c98813602387372644453d1 --- /dev/null +++ b/Government/15k/si-ta/runs/Oct03_05-49-10_NLPC-Sita-MLM/events.out.tfevents.1759450750.NLPC-Sita-MLM.2581313.20 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afcfb5666b019faac066dc88921123da38b3f524bc8084101bb1bff114512ff0 +size 7072 diff --git a/Government/15k/si-ta_final_model/config.json b/Government/15k/si-ta_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/15k/si-ta_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/15k/si-ta_final_model/generation_config.json b/Government/15k/si-ta_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/15k/si-ta_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/15k/si-ta_final_model/pytorch_model.bin b/Government/15k/si-ta_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..c181011173525ad51d099d9a028b73135fd07b50 --- /dev/null +++ b/Government/15k/si-ta_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a4a59b2d7cd35252f180d72b348332ad492428e98ac47056af8646a1ba8474f +size 2468870723 diff --git a/Government/15k/si-ta_final_model/training_args.bin b/Government/15k/si-ta_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..435a9265f0c934ab578458915a0133877013493d --- /dev/null +++ b/Government/15k/si-ta_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a2293b9085e34627af7ea46cca8cfa74b0407ac035435036f1685295b4d6ec6 +size 4143 diff --git a/Government/15k/ta-en/runs/Oct03_01-22-13_NLPC-Sita-MLM/events.out.tfevents.1759434733.NLPC-Sita-MLM.2581313.14 b/Government/15k/ta-en/runs/Oct03_01-22-13_NLPC-Sita-MLM/events.out.tfevents.1759434733.NLPC-Sita-MLM.2581313.14 new file mode 100644 index 0000000000000000000000000000000000000000..e132fc47d560051f91b6c752823ad4d6983e74c9 --- /dev/null +++ b/Government/15k/ta-en/runs/Oct03_01-22-13_NLPC-Sita-MLM/events.out.tfevents.1759434733.NLPC-Sita-MLM.2581313.14 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7dccddcd3091a80f79836af0140db40ec929ffc2fa97d8540ab0188120f62faa +size 7072 diff --git a/Government/15k/ta-en_final_model/config.json b/Government/15k/ta-en_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/15k/ta-en_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/15k/ta-en_final_model/generation_config.json b/Government/15k/ta-en_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/15k/ta-en_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/15k/ta-en_final_model/pytorch_model.bin b/Government/15k/ta-en_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..89050cd84c37686253ea4d691c58ca4d41b9d2a9 --- /dev/null +++ b/Government/15k/ta-en_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbad21f9cd5cc827c51befb768cfb5532294ff9a1dfa6012e81900432ee68c81 +size 2468870723 diff --git a/Government/15k/ta-en_final_model/training_args.bin b/Government/15k/ta-en_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..70e565d8b3804705eae46d6222dc99ccca242347 --- /dev/null +++ b/Government/15k/ta-en_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f9680bcd18dc264dc15fef4f02dfa5e2edc68738c7967164c05be7fb5f2daa2 +size 4143 diff --git a/Government/15k/ta-si/runs/Oct03_08-20-00_NLPC-Sita-MLM/events.out.tfevents.1759459800.NLPC-Sita-MLM.2581313.26 b/Government/15k/ta-si/runs/Oct03_08-20-00_NLPC-Sita-MLM/events.out.tfevents.1759459800.NLPC-Sita-MLM.2581313.26 new file mode 100644 index 0000000000000000000000000000000000000000..6961c749a7178f864b03f4d544fd92c6da3cf69e --- /dev/null +++ b/Government/15k/ta-si/runs/Oct03_08-20-00_NLPC-Sita-MLM/events.out.tfevents.1759459800.NLPC-Sita-MLM.2581313.26 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:348d7dee0467b5578cc13426a81322b25e1b3e3613b0b8c636db0f99b50d96e8 +size 7072 diff --git a/Government/15k/ta-si_final_model/config.json b/Government/15k/ta-si_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/15k/ta-si_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/15k/ta-si_final_model/generation_config.json b/Government/15k/ta-si_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/15k/ta-si_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/15k/ta-si_final_model/pytorch_model.bin b/Government/15k/ta-si_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..4aa11e6a5a56c00832802249ce02ac589a276e93 --- /dev/null +++ b/Government/15k/ta-si_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6755ea3b1d209a88f27d843947eac4229557196fe086ba58b15b20ea2d54f69 +size 2468870723 diff --git a/Government/15k/ta-si_final_model/training_args.bin b/Government/15k/ta-si_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..08331452e0a61942acb3014a78ee65482af8c4df --- /dev/null +++ b/Government/15k/ta-si_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b07eebd1d2c88ff16d942e3eb3c2d5920c58379fc52c4ce097db9f1c5c336317 +size 4143 diff --git a/Government/15k/tok-en-si/sentencepiece.bpe.model b/Government/15k/tok-en-si/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/15k/tok-en-si/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/15k/tok-en-si/special_tokens_map.json b/Government/15k/tok-en-si/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/15k/tok-en-si/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/15k/tok-en-si/tokenizer_config.json b/Government/15k/tok-en-si/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fd1139979a7c7c4188b26d922af637f4bfb23e1f --- /dev/null +++ b/Government/15k/tok-en-si/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "eng_Latn", + "tgt_lang": "sin_Sinh", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/15k/tok-en-ta/sentencepiece.bpe.model b/Government/15k/tok-en-ta/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/15k/tok-en-ta/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/15k/tok-en-ta/special_tokens_map.json b/Government/15k/tok-en-ta/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/15k/tok-en-ta/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/15k/tok-en-ta/tokenizer_config.json b/Government/15k/tok-en-ta/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ef8f263366a29cf3dd0c97df4b45c41fbbf160b5 --- /dev/null +++ b/Government/15k/tok-en-ta/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "eng_Latn", + "tgt_lang": "tam_Taml", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/15k/tok-si-en/sentencepiece.bpe.model b/Government/15k/tok-si-en/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/15k/tok-si-en/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/15k/tok-si-en/special_tokens_map.json b/Government/15k/tok-si-en/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/15k/tok-si-en/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/15k/tok-si-en/tokenizer_config.json b/Government/15k/tok-si-en/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f80b41ef163bc0a47ebef66aa6c1210dfe0b9792 --- /dev/null +++ b/Government/15k/tok-si-en/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "sin_Sinh", + "tgt_lang": "eng_Latn", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/15k/tok-si-ta/sentencepiece.bpe.model b/Government/15k/tok-si-ta/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/15k/tok-si-ta/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/15k/tok-si-ta/special_tokens_map.json b/Government/15k/tok-si-ta/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/15k/tok-si-ta/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/15k/tok-si-ta/tokenizer_config.json b/Government/15k/tok-si-ta/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..386158eef85de789e966861e91bebf69e9a4f32b --- /dev/null +++ b/Government/15k/tok-si-ta/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "sin_Sinh", + "tgt_lang": "tam_Taml", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/15k/tok-ta-en/sentencepiece.bpe.model b/Government/15k/tok-ta-en/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/15k/tok-ta-en/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/15k/tok-ta-en/special_tokens_map.json b/Government/15k/tok-ta-en/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/15k/tok-ta-en/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/15k/tok-ta-en/tokenizer_config.json b/Government/15k/tok-ta-en/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6deaa799386cb7438bf8a0f96250f961c73dec30 --- /dev/null +++ b/Government/15k/tok-ta-en/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "tam_Taml", + "tgt_lang": "eng_Latn", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/15k/tok-ta-si/sentencepiece.bpe.model b/Government/15k/tok-ta-si/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/15k/tok-ta-si/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/15k/tok-ta-si/special_tokens_map.json b/Government/15k/tok-ta-si/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/15k/tok-ta-si/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/15k/tok-ta-si/tokenizer_config.json b/Government/15k/tok-ta-si/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9526d1cacd39f5ee55eece61fdacbbc6ce50136d --- /dev/null +++ b/Government/15k/tok-ta-si/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "tam_Taml", + "tgt_lang": "sin_Sinh", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/1k/en-si_final_model/config.json b/Government/1k/en-si_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/1k/en-si_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/1k/en-si_final_model/generation_config.json b/Government/1k/en-si_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/1k/en-si_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/1k/en-si_final_model/pytorch_model.bin b/Government/1k/en-si_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..ba4db0386b224f0ab1567eb55e45adc5e7802ea9 --- /dev/null +++ b/Government/1k/en-si_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e501a4672dd4a4f6b4055da7c68ba09247f21322d00c56b66ff86403a2e5e86 +size 2468870723 diff --git a/Government/1k/en-si_final_model/training_args.bin b/Government/1k/en-si_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..751bc43fdaabfd629095b05e4ba0aaee76a1496d --- /dev/null +++ b/Government/1k/en-si_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0778975da280582519d04276eb5365070abf50727244815ddaea6d2a75f63664 +size 4143 diff --git a/Government/1k/en-ta/runs/Oct02_21-34-40_NLPC-Sita-MLM/events.out.tfevents.1759421080.NLPC-Sita-MLM.2581313.5 b/Government/1k/en-ta/runs/Oct02_21-34-40_NLPC-Sita-MLM/events.out.tfevents.1759421080.NLPC-Sita-MLM.2581313.5 new file mode 100644 index 0000000000000000000000000000000000000000..9a58e5ea3b75b8fb1bfe0a9df15258a1a6b105b4 --- /dev/null +++ b/Government/1k/en-ta/runs/Oct02_21-34-40_NLPC-Sita-MLM/events.out.tfevents.1759421080.NLPC-Sita-MLM.2581313.5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea81e0aca12dbf705650795ddce078de9cf663834eed6d86a0fe1e491f9e209c +size 5651 diff --git a/Government/1k/en-ta_final_model/config.json b/Government/1k/en-ta_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/1k/en-ta_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/1k/en-ta_final_model/generation_config.json b/Government/1k/en-ta_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/1k/en-ta_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/1k/en-ta_final_model/pytorch_model.bin b/Government/1k/en-ta_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..37d021797988b8d6bc2fcc1245e3ada55245c61d --- /dev/null +++ b/Government/1k/en-ta_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:326673dfd09db05cfd44b7391ff3bf2fe7fecbae6e52de4ace3d90f6b25acfe6 +size 2468870723 diff --git a/Government/1k/en-ta_final_model/training_args.bin b/Government/1k/en-ta_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..b7f5a13ec41607b3237388a84d0eadf8ca840e09 --- /dev/null +++ b/Government/1k/en-ta_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94d49830b08a094dad46ec3e94b924cb228687d34e1b905b906a9bb0c4d8a125 +size 4143 diff --git a/Government/1k/nllb_bible_eval_government_en-si_trained_1k.csv b/Government/1k/nllb_bible_eval_government_en-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..e21567b7d21db540b278592413242aaceffa2b6a --- /dev/null +++ b/Government/1k/nllb_bible_eval_government_en-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,23.516304960748695,20.362706332865574,1.6177223328136954,7.698804887228647,7.142460377516814,0.6451798568218946 diff --git a/Government/1k/nllb_bible_eval_government_en-ta_trained_1k.csv b/Government/1k/nllb_bible_eval_government_en-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..9529911dbf9a0152d190ad0257d0920f3bd5eee2 --- /dev/null +++ b/Government/1k/nllb_bible_eval_government_en-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,34.9595922976334,30.016010590575693,3.6955114039946846,11.425554236738714,10.234295696241674,0.7152103856056928 diff --git a/Government/1k/nllb_bible_eval_government_si-en_trained_1k.csv b/Government/1k/nllb_bible_eval_government_si-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..fb04a01228d4813532fd191e718c26b92f26416f --- /dev/null +++ b/Government/1k/nllb_bible_eval_government_si-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,45.519679436405895,44.56427531232765,21.64082810028898,23.125564774458322,22.756168684278343,0.7835294347405434 diff --git a/Government/1k/nllb_bible_eval_government_ta-en_trained_1k.csv b/Government/1k/nllb_bible_eval_government_ta-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..36886ce5dab8bf5d9201487f5f2b7e4a71f54c12 --- /dev/null +++ b/Government/1k/nllb_bible_eval_government_ta-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,39.75161387490352,38.70065033403796,16.618038541811128,18.634568567182622,18.002394763247366,0.756501064568758 diff --git a/Government/1k/nllb_floresplus_eval_government_en-si_trained_1k.csv b/Government/1k/nllb_floresplus_eval_government_en-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..0f77be5993eaa5bc72d1835d3b71edb40c18471e --- /dev/null +++ b/Government/1k/nllb_floresplus_eval_government_en-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,7.215223944683759,5.744608434304767,0.04893639724609202,0.1648504019946428,0.2229497306164265,0.320197635400201 diff --git a/Government/1k/nllb_floresplus_eval_government_en-ta_trained_1k.csv b/Government/1k/nllb_floresplus_eval_government_en-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..c4dfe2d04656c034ebd547bdcb203818c41d502e --- /dev/null +++ b/Government/1k/nllb_floresplus_eval_government_en-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,53.865656846497664,47.80073113672015,12.633814796264634,28.59163422478019,28.399945483860275,0.8676608577665132 diff --git a/Government/1k/nllb_floresplus_eval_government_si-en_trained_1k.csv b/Government/1k/nllb_floresplus_eval_government_si-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..a6156f532a891c6f9182f4d8f3ba788b58f0a7c9 --- /dev/null +++ b/Government/1k/nllb_floresplus_eval_government_si-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,58.104840438076465,56.060526082433626,30.543987665322973,32.04481816309639,32.80023559608612,0.8685261862786863 diff --git a/Government/1k/nllb_floresplus_eval_government_si-ta_trained_1k.csv b/Government/1k/nllb_floresplus_eval_government_si-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..bb526f1d7c424897c43f6c3256aa30e32e77f84e --- /dev/null +++ b/Government/1k/nllb_floresplus_eval_government_si-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,49.377805890374155,43.35548016326561,9.269762772435334,23.17183827724759,23.25917858100641,0.855572857316068 diff --git a/Government/1k/nllb_floresplus_eval_government_ta-en_trained_1k.csv b/Government/1k/nllb_floresplus_eval_government_ta-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..5386392e74653909abafd43a5dd573bb8739bdc8 --- /dev/null +++ b/Government/1k/nllb_floresplus_eval_government_ta-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,56.0838536046274,53.98404543363633,28.325535590273322,29.500632062307492,30.417167497105098,0.8509534669546742 diff --git a/Government/1k/nllb_floresplus_eval_government_ta-si_trained_1k.csv b/Government/1k/nllb_floresplus_eval_government_ta-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..5df45b28c9f77212705e03bbf6410e9d99a5f5e2 --- /dev/null +++ b/Government/1k/nllb_floresplus_eval_government_ta-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,40.185809049862115,36.195736062588395,8.655722843861811,22.222520270610033,25.833272205353627,0.8500500880329034 diff --git a/Government/1k/nllb_government_eval_bible_en-si_trained_1k.csv b/Government/1k/nllb_government_eval_bible_en-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..e21567b7d21db540b278592413242aaceffa2b6a --- /dev/null +++ b/Government/1k/nllb_government_eval_bible_en-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,23.516304960748695,20.362706332865574,1.6177223328136954,7.698804887228647,7.142460377516814,0.6451798568218946 diff --git a/Government/1k/nllb_government_eval_bible_en-ta_trained_1k.csv b/Government/1k/nllb_government_eval_bible_en-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..9529911dbf9a0152d190ad0257d0920f3bd5eee2 --- /dev/null +++ b/Government/1k/nllb_government_eval_bible_en-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,34.9595922976334,30.016010590575693,3.6955114039946846,11.425554236738714,10.234295696241674,0.7152103856056928 diff --git a/Government/1k/nllb_government_eval_bible_si-en_trained_1k.csv b/Government/1k/nllb_government_eval_bible_si-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..fb04a01228d4813532fd191e718c26b92f26416f --- /dev/null +++ b/Government/1k/nllb_government_eval_bible_si-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,45.519679436405895,44.56427531232765,21.64082810028898,23.125564774458322,22.756168684278343,0.7835294347405434 diff --git a/Government/1k/nllb_government_eval_bible_ta-en_trained_1k.csv b/Government/1k/nllb_government_eval_bible_ta-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..36886ce5dab8bf5d9201487f5f2b7e4a71f54c12 --- /dev/null +++ b/Government/1k/nllb_government_eval_bible_ta-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,39.75161387490352,38.70065033403796,16.618038541811128,18.634568567182622,18.002394763247366,0.756501064568758 diff --git a/Government/1k/nllb_government_eval_en-si_trained_1k.csv b/Government/1k/nllb_government_eval_en-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..c489d37e6ec1afca514aa17cc3429b2faf5bce29 --- /dev/null +++ b/Government/1k/nllb_government_eval_en-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,7.749347640855546,6.205670520341053,0.04316388249769439,0.20198875839229394,0.34597429824431225,0.379277535039626 diff --git a/Government/1k/nllb_government_eval_en-ta_trained_1k.csv b/Government/1k/nllb_government_eval_en-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..68c9a2160010368ed3fde8718a0d759a4cdcd504 --- /dev/null +++ b/Government/1k/nllb_government_eval_en-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,50.97834001428786,45.04861754397153,13.416768162087196,23.580736173270395,23.62066277988367,0.8781776242240554 diff --git a/Government/1k/nllb_government_eval_government_en-si_trained_1k.csv b/Government/1k/nllb_government_eval_government_en-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..6bbc803c5a88ab56d30fce3483d63a1a85ddf916 --- /dev/null +++ b/Government/1k/nllb_government_eval_government_en-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,7.693363479451183,6.136089531064441,0.02034245584111586,0.1361625082452787,0.27086771946901417,0.37701798579096796 diff --git a/Government/1k/nllb_government_eval_government_en-ta_trained_1k.csv b/Government/1k/nllb_government_eval_government_en-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..adf99b8f6aa72e9f19bb59dc22aa29c78eb481e1 --- /dev/null +++ b/Government/1k/nllb_government_eval_government_en-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,51.49403723919621,45.43089824989166,13.736301489950655,24.093670097241016,24.023475896842484,0.8820261645913124 diff --git a/Government/1k/nllb_government_eval_government_si-en_trained_1k.csv b/Government/1k/nllb_government_eval_government_si-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..936d4698affaf0d4a8f3bdb16c3ea61218078aef --- /dev/null +++ b/Government/1k/nllb_government_eval_government_si-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,57.7801523647287,54.89723004749681,28.536492739972214,28.88016835917535,30.227912618064472,0.8545789907276631 diff --git a/Government/1k/nllb_government_eval_government_si-ta_trained_1k.csv b/Government/1k/nllb_government_eval_government_si-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..2552f682e083760e38cc461201ae9991a379ca80 --- /dev/null +++ b/Government/1k/nllb_government_eval_government_si-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,50.2443796188764,44.10493824421842,12.538969053246612,22.42627877247282,22.439774320757984,0.8743745173215867 diff --git a/Government/1k/nllb_government_eval_government_ta-en_trained_1k.csv b/Government/1k/nllb_government_eval_government_ta-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..465cbbf18ede6a2b743b6753d44445c62b740984 --- /dev/null +++ b/Government/1k/nllb_government_eval_government_ta-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,56.05655518747233,53.314253050443796,27.395680142968754,27.31640954628235,28.43727114616367,0.8340802385210991 diff --git a/Government/1k/nllb_government_eval_government_ta-si_trained_1k.csv b/Government/1k/nllb_government_eval_government_ta-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..c8d487eb1ccb6515a83b5cd06ddbfe38eba4fdb8 --- /dev/null +++ b/Government/1k/nllb_government_eval_government_ta-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,46.95468711618753,42.71863536497049,13.946403489944109,28.67080157940711,35.66790765231362,0.8795496428310872 diff --git a/Government/1k/nllb_government_eval_si-en_trained_1k.csv b/Government/1k/nllb_government_eval_si-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..19073fc787240f4f5568559e2b104506d0e08c5a --- /dev/null +++ b/Government/1k/nllb_government_eval_si-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,57.71505618062982,54.86743776584928,28.688330462901895,28.85449537574812,30.1395690202118,0.8544098007153184 diff --git a/Government/1k/nllb_government_eval_si-ta_trained_1k.csv b/Government/1k/nllb_government_eval_si-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..486e4d06b32b8b6c2339d3e152d1a36df4527735 --- /dev/null +++ b/Government/1k/nllb_government_eval_si-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,49.7089253160222,43.74127256066377,12.464929082045591,22.078689389223623,22.219746172544177,0.8703997820246063 diff --git a/Government/1k/nllb_government_eval_ta-en_trained_1k.csv b/Government/1k/nllb_government_eval_ta-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..9be3802c2ff0817f278731ed776f9682f2763138 --- /dev/null +++ b/Government/1k/nllb_government_eval_ta-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,55.705341814565315,53.0272543322555,27.34613133621202,27.04317106109931,28.20935095819385,0.8315010662482657 diff --git a/Government/1k/nllb_government_eval_ta-si_trained_1k.csv b/Government/1k/nllb_government_eval_ta-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..5863a662ef7144309a6a45668f44fdf34d408601 --- /dev/null +++ b/Government/1k/nllb_government_eval_ta-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,46.47785684436921,42.324758108330904,13.548474087018985,28.551075045101932,35.30825463454368,0.8767766253464309 diff --git a/Government/1k/nllb_health_eval_government_en-si_trained_1k.csv b/Government/1k/nllb_health_eval_government_en-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..7a85b69ab145970a2f714337799cd30942b73800 --- /dev/null +++ b/Government/1k/nllb_health_eval_government_en-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,7.943758973222321,6.379120539858632,0.027669500922065122,0.1918128396954742,0.21253259251099058,0.31514034632886395 diff --git a/Government/1k/nllb_health_eval_government_en-ta_trained_1k.csv b/Government/1k/nllb_health_eval_government_en-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..663e186689b8e1e9859388395e0e3d7a42e52544 --- /dev/null +++ b/Government/1k/nllb_health_eval_government_en-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,60.68331352514558,54.275432087342246,16.359838256117694,35.505708648653545,34.3043864590842,0.8473187069296837 diff --git a/Government/1k/nllb_health_eval_government_si-en_trained_1k.csv b/Government/1k/nllb_health_eval_government_si-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ae0d6664e56dacc822e4cbaba478ebadfea001c4 --- /dev/null +++ b/Government/1k/nllb_health_eval_government_si-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,58.823952962234195,55.76889243030728,24.96873919761974,31.08439877453402,31.533984745001874,0.85347719641214 diff --git a/Government/1k/nllb_health_eval_government_si-ta_trained_1k.csv b/Government/1k/nllb_health_eval_government_si-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..8fea41ac88bd27ddc57991534ef51476f57b46fd --- /dev/null +++ b/Government/1k/nllb_health_eval_government_si-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,53.78217793027862,47.105039169606194,9.66112278327546,25.42321036216343,24.746129997956235,0.8144897919300259 diff --git a/Government/1k/nllb_health_eval_government_ta-en_trained_1k.csv b/Government/1k/nllb_health_eval_government_ta-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..88241097ec8d9a0cfbf20a94e4c0df86c67f0ddb --- /dev/null +++ b/Government/1k/nllb_health_eval_government_ta-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,59.45104570223155,56.60695794787922,28.579661522093712,32.17800497697126,32.881421889606045,0.8324152611792087 diff --git a/Government/1k/nllb_health_eval_government_ta-si_trained_1k.csv b/Government/1k/nllb_health_eval_government_ta-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..8d69fb35c8f5095128c60f1b7326851861b0256e --- /dev/null +++ b/Government/1k/nllb_health_eval_government_ta-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,38.80244151460313,35.118936902723284,9.915471801860118,22.41068104302556,25.31076371035167,0.7955503560108493 diff --git a/Government/1k/nllb_jailbreak_eval_government_en-si_trained_1k.csv b/Government/1k/nllb_jailbreak_eval_government_en-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..4cbe30c05b0ad107fef28299846998a5d904a547 --- /dev/null +++ b/Government/1k/nllb_jailbreak_eval_government_en-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,9.44752151854744,8.10370681971603,0.5929600023807652,1.2132553605616745,1.272400283286709,0.45134091030061246 diff --git a/Government/1k/nllb_jailbreak_eval_government_en-ta_trained_1k.csv b/Government/1k/nllb_jailbreak_eval_government_en-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..13e8753bfca94893c34f4f5cb706c3004fdbba6b --- /dev/null +++ b/Government/1k/nllb_jailbreak_eval_government_en-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,58.10779627770626,53.1788446442861,21.707498859768968,35.19240880440027,34.731288781745924,0.8380884910523891 diff --git a/Government/1k/nllb_jailbreak_eval_government_si-en_trained_1k.csv b/Government/1k/nllb_jailbreak_eval_government_si-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..6fc8420795fc06b9bf8bf4410bada7777b12046b --- /dev/null +++ b/Government/1k/nllb_jailbreak_eval_government_si-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,54.9625739402548,53.21327625787566,30.63372234457116,30.852209509922588,31.487794552918498,0.8601958205401897 diff --git a/Government/1k/nllb_jailbreak_eval_government_si-ta_trained_1k.csv b/Government/1k/nllb_jailbreak_eval_government_si-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..73bc781cefc0823196b8eae015f456381ed3853d --- /dev/null +++ b/Government/1k/nllb_jailbreak_eval_government_si-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,53.71652084735038,48.50413378360178,16.66258277987578,27.250011859180134,26.361991957852965,0.8376016582548619 diff --git a/Government/1k/nllb_jailbreak_eval_government_ta-en_trained_1k.csv b/Government/1k/nllb_jailbreak_eval_government_ta-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..a85299e54c5aecdb4354ff4b0e4d1ee9fbd94ad3 --- /dev/null +++ b/Government/1k/nllb_jailbreak_eval_government_ta-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,55.75054509800249,54.428314269123,33.533878099037544,30.933041977866377,31.898920036378996,0.8466845436096191 diff --git a/Government/1k/nllb_jailbreak_eval_government_ta-si_trained_1k.csv b/Government/1k/nllb_jailbreak_eval_government_ta-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..55c6c92c492da4b6aa50d033079b67fb7b6d4d0d --- /dev/null +++ b/Government/1k/nllb_jailbreak_eval_government_ta-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,40.2176191045428,37.05086886311183,13.149641719115921,22.21999833121419,25.24383735605519,0.820767950668931 diff --git a/Government/1k/nllb_literature-children_eval_government_en-si_trained_1k.csv b/Government/1k/nllb_literature-children_eval_government_en-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ddf5e7dd622a4d49bf6edd45d7e6b8444d9444d9 --- /dev/null +++ b/Government/1k/nllb_literature-children_eval_government_en-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,10.20057431150489,8.942477418219665,0.835031192198746,1.6339000097540557,1.521072472920624,0.5377916059345007 diff --git a/Government/1k/nllb_literature-children_eval_government_en-ta_trained_1k.csv b/Government/1k/nllb_literature-children_eval_government_en-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..cbd93a5827025436114eafecb77376f12d3edd04 --- /dev/null +++ b/Government/1k/nllb_literature-children_eval_government_en-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,53.89895728855406,48.89007726591565,14.649069178790135,28.643382408052908,28.11198441735683,0.8140736782785367 diff --git a/Government/1k/nllb_literature-children_eval_government_si-en_trained_1k.csv b/Government/1k/nllb_literature-children_eval_government_si-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..9691d0f0ea8a42b2b557ff39ef0180fcfd5cb92f --- /dev/null +++ b/Government/1k/nllb_literature-children_eval_government_si-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,44.48527430359607,43.45372079058745,20.5791049450655,19.968255129646845,20.22726305440647,0.8111989430785179 diff --git a/Government/1k/nllb_literature-children_eval_government_si-ta_trained_1k.csv b/Government/1k/nllb_literature-children_eval_government_si-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b4a66cb88192831711a10ea12354999d7d95d163 --- /dev/null +++ b/Government/1k/nllb_literature-children_eval_government_si-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,45.13866684662315,40.07546567521024,8.198705487657577,18.005897082619562,17.594096148416693,0.8022552307798257 diff --git a/Government/1k/nllb_literature-children_eval_government_ta-en_trained_1k.csv b/Government/1k/nllb_literature-children_eval_government_ta-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..06c44d4fae29bd214871fd23c73b4c8e3e8326f2 --- /dev/null +++ b/Government/1k/nllb_literature-children_eval_government_ta-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,50.85534112089745,50.03031690407106,26.348126940987406,25.71520616359661,26.279905137766534,0.8200640871568963 diff --git a/Government/1k/nllb_literature-children_eval_government_ta-si_trained_1k.csv b/Government/1k/nllb_literature-children_eval_government_ta-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b6e72d5c53c804764a47fe431ff2473f001dfad1 --- /dev/null +++ b/Government/1k/nllb_literature-children_eval_government_ta-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,32.21343052357557,29.07701645656392,5.4597256404869,12.505948053229249,14.27471265084141,0.8002993582459851 diff --git a/Government/1k/nllb_literature-kathlyn_eval_government_en-si_trained_1k.csv b/Government/1k/nllb_literature-kathlyn_eval_government_en-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ddf5e7dd622a4d49bf6edd45d7e6b8444d9444d9 --- /dev/null +++ b/Government/1k/nllb_literature-kathlyn_eval_government_en-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,10.20057431150489,8.942477418219665,0.835031192198746,1.6339000097540557,1.521072472920624,0.5377916059345007 diff --git a/Government/1k/nllb_literature-kathlyn_eval_government_en-ta_trained_1k.csv b/Government/1k/nllb_literature-kathlyn_eval_government_en-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..cbd93a5827025436114eafecb77376f12d3edd04 --- /dev/null +++ b/Government/1k/nllb_literature-kathlyn_eval_government_en-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,53.89895728855406,48.89007726591565,14.649069178790135,28.643382408052908,28.11198441735683,0.8140736782785367 diff --git a/Government/1k/nllb_literature-kathlyn_eval_government_si-en_trained_1k.csv b/Government/1k/nllb_literature-kathlyn_eval_government_si-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..9691d0f0ea8a42b2b557ff39ef0180fcfd5cb92f --- /dev/null +++ b/Government/1k/nllb_literature-kathlyn_eval_government_si-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,44.48527430359607,43.45372079058745,20.5791049450655,19.968255129646845,20.22726305440647,0.8111989430785179 diff --git a/Government/1k/nllb_literature-kathlyn_eval_government_si-ta_trained_1k.csv b/Government/1k/nllb_literature-kathlyn_eval_government_si-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b4a66cb88192831711a10ea12354999d7d95d163 --- /dev/null +++ b/Government/1k/nllb_literature-kathlyn_eval_government_si-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,45.13866684662315,40.07546567521024,8.198705487657577,18.005897082619562,17.594096148416693,0.8022552307798257 diff --git a/Government/1k/nllb_literature-kathlyn_eval_government_ta-en_trained_1k.csv b/Government/1k/nllb_literature-kathlyn_eval_government_ta-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..06c44d4fae29bd214871fd23c73b4c8e3e8326f2 --- /dev/null +++ b/Government/1k/nllb_literature-kathlyn_eval_government_ta-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,50.85534112089745,50.03031690407106,26.348126940987406,25.71520616359661,26.279905137766534,0.8200640871568963 diff --git a/Government/1k/nllb_literature-kathlyn_eval_government_ta-si_trained_1k.csv b/Government/1k/nllb_literature-kathlyn_eval_government_ta-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b6e72d5c53c804764a47fe431ff2473f001dfad1 --- /dev/null +++ b/Government/1k/nllb_literature-kathlyn_eval_government_ta-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,32.21343052357557,29.07701645656392,5.4597256404869,12.505948053229249,14.27471265084141,0.8002993582459851 diff --git a/Government/1k/nllb_literature-old_eval_government_en-si_trained_1k.csv b/Government/1k/nllb_literature-old_eval_government_en-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ddf5e7dd622a4d49bf6edd45d7e6b8444d9444d9 --- /dev/null +++ b/Government/1k/nllb_literature-old_eval_government_en-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,10.20057431150489,8.942477418219665,0.835031192198746,1.6339000097540557,1.521072472920624,0.5377916059345007 diff --git a/Government/1k/nllb_literature-old_eval_government_en-ta_trained_1k.csv b/Government/1k/nllb_literature-old_eval_government_en-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..cbd93a5827025436114eafecb77376f12d3edd04 --- /dev/null +++ b/Government/1k/nllb_literature-old_eval_government_en-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,53.89895728855406,48.89007726591565,14.649069178790135,28.643382408052908,28.11198441735683,0.8140736782785367 diff --git a/Government/1k/nllb_literature-old_eval_government_si-en_trained_1k.csv b/Government/1k/nllb_literature-old_eval_government_si-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..9691d0f0ea8a42b2b557ff39ef0180fcfd5cb92f --- /dev/null +++ b/Government/1k/nllb_literature-old_eval_government_si-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,44.48527430359607,43.45372079058745,20.5791049450655,19.968255129646845,20.22726305440647,0.8111989430785179 diff --git a/Government/1k/nllb_literature-old_eval_government_si-ta_trained_1k.csv b/Government/1k/nllb_literature-old_eval_government_si-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b4a66cb88192831711a10ea12354999d7d95d163 --- /dev/null +++ b/Government/1k/nllb_literature-old_eval_government_si-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,45.13866684662315,40.07546567521024,8.198705487657577,18.005897082619562,17.594096148416693,0.8022552307798257 diff --git a/Government/1k/nllb_literature-old_eval_government_ta-en_trained_1k.csv b/Government/1k/nllb_literature-old_eval_government_ta-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..06c44d4fae29bd214871fd23c73b4c8e3e8326f2 --- /dev/null +++ b/Government/1k/nllb_literature-old_eval_government_ta-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,50.85534112089745,50.03031690407106,26.348126940987406,25.71520616359661,26.279905137766534,0.8200640871568963 diff --git a/Government/1k/nllb_literature-old_eval_government_ta-si_trained_1k.csv b/Government/1k/nllb_literature-old_eval_government_ta-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b6e72d5c53c804764a47fe431ff2473f001dfad1 --- /dev/null +++ b/Government/1k/nllb_literature-old_eval_government_ta-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,32.21343052357557,29.07701645656392,5.4597256404869,12.505948053229249,14.27471265084141,0.8002993582459851 diff --git a/Government/1k/nllb_literature_eval_government_en-si_trained_1k.csv b/Government/1k/nllb_literature_eval_government_en-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ddf5e7dd622a4d49bf6edd45d7e6b8444d9444d9 --- /dev/null +++ b/Government/1k/nllb_literature_eval_government_en-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,10.20057431150489,8.942477418219665,0.835031192198746,1.6339000097540557,1.521072472920624,0.5377916059345007 diff --git a/Government/1k/nllb_literature_eval_government_en-ta_trained_1k.csv b/Government/1k/nllb_literature_eval_government_en-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..cbd93a5827025436114eafecb77376f12d3edd04 --- /dev/null +++ b/Government/1k/nllb_literature_eval_government_en-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,53.89895728855406,48.89007726591565,14.649069178790135,28.643382408052908,28.11198441735683,0.8140736782785367 diff --git a/Government/1k/nllb_literature_eval_government_si-en_trained_1k.csv b/Government/1k/nllb_literature_eval_government_si-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..9691d0f0ea8a42b2b557ff39ef0180fcfd5cb92f --- /dev/null +++ b/Government/1k/nllb_literature_eval_government_si-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,44.48527430359607,43.45372079058745,20.5791049450655,19.968255129646845,20.22726305440647,0.8111989430785179 diff --git a/Government/1k/nllb_literature_eval_government_si-ta_trained_1k.csv b/Government/1k/nllb_literature_eval_government_si-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b4a66cb88192831711a10ea12354999d7d95d163 --- /dev/null +++ b/Government/1k/nllb_literature_eval_government_si-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,45.13866684662315,40.07546567521024,8.198705487657577,18.005897082619562,17.594096148416693,0.8022552307798257 diff --git a/Government/1k/nllb_literature_eval_government_ta-en_trained_1k.csv b/Government/1k/nllb_literature_eval_government_ta-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..06c44d4fae29bd214871fd23c73b4c8e3e8326f2 --- /dev/null +++ b/Government/1k/nllb_literature_eval_government_ta-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,50.85534112089745,50.03031690407106,26.348126940987406,25.71520616359661,26.279905137766534,0.8200640871568963 diff --git a/Government/1k/nllb_literature_eval_government_ta-si_trained_1k.csv b/Government/1k/nllb_literature_eval_government_ta-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b6e72d5c53c804764a47fe431ff2473f001dfad1 --- /dev/null +++ b/Government/1k/nllb_literature_eval_government_ta-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,32.21343052357557,29.07701645656392,5.4597256404869,12.505948053229249,14.27471265084141,0.8002993582459851 diff --git a/Government/1k/nllb_maths_eval_government_en-si_trained_1k.csv b/Government/1k/nllb_maths_eval_government_en-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..fbfe97054eed882d8c2e0aa9e8f274f6eee7efd9 --- /dev/null +++ b/Government/1k/nllb_maths_eval_government_en-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,7.135553179408386,5.990287751321683,0.23875409984545115,0.5156325949052365,0.4899032462176094,0.3933246617615223 diff --git a/Government/1k/nllb_maths_eval_government_en-ta_trained_1k.csv b/Government/1k/nllb_maths_eval_government_en-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..136976d9c6ff55dbac87dc79b1e6a53af8d330a9 --- /dev/null +++ b/Government/1k/nllb_maths_eval_government_en-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,52.00734110365251,48.036037740621786,21.279910114693163,28.573803977208566,28.00619078972203,0.8331779134869576 diff --git a/Government/1k/nllb_maths_eval_government_si-en_trained_1k.csv b/Government/1k/nllb_maths_eval_government_si-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b114162f5b472b287fc7cec34e972b9cfb66e491 --- /dev/null +++ b/Government/1k/nllb_maths_eval_government_si-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,58.57429503758378,57.988276525550745,39.17008812369143,39.317515212385764,39.553216712025744,0.8474531008601188 diff --git a/Government/1k/nllb_maths_eval_government_ta-en_trained_1k.csv b/Government/1k/nllb_maths_eval_government_ta-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..376d583269cd367b0d4cc9075e0ca1cf074d9f5c --- /dev/null +++ b/Government/1k/nllb_maths_eval_government_ta-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,54.64784196754577,53.89762744806972,34.78873357474227,34.48074447431171,35.23792224710771,0.8342977927327156 diff --git a/Government/1k/nllb_news_eval_government_en-si_trained_1k.csv b/Government/1k/nllb_news_eval_government_en-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..9f1045ef147709d7c6bbb04e502ac7dff00b134b --- /dev/null +++ b/Government/1k/nllb_news_eval_government_en-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,9.318291648325081,7.602404261012937,0.20717287393002193,0.7506842504471134,0.9476541691597596,0.4086479433476925 diff --git a/Government/1k/nllb_news_eval_government_en-ta_trained_1k.csv b/Government/1k/nllb_news_eval_government_en-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..dc5ba248dc2a526724b9fef0a330a6b2c7f81f8a --- /dev/null +++ b/Government/1k/nllb_news_eval_government_en-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,61.18633229487469,55.994558658194705,21.79529355470753,39.77194115398926,39.28750945693206,0.852673879768475 diff --git a/Government/1k/nllb_news_eval_government_si-en_trained_1k.csv b/Government/1k/nllb_news_eval_government_si-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..f28e9a1b58c6d283b9a9700eb5f4b1f5299156ca --- /dev/null +++ b/Government/1k/nllb_news_eval_government_si-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,54.5944648728102,51.89513492854887,23.597437634350108,25.583373745829206,26.626821242478208,0.8546181761026382 diff --git a/Government/1k/nllb_news_eval_government_si-ta_trained_1k.csv b/Government/1k/nllb_news_eval_government_si-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..fe1eda12b44c2a3a5a2d85f136671c9f4b1cf38b --- /dev/null +++ b/Government/1k/nllb_news_eval_government_si-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,53.2617833571283,47.44848569038954,11.42307905567309,26.463722638642164,26.46904198918585,0.859532229016505 diff --git a/Government/1k/nllb_news_eval_government_ta-en_trained_1k.csv b/Government/1k/nllb_news_eval_government_ta-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..98788bc579aa33c7dd4bf21f424b25f58cb3ad0d --- /dev/null +++ b/Government/1k/nllb_news_eval_government_ta-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,60.693336114832796,58.92145564017714,34.55620099741434,34.9155807710764,35.97092486853162,0.8604976444179753 diff --git a/Government/1k/nllb_news_eval_government_ta-si_trained_1k.csv b/Government/1k/nllb_news_eval_government_ta-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..7bf2e69d9b38a346edc7fac1b9ff057f84507e27 --- /dev/null +++ b/Government/1k/nllb_news_eval_government_ta-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,38.72222656917083,34.64803824339464,7.588713776478428,21.290457189445423,24.660552995976907,0.8313557970968466 diff --git a/Government/1k/nllb_nllb_eval_government_en-si_trained_1k.csv b/Government/1k/nllb_nllb_eval_government_en-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..d0744250975ef605c46bf6d63d38dd04ac43019a --- /dev/null +++ b/Government/1k/nllb_nllb_eval_government_en-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,12.324043069958455,10.868875229575847,0.8902522416379338,1.8066512013725395,1.8088916748363502,0.5654888506523086 diff --git a/Government/1k/nllb_nllb_eval_government_en-ta_trained_1k.csv b/Government/1k/nllb_nllb_eval_government_en-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..72234384092f63af0b20fecb1863c903772ed8a0 --- /dev/null +++ b/Government/1k/nllb_nllb_eval_government_en-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,52.48340779674099,48.03652080074238,17.20047536975647,29.440567616607588,28.722471108002285,0.8617566767791719 diff --git a/Government/1k/nllb_nllb_eval_government_si-en_trained_1k.csv b/Government/1k/nllb_nllb_eval_government_si-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..00e425c4f366c5bccd26f1f293d0a1e6580dda23 --- /dev/null +++ b/Government/1k/nllb_nllb_eval_government_si-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,63.41284019658625,62.61060550849287,42.91050967755352,42.32787525245691,42.58545376539262,0.9012708021769569 diff --git a/Government/1k/nllb_nllb_eval_government_si-ta_trained_1k.csv b/Government/1k/nllb_nllb_eval_government_si-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..57ffaf7437349108b18e68ad5a287eb924a56007 --- /dev/null +++ b/Government/1k/nllb_nllb_eval_government_si-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,53.82644072837088,48.96399499026332,15.64400772655998,28.21661867520769,27.593187279545745,0.8876809190942 diff --git a/Government/1k/nllb_nllb_eval_government_ta-en_trained_1k.csv b/Government/1k/nllb_nllb_eval_government_ta-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..bbcb5d406895a3d3ba830344fba02233a0165370 --- /dev/null +++ b/Government/1k/nllb_nllb_eval_government_ta-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,63.30388362788709,62.561316987392054,43.1055615067307,42.21331735764612,42.484750301588114,0.8913898070157318 diff --git a/Government/1k/nllb_nllb_eval_government_ta-si_trained_1k.csv b/Government/1k/nllb_nllb_eval_government_ta-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..63be9548ada4e07e9c708299e15bb2c9a550b651 --- /dev/null +++ b/Government/1k/nllb_nllb_eval_government_ta-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,43.44096000550498,40.045780326680244,12.345930720666471,23.557734781020194,27.371812657261128,0.8940775383131777 diff --git a/Government/1k/nllb_opensubtitles_eval_government_en-si_trained_1k.csv b/Government/1k/nllb_opensubtitles_eval_government_en-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..a473e8778846bcc8d1621c6463117ef99de4843c --- /dev/null +++ b/Government/1k/nllb_opensubtitles_eval_government_en-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,32.830743778102054,30.83796375018137,8.46478022121088,15.040401102261876,14.104804439363617,0.7946870295628905 diff --git a/Government/1k/nllb_opensubtitles_eval_government_en-ta_trained_1k.csv b/Government/1k/nllb_opensubtitles_eval_government_en-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..bc21de8d97d669635b9b9225493545b14a1b6ccd --- /dev/null +++ b/Government/1k/nllb_opensubtitles_eval_government_en-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,23.543217403368963,22.582984314652695,8.67274438400228,14.710985334183531,14.895692831927843,0.6994686460494995 diff --git a/Government/1k/nllb_opensubtitles_eval_government_si-en_trained_1k.csv b/Government/1k/nllb_opensubtitles_eval_government_si-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..65f8170e2d5bc3638d6b38ece423509843a3cb25 --- /dev/null +++ b/Government/1k/nllb_opensubtitles_eval_government_si-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,64.47308072896288,63.74329046003354,43.67043920882924,43.98564205774316,44.25832162381529,0.8969421654343604 diff --git a/Government/1k/nllb_opensubtitles_eval_government_si-ta_trained_1k.csv b/Government/1k/nllb_opensubtitles_eval_government_si-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ef9fe8433a09f05a6a826ebbdf2753717bed59c2 --- /dev/null +++ b/Government/1k/nllb_opensubtitles_eval_government_si-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,36.323185021540496,33.21552307918757,11.559699054015395,20.3981987484158,20.47530392566763,0.7550140042155981 diff --git a/Government/1k/nllb_opensubtitles_eval_government_ta-en_trained_1k.csv b/Government/1k/nllb_opensubtitles_eval_government_ta-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..f02d01bdd5c6fb5e67b42a7e6a79246765e73831 --- /dev/null +++ b/Government/1k/nllb_opensubtitles_eval_government_ta-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,60.51562192009797,59.771888856269854,38.24495304092191,38.30404615626526,38.57233271486981,0.8747753919959068 diff --git a/Government/1k/nllb_opensubtitles_eval_government_ta-si_trained_1k.csv b/Government/1k/nllb_opensubtitles_eval_government_ta-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..6f85960ff29e34efd291de7019529e3449898306 --- /dev/null +++ b/Government/1k/nllb_opensubtitles_eval_government_ta-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,41.19187592870378,38.419747113569834,13.543599122251214,23.986651824064726,26.149743096018014,0.8803802770078182 diff --git a/Government/1k/nllb_wikipedia_eval_government_en-si_trained_1k.csv b/Government/1k/nllb_wikipedia_eval_government_en-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..e2712599b90be3a59f89c67b56984424c9a49175 --- /dev/null +++ b/Government/1k/nllb_wikipedia_eval_government_en-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,8.11906180769877,6.549857745501184,0.12370604148899343,0.43368041597737306,0.5140921731137955,0.3466547068357468 diff --git a/Government/1k/nllb_wikipedia_eval_government_en-ta_trained_1k.csv b/Government/1k/nllb_wikipedia_eval_government_en-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..f39685e6fee8beb31c8200b346159cfcd3e55911 --- /dev/null +++ b/Government/1k/nllb_wikipedia_eval_government_en-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,56.255478335036344,49.83306218734586,13.02195300864623,30.373405621589413,30.195715254109636,0.8780179094821214 diff --git a/Government/1k/nllb_wikipedia_eval_government_si-en_trained_1k.csv b/Government/1k/nllb_wikipedia_eval_government_si-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..f37c08b55ee322f53d1b4ba4bfa420b6667ec284 --- /dev/null +++ b/Government/1k/nllb_wikipedia_eval_government_si-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,55.132224891054925,53.208937399375245,27.568990711729228,29.601467860214452,30.229356099978883,0.8444438032507896 diff --git a/Government/1k/nllb_wikipedia_eval_government_si-ta_trained_1k.csv b/Government/1k/nllb_wikipedia_eval_government_si-ta_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..9c6cc4844cced463f107f9c275fef0233f57a15c --- /dev/null +++ b/Government/1k/nllb_wikipedia_eval_government_si-ta_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,48.56815297697707,42.42238610194449,7.914428956367575,22.110250492175208,21.96760787336064,0.8537849247455597 diff --git a/Government/1k/nllb_wikipedia_eval_government_ta-en_trained_1k.csv b/Government/1k/nllb_wikipedia_eval_government_ta-en_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..c0750142672a383939c7b5efa82a3f2b9ac62411 --- /dev/null +++ b/Government/1k/nllb_wikipedia_eval_government_ta-en_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,59.45726298732943,57.526204765255976,32.25733837627613,33.865652765902965,34.71708040021201,0.8478907163143158 diff --git a/Government/1k/nllb_wikipedia_eval_government_ta-si_trained_1k.csv b/Government/1k/nllb_wikipedia_eval_government_ta-si_trained_1k.csv new file mode 100644 index 0000000000000000000000000000000000000000..280bc1fc43e412eaeca60841351a11346c4b0a17 --- /dev/null +++ b/Government/1k/nllb_wikipedia_eval_government_ta-si_trained_1k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,39.239179268723184,35.08154507375792,7.089033852800834,20.430535571924906,23.64374864390406,0.8513488554060459 diff --git a/Government/1k/si-en/runs/Oct02_23-56-20_NLPC-Sita-MLM/events.out.tfevents.1759429580.NLPC-Sita-MLM.2581313.11 b/Government/1k/si-en/runs/Oct02_23-56-20_NLPC-Sita-MLM/events.out.tfevents.1759429580.NLPC-Sita-MLM.2581313.11 new file mode 100644 index 0000000000000000000000000000000000000000..bd7f85fdffa71b0781018b1ab0d19f9a70ca4a8b --- /dev/null +++ b/Government/1k/si-en/runs/Oct02_23-56-20_NLPC-Sita-MLM/events.out.tfevents.1759429580.NLPC-Sita-MLM.2581313.11 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4fa33e5c7155dc3d4e67c0cc13f2c2d77d0801b7d9590427c7500a019271cb1 +size 5651 diff --git a/Government/1k/si-en_final_model/config.json b/Government/1k/si-en_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/1k/si-en_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/1k/si-en_final_model/generation_config.json b/Government/1k/si-en_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/1k/si-en_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/1k/si-en_final_model/pytorch_model.bin b/Government/1k/si-en_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..cb27e079fc30c46a277cbe7f1958d9851871702d --- /dev/null +++ b/Government/1k/si-en_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d42cdff7d9cf3aa89cd95d407f4b24354e9a7c4204ab6a9c07cb0aadd8dbc7c +size 2468870723 diff --git a/Government/1k/si-en_final_model/training_args.bin b/Government/1k/si-en_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..799f9529c9fe807c6f628057bec92821397a93ee --- /dev/null +++ b/Government/1k/si-en_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:229e572adcd75768492919873b14c9a7c46e6062e97e55e37e83221b39f5e6cb +size 4143 diff --git a/Government/1k/si-ta/runs/Oct03_06-49-07_NLPC-Sita-MLM/events.out.tfevents.1759454348.NLPC-Sita-MLM.2581313.23 b/Government/1k/si-ta/runs/Oct03_06-49-07_NLPC-Sita-MLM/events.out.tfevents.1759454348.NLPC-Sita-MLM.2581313.23 new file mode 100644 index 0000000000000000000000000000000000000000..7bc0e6035fa2b13a65b34d832d4c203f24a8b6cd --- /dev/null +++ b/Government/1k/si-ta/runs/Oct03_06-49-07_NLPC-Sita-MLM/events.out.tfevents.1759454348.NLPC-Sita-MLM.2581313.23 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e23d9d76ac8cae765cfde214daab54785de9cf2348d876ffbb13134ec1d79fe +size 5651 diff --git a/Government/1k/si-ta_final_model/config.json b/Government/1k/si-ta_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/1k/si-ta_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/1k/si-ta_final_model/generation_config.json b/Government/1k/si-ta_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/1k/si-ta_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/1k/si-ta_final_model/pytorch_model.bin b/Government/1k/si-ta_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..40bf2aaf3d89cbabcf5a9dbdf2c47d86303147d4 --- /dev/null +++ b/Government/1k/si-ta_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f67f82dc6a746a4709b5bbd1981d6505a749eb563e42f10c8e98bbd1c9557ed1 +size 2468870723 diff --git a/Government/1k/si-ta_final_model/training_args.bin b/Government/1k/si-ta_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..a5c8e3020e0e93f25f6c1d52fa47fba9d3eb1fe8 --- /dev/null +++ b/Government/1k/si-ta_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0ef54fbfb23ea70970b74c4e99cd5975765a42eef0ff03a501d9799c00c091a +size 4143 diff --git a/Government/1k/ta-en/runs/Oct03_02-17-51_NLPC-Sita-MLM/events.out.tfevents.1759438072.NLPC-Sita-MLM.2581313.17 b/Government/1k/ta-en/runs/Oct03_02-17-51_NLPC-Sita-MLM/events.out.tfevents.1759438072.NLPC-Sita-MLM.2581313.17 new file mode 100644 index 0000000000000000000000000000000000000000..7d730f6f56b616441f5f9fa5ed94b14c91570af0 --- /dev/null +++ b/Government/1k/ta-en/runs/Oct03_02-17-51_NLPC-Sita-MLM/events.out.tfevents.1759438072.NLPC-Sita-MLM.2581313.17 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:189c752498d523d1ddf70f54254b58bf814e3f45e9447a2f329994a3fe760cd1 +size 5651 diff --git a/Government/1k/ta-en_final_model/config.json b/Government/1k/ta-en_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/1k/ta-en_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/1k/ta-en_final_model/generation_config.json b/Government/1k/ta-en_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/1k/ta-en_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/1k/ta-en_final_model/pytorch_model.bin b/Government/1k/ta-en_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..01050a659e50fd787998558dfe3651d390d53d3b --- /dev/null +++ b/Government/1k/ta-en_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11057c87d60ea8f2b6399208a305d696fb9b3021552031d5ab32b30f20e7eb85 +size 2468870723 diff --git a/Government/1k/ta-en_final_model/training_args.bin b/Government/1k/ta-en_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..c36c2d1a62f0e0437229932c266e6bb1666424bc --- /dev/null +++ b/Government/1k/ta-en_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5dfb6a38084be95031c82fe407808644686731f8fd3fbc691a16f4f6bed4c107 +size 4143 diff --git a/Government/1k/ta-si/runs/Oct03_09-19-58_NLPC-Sita-MLM/events.out.tfevents.1759463398.NLPC-Sita-MLM.2581313.29 b/Government/1k/ta-si/runs/Oct03_09-19-58_NLPC-Sita-MLM/events.out.tfevents.1759463398.NLPC-Sita-MLM.2581313.29 new file mode 100644 index 0000000000000000000000000000000000000000..5b2de28c567132dd7997319e2af3aea74e8add15 --- /dev/null +++ b/Government/1k/ta-si/runs/Oct03_09-19-58_NLPC-Sita-MLM/events.out.tfevents.1759463398.NLPC-Sita-MLM.2581313.29 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4917a2ffdbd6a4074a3b5886097afe0f8c191ef941c0c2320c07ab4c5ee60541 +size 5651 diff --git a/Government/1k/ta-si_final_model/config.json b/Government/1k/ta-si_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/1k/ta-si_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/1k/ta-si_final_model/generation_config.json b/Government/1k/ta-si_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/1k/ta-si_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/1k/ta-si_final_model/pytorch_model.bin b/Government/1k/ta-si_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..971f810cc9f3a8057b40bd634301cd8cc03433d3 --- /dev/null +++ b/Government/1k/ta-si_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa36e75a7a2243ba48f5b4bea6636a9ba65b065c11bfbc7762e98031f28ce144 +size 2468870723 diff --git a/Government/1k/ta-si_final_model/training_args.bin b/Government/1k/ta-si_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..bc700cf67fd67a0a8351e7e97bceb1f1a91441dc --- /dev/null +++ b/Government/1k/ta-si_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90db111d6b592362d1ec3ebb4b4730004dbc11ff42e8292ce7c73844928f089c +size 4143 diff --git a/Government/1k/tok-en-si/sentencepiece.bpe.model b/Government/1k/tok-en-si/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/1k/tok-en-si/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/1k/tok-en-si/special_tokens_map.json b/Government/1k/tok-en-si/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/1k/tok-en-si/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/1k/tok-en-si/tokenizer_config.json b/Government/1k/tok-en-si/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fd1139979a7c7c4188b26d922af637f4bfb23e1f --- /dev/null +++ b/Government/1k/tok-en-si/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "eng_Latn", + "tgt_lang": "sin_Sinh", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/1k/tok-en-ta/sentencepiece.bpe.model b/Government/1k/tok-en-ta/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/1k/tok-en-ta/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/1k/tok-en-ta/special_tokens_map.json b/Government/1k/tok-en-ta/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/1k/tok-en-ta/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/1k/tok-en-ta/tokenizer_config.json b/Government/1k/tok-en-ta/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ef8f263366a29cf3dd0c97df4b45c41fbbf160b5 --- /dev/null +++ b/Government/1k/tok-en-ta/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "eng_Latn", + "tgt_lang": "tam_Taml", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/1k/tok-si-en/sentencepiece.bpe.model b/Government/1k/tok-si-en/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/1k/tok-si-en/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/1k/tok-si-en/special_tokens_map.json b/Government/1k/tok-si-en/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/1k/tok-si-en/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/1k/tok-si-en/tokenizer_config.json b/Government/1k/tok-si-en/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f80b41ef163bc0a47ebef66aa6c1210dfe0b9792 --- /dev/null +++ b/Government/1k/tok-si-en/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "sin_Sinh", + "tgt_lang": "eng_Latn", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/1k/tok-si-ta/sentencepiece.bpe.model b/Government/1k/tok-si-ta/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/1k/tok-si-ta/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/1k/tok-si-ta/special_tokens_map.json b/Government/1k/tok-si-ta/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/1k/tok-si-ta/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/1k/tok-si-ta/tokenizer_config.json b/Government/1k/tok-si-ta/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..386158eef85de789e966861e91bebf69e9a4f32b --- /dev/null +++ b/Government/1k/tok-si-ta/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "sin_Sinh", + "tgt_lang": "tam_Taml", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/1k/tok-ta-en/sentencepiece.bpe.model b/Government/1k/tok-ta-en/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/1k/tok-ta-en/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/1k/tok-ta-en/special_tokens_map.json b/Government/1k/tok-ta-en/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/1k/tok-ta-en/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/1k/tok-ta-en/tokenizer_config.json b/Government/1k/tok-ta-en/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6deaa799386cb7438bf8a0f96250f961c73dec30 --- /dev/null +++ b/Government/1k/tok-ta-en/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "tam_Taml", + "tgt_lang": "eng_Latn", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/1k/tok-ta-si/sentencepiece.bpe.model b/Government/1k/tok-ta-si/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/1k/tok-ta-si/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/1k/tok-ta-si/special_tokens_map.json b/Government/1k/tok-ta-si/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/1k/tok-ta-si/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/1k/tok-ta-si/tokenizer_config.json b/Government/1k/tok-ta-si/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9526d1cacd39f5ee55eece61fdacbbc6ce50136d --- /dev/null +++ b/Government/1k/tok-ta-si/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "tam_Taml", + "tgt_lang": "sin_Sinh", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/20k/en-si_final_model/config.json b/Government/20k/en-si_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/20k/en-si_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/20k/en-si_final_model/generation_config.json b/Government/20k/en-si_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/20k/en-si_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/20k/en-si_final_model/pytorch_model.bin b/Government/20k/en-si_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..857c886223ab31c07becae793922eaaf7cc36076 --- /dev/null +++ b/Government/20k/en-si_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c058319914586271b2763d73da2652f12b861f777d5a9403f5a79f42fcdf87fd +size 2468870723 diff --git a/Government/20k/en-si_final_model/training_args.bin b/Government/20k/en-si_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..24f96321a43acd5e5bab0c89d59474e22aeac110 --- /dev/null +++ b/Government/20k/en-si_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6810d9f1b1a519152f027581fb8399f44777a42177683d263bb4a5331239eb67 +size 4143 diff --git a/Government/20k/en-ta/runs/Oct02_19-58-27_NLPC-Sita-MLM/events.out.tfevents.1759415307.NLPC-Sita-MLM.2581313.1 b/Government/20k/en-ta/runs/Oct02_19-58-27_NLPC-Sita-MLM/events.out.tfevents.1759415307.NLPC-Sita-MLM.2581313.1 new file mode 100644 index 0000000000000000000000000000000000000000..41735a1fe025667b0ee7af9545db97a0b08f4bd8 --- /dev/null +++ b/Government/20k/en-ta/runs/Oct02_19-58-27_NLPC-Sita-MLM/events.out.tfevents.1759415307.NLPC-Sita-MLM.2581313.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7c29eeba71dea71910dbf8108814282283ce92662023468499858360504622e +size 7543 diff --git a/Government/20k/en-ta_final_model/config.json b/Government/20k/en-ta_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/20k/en-ta_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/20k/en-ta_final_model/generation_config.json b/Government/20k/en-ta_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/20k/en-ta_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/20k/en-ta_final_model/pytorch_model.bin b/Government/20k/en-ta_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..c5396641a90b35cbd6c1363d47392eb5da2507e9 --- /dev/null +++ b/Government/20k/en-ta_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f732c3378f519e5df290e99e38ed1add92172032f77628081310807c235874cd +size 2468870723 diff --git a/Government/20k/en-ta_final_model/training_args.bin b/Government/20k/en-ta_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..828a2c9f070eea7188106e9e07cf01f09f753586 --- /dev/null +++ b/Government/20k/en-ta_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d62b8e54c00b59f4928f10c9caceac0bb2041a44d8789883c699025e660b2825 +size 4143 diff --git a/Government/20k/nllb_bible_eval_government_en-si_trained_20k.csv b/Government/20k/nllb_bible_eval_government_en-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..37ee571a140a81a9e80db1e8220b1904b5d6a0cb --- /dev/null +++ b/Government/20k/nllb_bible_eval_government_en-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,31.761604881464233,27.547960015714562,2.7451206165921933,13.882638289577837,13.98031568061058,0.7690197389423847 diff --git a/Government/20k/nllb_bible_eval_government_en-ta_trained_20k.csv b/Government/20k/nllb_bible_eval_government_en-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..97f6ddd2b34009800e09512da0b94ec89740b1d4 --- /dev/null +++ b/Government/20k/nllb_bible_eval_government_en-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,38.28284821810499,32.467935193315014,2.620634215953889,12.909023254023582,11.71730023876448,0.7924355041086674 diff --git a/Government/20k/nllb_bible_eval_government_si-en_trained_20k.csv b/Government/20k/nllb_bible_eval_government_si-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b64695770afaa089ec70cbe19d34b39ef4553cb3 --- /dev/null +++ b/Government/20k/nllb_bible_eval_government_si-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,44.91420789245757,43.873439137375634,21.301441819365863,22.351071106075548,22.206533266775022,0.7740990107506514 diff --git a/Government/20k/nllb_bible_eval_government_ta-en_trained_20k.csv b/Government/20k/nllb_bible_eval_government_ta-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ad185672b2411317350be54f70d717ce1354f4d1 --- /dev/null +++ b/Government/20k/nllb_bible_eval_government_ta-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,39.352881441156754,38.407028109180736,16.471995156082222,18.271513709309875,17.61234070563459,0.7510786295533181 diff --git a/Government/20k/nllb_floresplus_eval_government_en-si_trained_20k.csv b/Government/20k/nllb_floresplus_eval_government_en-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..fe0efef689eab6e648ac5ddd419fa23fd2d42b4a --- /dev/null +++ b/Government/20k/nllb_floresplus_eval_government_en-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,43.34636755091586,39.07131672687825,9.227334131840173,25.83282090100737,29.605976140051986,0.8745417469604448 diff --git a/Government/20k/nllb_floresplus_eval_government_en-ta_trained_20k.csv b/Government/20k/nllb_floresplus_eval_government_en-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..4aeeae2ee689f3b53700d52448fefd255dcf403d --- /dev/null +++ b/Government/20k/nllb_floresplus_eval_government_en-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,50.54670262782469,44.38126135159692,9.807197413498848,23.720731513652108,23.272251376965823,0.8616439145161465 diff --git a/Government/20k/nllb_floresplus_eval_government_si-en_trained_20k.csv b/Government/20k/nllb_floresplus_eval_government_si-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..efc41e9320ca277f78c536770507eae6dfb35e10 --- /dev/null +++ b/Government/20k/nllb_floresplus_eval_government_si-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,56.516526222256225,54.31503846883062,27.52217835877474,29.48182997817969,30.23545759180062,0.864110741985174 diff --git a/Government/20k/nllb_floresplus_eval_government_si-ta_trained_20k.csv b/Government/20k/nllb_floresplus_eval_government_si-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..c9ced3b116f37ee5f937236ca435834a5dd81493 --- /dev/null +++ b/Government/20k/nllb_floresplus_eval_government_si-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,46.487538553989125,40.31292580462015,6.9582131221492585,19.146769036928447,18.933340558617992,0.8471615392965055 diff --git a/Government/20k/nllb_floresplus_eval_government_ta-en_trained_20k.csv b/Government/20k/nllb_floresplus_eval_government_ta-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..c370965c7c9f319f7af23f00be4f4780935f27cc --- /dev/null +++ b/Government/20k/nllb_floresplus_eval_government_ta-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,55.37387878212861,53.254707170996426,26.958480768793823,28.634835686816622,29.578988533306852,0.8464345760614034 diff --git a/Government/20k/nllb_floresplus_eval_government_ta-si_trained_20k.csv b/Government/20k/nllb_floresplus_eval_government_ta-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..0eec9b296b688d7b596fbd8c5ae3803d3a9bc6b2 --- /dev/null +++ b/Government/20k/nllb_floresplus_eval_government_ta-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,38.394709144731394,34.40134327692656,7.262813656482166,20.372393786070468,23.936931551113577,0.8433336493643847 diff --git a/Government/20k/nllb_government_eval_bible_en-si_trained_20k.csv b/Government/20k/nllb_government_eval_bible_en-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..37ee571a140a81a9e80db1e8220b1904b5d6a0cb --- /dev/null +++ b/Government/20k/nllb_government_eval_bible_en-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,31.761604881464233,27.547960015714562,2.7451206165921933,13.882638289577837,13.98031568061058,0.7690197389423847 diff --git a/Government/20k/nllb_government_eval_bible_en-ta_trained_20k.csv b/Government/20k/nllb_government_eval_bible_en-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..97f6ddd2b34009800e09512da0b94ec89740b1d4 --- /dev/null +++ b/Government/20k/nllb_government_eval_bible_en-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,38.28284821810499,32.467935193315014,2.620634215953889,12.909023254023582,11.71730023876448,0.7924355041086674 diff --git a/Government/20k/nllb_government_eval_bible_si-en_trained_20k.csv b/Government/20k/nllb_government_eval_bible_si-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b64695770afaa089ec70cbe19d34b39ef4553cb3 --- /dev/null +++ b/Government/20k/nllb_government_eval_bible_si-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,44.91420789245757,43.873439137375634,21.301441819365863,22.351071106075548,22.206533266775022,0.7740990107506514 diff --git a/Government/20k/nllb_government_eval_bible_ta-en_trained_20k.csv b/Government/20k/nllb_government_eval_bible_ta-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ad185672b2411317350be54f70d717ce1354f4d1 --- /dev/null +++ b/Government/20k/nllb_government_eval_bible_ta-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,39.352881441156754,38.407028109180736,16.471995156082222,18.271513709309875,17.61234070563459,0.7510786295533181 diff --git a/Government/20k/nllb_government_eval_en-si_trained_20k.csv b/Government/20k/nllb_government_eval_en-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..94420294e1ef125368c7d3428fa8564898904e90 --- /dev/null +++ b/Government/20k/nllb_government_eval_en-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,56.878869875934015,52.578890021001925,22.17879298077314,43.632198515862186,49.66696619897928,0.9219063605052861 diff --git a/Government/20k/nllb_government_eval_en-ta_trained_20k.csv b/Government/20k/nllb_government_eval_en-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..bac8e1e62074ec3221f0263b8fc4dfeff33abb13 --- /dev/null +++ b/Government/20k/nllb_government_eval_en-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,59.87668117240686,53.826241128288956,20.51262943254934,36.300055786428274,36.43035360795028,0.8976284621051063 diff --git a/Government/20k/nllb_government_eval_government_en-si_trained_20k.csv b/Government/20k/nllb_government_eval_government_en-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..7846458bbd67f6170512d9bde7e1052fb5f4aeef --- /dev/null +++ b/Government/20k/nllb_government_eval_government_en-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,57.21169238265881,52.83729260161385,22.43693673311732,43.73262113530742,50.00083444741132,0.9225092524886132 diff --git a/Government/20k/nllb_government_eval_government_en-ta_trained_20k.csv b/Government/20k/nllb_government_eval_government_en-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..4a09772d0e2e7f530c1d68bd7ec5f9cbd8ae2098 --- /dev/null +++ b/Government/20k/nllb_government_eval_government_en-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,60.832506487062155,54.728394037524666,21.977958296988536,37.40296049724705,37.44162196006756,0.901136607259512 diff --git a/Government/20k/nllb_government_eval_government_si-en_trained_20k.csv b/Government/20k/nllb_government_eval_government_si-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..e8a52a4d388a2fa8795cf0e79466ffabd1f27bd7 --- /dev/null +++ b/Government/20k/nllb_government_eval_government_si-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,64.29823445582777,61.601305651445806,36.75204629114596,38.70142850164052,39.84540391536018,0.8764319905340672 diff --git a/Government/20k/nllb_government_eval_government_si-ta_trained_20k.csv b/Government/20k/nllb_government_eval_government_si-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..680f307524aa0a5cf9bc0b6ed8f2e0595f3d9842 --- /dev/null +++ b/Government/20k/nllb_government_eval_government_si-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,59.14881079657924,52.82544950369273,20.215442354544713,35.25547044957205,35.461619164613,0.894321031153202 diff --git a/Government/20k/nllb_government_eval_government_ta-en_trained_20k.csv b/Government/20k/nllb_government_eval_government_ta-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..09dc91d79c94b9a8e743940f758de898354f6f86 --- /dev/null +++ b/Government/20k/nllb_government_eval_government_ta-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,63.70925685695593,61.05296545304575,36.695646360709205,38.62036185696437,39.576773417499396,0.8613841069936752 diff --git a/Government/20k/nllb_government_eval_government_ta-si_trained_20k.csv b/Government/20k/nllb_government_eval_government_ta-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..2d829af75ebddaa16b6b526ceda4ac5027041855 --- /dev/null +++ b/Government/20k/nllb_government_eval_government_ta-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,55.04726933015037,50.79367153843363,21.294405869737158,40.66185853738951,46.80437621858133,0.904481227517128 diff --git a/Government/20k/nllb_government_eval_si-en_trained_20k.csv b/Government/20k/nllb_government_eval_si-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..573fe2b4ccf0286976a1a1a1d38e3a808d7f2a39 --- /dev/null +++ b/Government/20k/nllb_government_eval_si-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,64.2597493937966,61.627081231815964,36.99769590271539,39.030336451792955,40.06731932084912,0.8756988644394523 diff --git a/Government/20k/nllb_government_eval_si-ta_trained_20k.csv b/Government/20k/nllb_government_eval_si-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..cc0bcaba424b59a6eb1d1c89c2e263c323de6982 --- /dev/null +++ b/Government/20k/nllb_government_eval_si-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,58.84341193541841,52.65132525410122,19.854716756012458,34.88658677249267,35.143331585528806,0.8921245612887002 diff --git a/Government/20k/nllb_government_eval_ta-en_trained_20k.csv b/Government/20k/nllb_government_eval_ta-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..c1571fee16a559f918b0c3d6f6eed05c7ec123eb --- /dev/null +++ b/Government/20k/nllb_government_eval_ta-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,63.32413610075511,60.70146231789103,36.30554751786256,37.99501331575756,38.94155713350042,0.8577275916389726 diff --git a/Government/20k/nllb_government_eval_ta-si_trained_20k.csv b/Government/20k/nllb_government_eval_ta-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..e5226e739f5447ff77e904b10b6a929fe2efff8f --- /dev/null +++ b/Government/20k/nllb_government_eval_ta-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,54.49960546049106,50.33699582941601,20.77091060326119,40.20947937632308,46.330882655663814,0.9021508307866932 diff --git a/Government/20k/nllb_health_eval_government_en-si_trained_20k.csv b/Government/20k/nllb_health_eval_government_en-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..476e2d7b4b9d7cccafbe3dbb26d8aff63b266729 --- /dev/null +++ b/Government/20k/nllb_health_eval_government_en-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,43.194996889411044,38.93219459599893,11.245618379250637,26.743125840871944,29.755853396932068,0.8339593650082524 diff --git a/Government/20k/nllb_health_eval_government_en-ta_trained_20k.csv b/Government/20k/nllb_health_eval_government_en-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..cb5df1297d790f330e48b20a18af4c53257a4040 --- /dev/null +++ b/Government/20k/nllb_health_eval_government_en-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,57.864730475266526,51.2403734299827,13.537007166607967,30.487751872506202,29.57607711885231,0.8412932092249393 diff --git a/Government/20k/nllb_health_eval_government_si-en_trained_20k.csv b/Government/20k/nllb_health_eval_government_si-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..31341d0cd2da49f518f5948e9260db3ef13b264c --- /dev/null +++ b/Government/20k/nllb_health_eval_government_si-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,57.10297271238651,54.03252746431237,22.972128657325474,28.487712811756758,28.890848978998367,0.8467513275280427 diff --git a/Government/20k/nllb_health_eval_government_si-ta_trained_20k.csv b/Government/20k/nllb_health_eval_government_si-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..495b71582585bb7ec6e49a633b7657ef53d103c8 --- /dev/null +++ b/Government/20k/nllb_health_eval_government_si-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,50.260324332065466,43.4422812064727,7.505271716002482,21.512867888862676,20.674334748665892,0.8072064611110457 diff --git a/Government/20k/nllb_health_eval_government_ta-en_trained_20k.csv b/Government/20k/nllb_health_eval_government_ta-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..4774f5d519495c014300b76121536ecdeac865ec --- /dev/null +++ b/Government/20k/nllb_health_eval_government_ta-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,58.8758507333153,56.191984827088895,28.76203338344732,31.58639392202739,32.41846059697566,0.8308889925479889 diff --git a/Government/20k/nllb_health_eval_government_ta-si_trained_20k.csv b/Government/20k/nllb_health_eval_government_ta-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..d222ba6858dc8d6d52c554621ab5e14babc50dcb --- /dev/null +++ b/Government/20k/nllb_health_eval_government_ta-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,37.08153859266698,33.15436499964011,7.982183313326316,19.493380535411852,21.932104589943624,0.7907066747964341 diff --git a/Government/20k/nllb_jailbreak_eval_government_en-si_trained_20k.csv b/Government/20k/nllb_jailbreak_eval_government_en-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..e71048452c1a29b238afdb974b9e6d900dbfedf7 --- /dev/null +++ b/Government/20k/nllb_jailbreak_eval_government_en-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,41.75732488247362,38.216343436817596,12.26004639904115,22.983043562325175,25.839150856485922,0.8406499291956425 diff --git a/Government/20k/nllb_jailbreak_eval_government_en-ta_trained_20k.csv b/Government/20k/nllb_jailbreak_eval_government_en-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..180dae9824502844f5b47886184ecb398af12707 --- /dev/null +++ b/Government/20k/nllb_jailbreak_eval_government_en-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,56.40288106711744,51.35929482121719,19.369457994934034,31.485539332141826,30.495628128187413,0.8543994006514549 diff --git a/Government/20k/nllb_jailbreak_eval_government_si-en_trained_20k.csv b/Government/20k/nllb_jailbreak_eval_government_si-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..359a1866f86bd2bbdac0e3bf6fd70056af07cb80 --- /dev/null +++ b/Government/20k/nllb_jailbreak_eval_government_si-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,53.41263845863003,51.60699080241575,27.499773206929646,28.729020323413017,29.376364538435467,0.8524583500027657 diff --git a/Government/20k/nllb_jailbreak_eval_government_si-ta_trained_20k.csv b/Government/20k/nllb_jailbreak_eval_government_si-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..dbadfb7417c927a6dec7c042ee96321e22e8a610 --- /dev/null +++ b/Government/20k/nllb_jailbreak_eval_government_si-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,50.91020212917918,45.596668896008374,13.316260018099728,23.26058858982264,22.461570864255275,0.8351807125210762 diff --git a/Government/20k/nllb_jailbreak_eval_government_ta-en_trained_20k.csv b/Government/20k/nllb_jailbreak_eval_government_ta-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..f6f1f42fa8e94b9a3e1fccc4ffaf0a43b261c592 --- /dev/null +++ b/Government/20k/nllb_jailbreak_eval_government_ta-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,54.974062519610186,53.75881317245834,31.844467583229925,30.594213601152546,31.404409672478806,0.8428259556293487 diff --git a/Government/20k/nllb_jailbreak_eval_government_ta-si_trained_20k.csv b/Government/20k/nllb_jailbreak_eval_government_ta-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..53b6a2817882bf9028e45bb4ac2c908098bf8411 --- /dev/null +++ b/Government/20k/nllb_jailbreak_eval_government_ta-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,37.12786168243185,34.14130791151598,10.769647874978967,19.216753233688237,21.490824300211674,0.8091989916861058 diff --git a/Government/20k/nllb_literature-children_eval_government_en-si_trained_20k.csv b/Government/20k/nllb_literature-children_eval_government_en-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..7d0ed4f074db900de2989e271204728d69f597b7 --- /dev/null +++ b/Government/20k/nllb_literature-children_eval_government_en-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,30.323146952610823,27.219078742914693,4.997742728382483,11.572875707289988,13.355103424191405,0.8145392687618732 diff --git a/Government/20k/nllb_literature-children_eval_government_en-ta_trained_20k.csv b/Government/20k/nllb_literature-children_eval_government_en-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..dafb407c610152bd0c2512a5d9660d21ba18e4f6 --- /dev/null +++ b/Government/20k/nllb_literature-children_eval_government_en-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,52.47163916860495,47.0332261198968,12.643277181350125,25.68446427296771,24.678714971778387,0.8305922095077457 diff --git a/Government/20k/nllb_literature-children_eval_government_si-en_trained_20k.csv b/Government/20k/nllb_literature-children_eval_government_si-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..d10d42d4ecf2af901e1d46764b1c21c4c5080ee0 --- /dev/null +++ b/Government/20k/nllb_literature-children_eval_government_si-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,43.40740277127494,42.345012525977396,19.070368110625598,18.888681230286217,19.042522371038515,0.8063090334832669 diff --git a/Government/20k/nllb_literature-children_eval_government_si-ta_trained_20k.csv b/Government/20k/nllb_literature-children_eval_government_si-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..1e81220a9710927640efd28ddfeb13a3bd2d50e5 --- /dev/null +++ b/Government/20k/nllb_literature-children_eval_government_si-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,43.93742835498326,38.49243562653017,6.303843648287157,15.444236178288653,14.804014853927448,0.7984641714718654 diff --git a/Government/20k/nllb_literature-children_eval_government_ta-en_trained_20k.csv b/Government/20k/nllb_literature-children_eval_government_ta-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..552ae23fca30fc2cbe11e91a5d27c9742c5510cd --- /dev/null +++ b/Government/20k/nllb_literature-children_eval_government_ta-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,50.505420243857714,49.75731011942967,26.53612539809372,25.79033326888846,26.262142190743685,0.815742032567936 diff --git a/Government/20k/nllb_literature-children_eval_government_ta-si_trained_20k.csv b/Government/20k/nllb_literature-children_eval_government_ta-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..6106bcaa85dadca97ee0cda0f1d9b0afc0aa2980 --- /dev/null +++ b/Government/20k/nllb_literature-children_eval_government_ta-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,30.15441975702176,27.009932319452407,4.1769704779573775,10.643300021881029,12.345638717868718,0.7941161549406935 diff --git a/Government/20k/nllb_literature-kathlyn_eval_government_en-si_trained_20k.csv b/Government/20k/nllb_literature-kathlyn_eval_government_en-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..7d0ed4f074db900de2989e271204728d69f597b7 --- /dev/null +++ b/Government/20k/nllb_literature-kathlyn_eval_government_en-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,30.323146952610823,27.219078742914693,4.997742728382483,11.572875707289988,13.355103424191405,0.8145392687618732 diff --git a/Government/20k/nllb_literature-kathlyn_eval_government_en-ta_trained_20k.csv b/Government/20k/nllb_literature-kathlyn_eval_government_en-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..dafb407c610152bd0c2512a5d9660d21ba18e4f6 --- /dev/null +++ b/Government/20k/nllb_literature-kathlyn_eval_government_en-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,52.47163916860495,47.0332261198968,12.643277181350125,25.68446427296771,24.678714971778387,0.8305922095077457 diff --git a/Government/20k/nllb_literature-kathlyn_eval_government_si-en_trained_20k.csv b/Government/20k/nllb_literature-kathlyn_eval_government_si-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..d10d42d4ecf2af901e1d46764b1c21c4c5080ee0 --- /dev/null +++ b/Government/20k/nllb_literature-kathlyn_eval_government_si-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,43.40740277127494,42.345012525977396,19.070368110625598,18.888681230286217,19.042522371038515,0.8063090334832669 diff --git a/Government/20k/nllb_literature-kathlyn_eval_government_si-ta_trained_20k.csv b/Government/20k/nllb_literature-kathlyn_eval_government_si-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..1e81220a9710927640efd28ddfeb13a3bd2d50e5 --- /dev/null +++ b/Government/20k/nllb_literature-kathlyn_eval_government_si-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,43.93742835498326,38.49243562653017,6.303843648287157,15.444236178288653,14.804014853927448,0.7984641714718654 diff --git a/Government/20k/nllb_literature-kathlyn_eval_government_ta-en_trained_20k.csv b/Government/20k/nllb_literature-kathlyn_eval_government_ta-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..552ae23fca30fc2cbe11e91a5d27c9742c5510cd --- /dev/null +++ b/Government/20k/nllb_literature-kathlyn_eval_government_ta-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,50.505420243857714,49.75731011942967,26.53612539809372,25.79033326888846,26.262142190743685,0.815742032567936 diff --git a/Government/20k/nllb_literature-kathlyn_eval_government_ta-si_trained_20k.csv b/Government/20k/nllb_literature-kathlyn_eval_government_ta-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..6106bcaa85dadca97ee0cda0f1d9b0afc0aa2980 --- /dev/null +++ b/Government/20k/nllb_literature-kathlyn_eval_government_ta-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,30.15441975702176,27.009932319452407,4.1769704779573775,10.643300021881029,12.345638717868718,0.7941161549406935 diff --git a/Government/20k/nllb_literature-old_eval_government_en-si_trained_20k.csv b/Government/20k/nllb_literature-old_eval_government_en-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..7d0ed4f074db900de2989e271204728d69f597b7 --- /dev/null +++ b/Government/20k/nllb_literature-old_eval_government_en-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,30.323146952610823,27.219078742914693,4.997742728382483,11.572875707289988,13.355103424191405,0.8145392687618732 diff --git a/Government/20k/nllb_literature-old_eval_government_en-ta_trained_20k.csv b/Government/20k/nllb_literature-old_eval_government_en-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..dafb407c610152bd0c2512a5d9660d21ba18e4f6 --- /dev/null +++ b/Government/20k/nllb_literature-old_eval_government_en-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,52.47163916860495,47.0332261198968,12.643277181350125,25.68446427296771,24.678714971778387,0.8305922095077457 diff --git a/Government/20k/nllb_literature-old_eval_government_si-en_trained_20k.csv b/Government/20k/nllb_literature-old_eval_government_si-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..d10d42d4ecf2af901e1d46764b1c21c4c5080ee0 --- /dev/null +++ b/Government/20k/nllb_literature-old_eval_government_si-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,43.40740277127494,42.345012525977396,19.070368110625598,18.888681230286217,19.042522371038515,0.8063090334832669 diff --git a/Government/20k/nllb_literature-old_eval_government_si-ta_trained_20k.csv b/Government/20k/nllb_literature-old_eval_government_si-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..1e81220a9710927640efd28ddfeb13a3bd2d50e5 --- /dev/null +++ b/Government/20k/nllb_literature-old_eval_government_si-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,43.93742835498326,38.49243562653017,6.303843648287157,15.444236178288653,14.804014853927448,0.7984641714718654 diff --git a/Government/20k/nllb_literature-old_eval_government_ta-en_trained_20k.csv b/Government/20k/nllb_literature-old_eval_government_ta-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..552ae23fca30fc2cbe11e91a5d27c9742c5510cd --- /dev/null +++ b/Government/20k/nllb_literature-old_eval_government_ta-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,50.505420243857714,49.75731011942967,26.53612539809372,25.79033326888846,26.262142190743685,0.815742032567936 diff --git a/Government/20k/nllb_literature-old_eval_government_ta-si_trained_20k.csv b/Government/20k/nllb_literature-old_eval_government_ta-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..6106bcaa85dadca97ee0cda0f1d9b0afc0aa2980 --- /dev/null +++ b/Government/20k/nllb_literature-old_eval_government_ta-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,30.15441975702176,27.009932319452407,4.1769704779573775,10.643300021881029,12.345638717868718,0.7941161549406935 diff --git a/Government/20k/nllb_literature_eval_government_en-si_trained_20k.csv b/Government/20k/nllb_literature_eval_government_en-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..7d0ed4f074db900de2989e271204728d69f597b7 --- /dev/null +++ b/Government/20k/nllb_literature_eval_government_en-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,30.323146952610823,27.219078742914693,4.997742728382483,11.572875707289988,13.355103424191405,0.8145392687618732 diff --git a/Government/20k/nllb_literature_eval_government_en-ta_trained_20k.csv b/Government/20k/nllb_literature_eval_government_en-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..dafb407c610152bd0c2512a5d9660d21ba18e4f6 --- /dev/null +++ b/Government/20k/nllb_literature_eval_government_en-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,52.47163916860495,47.0332261198968,12.643277181350125,25.68446427296771,24.678714971778387,0.8305922095077457 diff --git a/Government/20k/nllb_literature_eval_government_si-en_trained_20k.csv b/Government/20k/nllb_literature_eval_government_si-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..d10d42d4ecf2af901e1d46764b1c21c4c5080ee0 --- /dev/null +++ b/Government/20k/nllb_literature_eval_government_si-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,43.40740277127494,42.345012525977396,19.070368110625598,18.888681230286217,19.042522371038515,0.8063090334832669 diff --git a/Government/20k/nllb_literature_eval_government_si-ta_trained_20k.csv b/Government/20k/nllb_literature_eval_government_si-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..1e81220a9710927640efd28ddfeb13a3bd2d50e5 --- /dev/null +++ b/Government/20k/nllb_literature_eval_government_si-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,43.93742835498326,38.49243562653017,6.303843648287157,15.444236178288653,14.804014853927448,0.7984641714718654 diff --git a/Government/20k/nllb_literature_eval_government_ta-en_trained_20k.csv b/Government/20k/nllb_literature_eval_government_ta-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..552ae23fca30fc2cbe11e91a5d27c9742c5510cd --- /dev/null +++ b/Government/20k/nllb_literature_eval_government_ta-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,50.505420243857714,49.75731011942967,26.53612539809372,25.79033326888846,26.262142190743685,0.815742032567936 diff --git a/Government/20k/nllb_literature_eval_government_ta-si_trained_20k.csv b/Government/20k/nllb_literature_eval_government_ta-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..6106bcaa85dadca97ee0cda0f1d9b0afc0aa2980 --- /dev/null +++ b/Government/20k/nllb_literature_eval_government_ta-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,30.15441975702176,27.009932319452407,4.1769704779573775,10.643300021881029,12.345638717868718,0.7941161549406935 diff --git a/Government/20k/nllb_maths_eval_government_en-si_trained_20k.csv b/Government/20k/nllb_maths_eval_government_en-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..05b9c89f67cbcc443655f0c814b4c9a12b366544 --- /dev/null +++ b/Government/20k/nllb_maths_eval_government_en-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,40.16798911664159,38.07374908008929,16.656258371139497,25.488915980680275,27.846527606251854,0.8411055938601494 diff --git a/Government/20k/nllb_maths_eval_government_en-ta_trained_20k.csv b/Government/20k/nllb_maths_eval_government_en-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..9edbf9fd859c39ebf2739a07f3b667c5af3a710f --- /dev/null +++ b/Government/20k/nllb_maths_eval_government_en-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,50.17759637654757,46.10077286825023,19.552887677839095,25.944749033962783,25.191118400261338,0.8320967829525471 diff --git a/Government/20k/nllb_maths_eval_government_si-en_trained_20k.csv b/Government/20k/nllb_maths_eval_government_si-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..37aec0e3f20dbc389aab2d44591e072f95281f77 --- /dev/null +++ b/Government/20k/nllb_maths_eval_government_si-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,57.20566080895111,56.58499485696334,37.49051482258888,37.57353683685339,38.15952997638359,0.844492649435997 diff --git a/Government/20k/nllb_maths_eval_government_ta-en_trained_20k.csv b/Government/20k/nllb_maths_eval_government_ta-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b9ff6ab4ded0e2dd154bdaac46158f4dddcafcfe --- /dev/null +++ b/Government/20k/nllb_maths_eval_government_ta-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,54.390175800976536,53.56059851807995,34.23743796064806,33.83403558143353,34.5719096302564,0.8334979208707809 diff --git a/Government/20k/nllb_news_eval_government_en-si_trained_20k.csv b/Government/20k/nllb_news_eval_government_en-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b8d45f9430de0e09e4b6e4a03d4c0f2b7c34e47c --- /dev/null +++ b/Government/20k/nllb_news_eval_government_en-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,45.4274593270867,41.001988715490064,11.590726452242302,27.283072483750267,32.96372236158503,0.879407027348876 diff --git a/Government/20k/nllb_news_eval_government_en-ta_trained_20k.csv b/Government/20k/nllb_news_eval_government_en-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b521b210f88f9f546f6e6e5e5f9a340f592a0752 --- /dev/null +++ b/Government/20k/nllb_news_eval_government_en-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,57.746743278002036,52.02965375140137,15.87429821987729,32.689456054430885,32.341695161283745,0.8664364232475977 diff --git a/Government/20k/nllb_news_eval_government_si-en_trained_20k.csv b/Government/20k/nllb_news_eval_government_si-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..f95f21ea9d188fec9c5f567ebee138f7cb05ac88 --- /dev/null +++ b/Government/20k/nllb_news_eval_government_si-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,55.56985871516454,52.88673440348383,23.521531094749502,26.347645725152898,27.213331193706768,0.8544390945136547 diff --git a/Government/20k/nllb_news_eval_government_si-ta_trained_20k.csv b/Government/20k/nllb_news_eval_government_si-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..5f4016e0ade8573a0b282eb54c57296cafc0bbf6 --- /dev/null +++ b/Government/20k/nllb_news_eval_government_si-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,49.75691223157466,43.71992192067232,8.308092514389523,21.38375516081867,21.43980144304131,0.8479972150056593 diff --git a/Government/20k/nllb_news_eval_government_ta-en_trained_20k.csv b/Government/20k/nllb_news_eval_government_ta-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ca05a04ca4f1201e59bfd14d283d8db036523ea0 --- /dev/null +++ b/Government/20k/nllb_news_eval_government_ta-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,59.561616365419844,57.769991001503406,32.340413891555414,33.55668401317413,34.61805405095608,0.8567443510973789 diff --git a/Government/20k/nllb_news_eval_government_ta-si_trained_20k.csv b/Government/20k/nllb_news_eval_government_ta-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..526b476a4d40dae194bb143a9adc8e0679900e09 --- /dev/null +++ b/Government/20k/nllb_news_eval_government_ta-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,37.28571821610571,33.188914491541766,6.5162986350236345,19.537762120428738,22.92513616951124,0.8252781428276531 diff --git a/Government/20k/nllb_nllb_eval_government_en-si_trained_20k.csv b/Government/20k/nllb_nllb_eval_government_en-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..72e46e0e1af3068aa823bb5a6a830604c2eaf34c --- /dev/null +++ b/Government/20k/nllb_nllb_eval_government_en-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,42.867376454352396,39.32278272621688,11.347460755796229,23.681411463770186,27.913056322713402,0.9010286902821903 diff --git a/Government/20k/nllb_nllb_eval_government_en-ta_trained_20k.csv b/Government/20k/nllb_nllb_eval_government_en-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..6568c19edd2eea46f91924a7911dd069e2c6037e --- /dev/null +++ b/Government/20k/nllb_nllb_eval_government_en-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,56.04076681501125,51.08520502581324,16.989963735299348,30.094701617954517,29.162302264301612,0.9051107998288279 diff --git a/Government/20k/nllb_nllb_eval_government_si-en_trained_20k.csv b/Government/20k/nllb_nllb_eval_government_si-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..28f63c7bd8ca185156d5c4a8b00441a45d09e6c9 --- /dev/null +++ b/Government/20k/nllb_nllb_eval_government_si-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,62.05599317037146,61.280281004093354,40.91603177497263,40.8178257183554,41.027606741137376,0.8979786402037664 diff --git a/Government/20k/nllb_nllb_eval_government_si-ta_trained_20k.csv b/Government/20k/nllb_nllb_eval_government_si-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..9f0b08d81281808f21e74355a3daf2155e8a4fe7 --- /dev/null +++ b/Government/20k/nllb_nllb_eval_government_si-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,53.24051678377969,48.073056935221835,13.568906081074187,26.30433318149175,25.297484700408354,0.896335870414392 diff --git a/Government/20k/nllb_nllb_eval_government_ta-en_trained_20k.csv b/Government/20k/nllb_nllb_eval_government_ta-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ee9506993af8544662f3c5db4b6bf0e2fe802ab2 --- /dev/null +++ b/Government/20k/nllb_nllb_eval_government_ta-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,62.539739987267374,61.82136271659,41.69018052840806,41.77167609895496,41.98730022828941,0.8877965231954724 diff --git a/Government/20k/nllb_nllb_eval_government_ta-si_trained_20k.csv b/Government/20k/nllb_nllb_eval_government_ta-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..4add9935789e49ba16b20130997f0da3cda611e3 --- /dev/null +++ b/Government/20k/nllb_nllb_eval_government_ta-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,40.70668509577528,37.42139152547206,10.40911683528101,21.362313248810686,25.230424046965325,0.8851708894780559 diff --git a/Government/20k/nllb_opensubtitles_eval_government_en-si_trained_20k.csv b/Government/20k/nllb_opensubtitles_eval_government_en-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..60b8e049d838fedb77a1c7bd964859a115896967 --- /dev/null +++ b/Government/20k/nllb_opensubtitles_eval_government_en-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,35.12391260428374,32.105489417981794,7.502054254124334,19.84046279426906,22.19210912077999,0.8655002781748772 diff --git a/Government/20k/nllb_opensubtitles_eval_government_en-ta_trained_20k.csv b/Government/20k/nllb_opensubtitles_eval_government_en-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..9cec5d2e14acf6aa7878ca5c3e8b12d62d619d57 --- /dev/null +++ b/Government/20k/nllb_opensubtitles_eval_government_en-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,48.443547247449175,44.6555521776317,15.59176727548575,27.03275688100976,26.801265810086395,0.85416314214468 diff --git a/Government/20k/nllb_opensubtitles_eval_government_si-en_trained_20k.csv b/Government/20k/nllb_opensubtitles_eval_government_si-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..80179e558746048d7b285066b2c96171cbc027fa --- /dev/null +++ b/Government/20k/nllb_opensubtitles_eval_government_si-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,61.747554780003625,60.91758251594824,38.6864159002422,39.59130143461301,39.86300529902638,0.8888738882243633 diff --git a/Government/20k/nllb_opensubtitles_eval_government_si-ta_trained_20k.csv b/Government/20k/nllb_opensubtitles_eval_government_si-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b3f0e5969a802ebe4478124de7f92a554ae85638 --- /dev/null +++ b/Government/20k/nllb_opensubtitles_eval_government_si-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,53.08459688059487,48.65732276542112,16.262591401317337,28.4537268730661,27.756742230218972,0.8849549402892589 diff --git a/Government/20k/nllb_opensubtitles_eval_government_ta-en_trained_20k.csv b/Government/20k/nllb_opensubtitles_eval_government_ta-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ce66c3ac535afe0a26aea38a72fcd287e924ad7c --- /dev/null +++ b/Government/20k/nllb_opensubtitles_eval_government_ta-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,59.737405956280995,59.04830807240404,37.091242067017184,37.87127352387377,38.09516634037666,0.8707491488307715 diff --git a/Government/20k/nllb_opensubtitles_eval_government_ta-si_trained_20k.csv b/Government/20k/nllb_opensubtitles_eval_government_ta-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..0e23b2dd9b8669ee0c5f4b36c75f3528ff3642e7 --- /dev/null +++ b/Government/20k/nllb_opensubtitles_eval_government_ta-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,32.37243219819934,29.3066901510309,5.980697619581673,16.681144463590545,18.476464714958414,0.8546403640657664 diff --git a/Government/20k/nllb_wikipedia_eval_government_en-si_trained_20k.csv b/Government/20k/nllb_wikipedia_eval_government_en-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b36b5577e0b25ad06264df04a4460a7e06b7af98 --- /dev/null +++ b/Government/20k/nllb_wikipedia_eval_government_en-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,41.55216385298722,37.118928968163544,7.435175493487085,22.869206294691708,26.457512815265737,0.8730469604730606 diff --git a/Government/20k/nllb_wikipedia_eval_government_en-ta_trained_20k.csv b/Government/20k/nllb_wikipedia_eval_government_en-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..f2ee93ffe67b06dc843448a350fb8f598c4d15e2 --- /dev/null +++ b/Government/20k/nllb_wikipedia_eval_government_en-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,53.91530090818076,47.48323717505256,10.894641087837911,28.106499127369172,27.98331556305807,0.8761209879219533 diff --git a/Government/20k/nllb_wikipedia_eval_government_si-en_trained_20k.csv b/Government/20k/nllb_wikipedia_eval_government_si-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..20a6f9151afbf50508685b779ac9287779218c9c --- /dev/null +++ b/Government/20k/nllb_wikipedia_eval_government_si-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,54.349890181366554,52.24562779681402,25.14899465512415,28.50412595867878,28.93093362503844,0.8428374643325806 diff --git a/Government/20k/nllb_wikipedia_eval_government_si-ta_trained_20k.csv b/Government/20k/nllb_wikipedia_eval_government_si-ta_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..9a2e51da86a6092d2752aa9dce3e94ef6666f53c --- /dev/null +++ b/Government/20k/nllb_wikipedia_eval_government_si-ta_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,47.74047028670271,41.385700049697526,7.188332326652838,21.410848685094138,21.393042684623886,0.8521688247025013 diff --git a/Government/20k/nllb_wikipedia_eval_government_ta-en_trained_20k.csv b/Government/20k/nllb_wikipedia_eval_government_ta-en_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..f13d44532fe1aad9fcbc9b503d15e8f42908b6bf --- /dev/null +++ b/Government/20k/nllb_wikipedia_eval_government_ta-en_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,59.11743537461197,57.25481268525855,31.142725749303022,33.95221240983595,34.52268266906763,0.8468463129401207 diff --git a/Government/20k/nllb_wikipedia_eval_government_ta-si_trained_20k.csv b/Government/20k/nllb_wikipedia_eval_government_ta-si_trained_20k.csv new file mode 100644 index 0000000000000000000000000000000000000000..037aed4faa25e4ccc3037287ee382dc8253d817c --- /dev/null +++ b/Government/20k/nllb_wikipedia_eval_government_ta-si_trained_20k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,38.315255110238624,34.157239328495564,6.343790929618099,19.59889799872615,22.762480237872946,0.8501061606109143 diff --git a/Government/20k/si-en/runs/Oct02_22-24-09_NLPC-Sita-MLM/events.out.tfevents.1759424049.NLPC-Sita-MLM.2581313.7 b/Government/20k/si-en/runs/Oct02_22-24-09_NLPC-Sita-MLM/events.out.tfevents.1759424049.NLPC-Sita-MLM.2581313.7 new file mode 100644 index 0000000000000000000000000000000000000000..dce5f43fed0e0c54e9448e7032d45edc87f854c6 --- /dev/null +++ b/Government/20k/si-en/runs/Oct02_22-24-09_NLPC-Sita-MLM/events.out.tfevents.1759424049.NLPC-Sita-MLM.2581313.7 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0bfc22e056392e576872cc7a50d366d59bcb86aa110bf7e24e2bc626cafddbf +size 7543 diff --git a/Government/20k/si-en_final_model/config.json b/Government/20k/si-en_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/20k/si-en_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/20k/si-en_final_model/generation_config.json b/Government/20k/si-en_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/20k/si-en_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/20k/si-en_final_model/pytorch_model.bin b/Government/20k/si-en_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..695c582c9e9df8687fdc277d9d862ecd676cd3a7 --- /dev/null +++ b/Government/20k/si-en_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79cc71e9eb4e136f3e442831c09d8b6284ddb0f6cceb152a3463ab05213e00a8 +size 2468870723 diff --git a/Government/20k/si-en_final_model/training_args.bin b/Government/20k/si-en_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..0634f83be9672a912bca2a070aa72af49cdc61d6 --- /dev/null +++ b/Government/20k/si-en_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:483882e5d9eb154521cd71ad192cbd16e6eef44e1ded5dfb968469b436e58885 +size 4143 diff --git a/Government/20k/si-ta/runs/Oct03_03-09-05_NLPC-Sita-MLM/events.out.tfevents.1759441145.NLPC-Sita-MLM.2581313.19 b/Government/20k/si-ta/runs/Oct03_03-09-05_NLPC-Sita-MLM/events.out.tfevents.1759441145.NLPC-Sita-MLM.2581313.19 new file mode 100644 index 0000000000000000000000000000000000000000..cf9b09a7040e0d3d33d616ac8ddf3506e3751c4f --- /dev/null +++ b/Government/20k/si-ta/runs/Oct03_03-09-05_NLPC-Sita-MLM/events.out.tfevents.1759441145.NLPC-Sita-MLM.2581313.19 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4030f7bb7de12344e0d0b84155eaa80092799c2bf6950e6274ba7d93a773d205 +size 7543 diff --git a/Government/20k/si-ta_final_model/config.json b/Government/20k/si-ta_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/20k/si-ta_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/20k/si-ta_final_model/generation_config.json b/Government/20k/si-ta_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/20k/si-ta_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/20k/si-ta_final_model/pytorch_model.bin b/Government/20k/si-ta_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..5cb1bbde64bf744aad3529fe1371ed44f995ee78 --- /dev/null +++ b/Government/20k/si-ta_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:129b81e8eeae0c6fa6b6299d6a39b0d6941416083f9cbfbc0270aab98b5508a6 +size 2468870723 diff --git a/Government/20k/si-ta_final_model/training_args.bin b/Government/20k/si-ta_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..f853b2bc3e56636f28bbf73f346eb4b25ce87780 --- /dev/null +++ b/Government/20k/si-ta_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3ffd9400f758145770abdeee7184db724c37c4f7280ee11befb734c2084ce76 +size 4143 diff --git a/Government/20k/ta-en/runs/Oct03_00-45-34_NLPC-Sita-MLM/events.out.tfevents.1759432535.NLPC-Sita-MLM.2581313.13 b/Government/20k/ta-en/runs/Oct03_00-45-34_NLPC-Sita-MLM/events.out.tfevents.1759432535.NLPC-Sita-MLM.2581313.13 new file mode 100644 index 0000000000000000000000000000000000000000..1c516405a98d9efd44a188bec336168129d6fe5a --- /dev/null +++ b/Government/20k/ta-en/runs/Oct03_00-45-34_NLPC-Sita-MLM/events.out.tfevents.1759432535.NLPC-Sita-MLM.2581313.13 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4876090e5be37064162805c0f770243f46f360b4c130f94a36572bf1eeb75cc2 +size 7543 diff --git a/Government/20k/ta-en_final_model/config.json b/Government/20k/ta-en_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/20k/ta-en_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/20k/ta-en_final_model/generation_config.json b/Government/20k/ta-en_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/20k/ta-en_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/20k/ta-en_final_model/pytorch_model.bin b/Government/20k/ta-en_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..4086867653ea6ee87b2e761be2d54991a5b4e178 --- /dev/null +++ b/Government/20k/ta-en_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89ce39124ceeee546ad3afd17085f7db3282f2bfde70628b888f0e6cc993d643 +size 2468870723 diff --git a/Government/20k/ta-en_final_model/training_args.bin b/Government/20k/ta-en_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..e7c3f14c36eaaad5f6a74933d62365cd7845baa1 --- /dev/null +++ b/Government/20k/ta-en_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fb69562441ae09d20b7c089c1508e2de4f404e06dc81b119c47e8bacc204585 +size 4143 diff --git a/Government/20k/ta-si/runs/Oct03_07-40-43_NLPC-Sita-MLM/events.out.tfevents.1759457443.NLPC-Sita-MLM.2581313.25 b/Government/20k/ta-si/runs/Oct03_07-40-43_NLPC-Sita-MLM/events.out.tfevents.1759457443.NLPC-Sita-MLM.2581313.25 new file mode 100644 index 0000000000000000000000000000000000000000..bfc4c046a267105cefd13df29e479a04b20c63f5 --- /dev/null +++ b/Government/20k/ta-si/runs/Oct03_07-40-43_NLPC-Sita-MLM/events.out.tfevents.1759457443.NLPC-Sita-MLM.2581313.25 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:266431519bd8374509fa42c892eca6eccdcdcf03c60568ffb588fd0e158a4360 +size 7543 diff --git a/Government/20k/ta-si_final_model/config.json b/Government/20k/ta-si_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/20k/ta-si_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/20k/ta-si_final_model/generation_config.json b/Government/20k/ta-si_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/20k/ta-si_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/20k/ta-si_final_model/pytorch_model.bin b/Government/20k/ta-si_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..53c991380a87162052386756346df6150d2b345f --- /dev/null +++ b/Government/20k/ta-si_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd870b7254796f8029c0c517fee356c00476449fc679f7cc5949e038ce1c89c4 +size 2468870723 diff --git a/Government/20k/ta-si_final_model/training_args.bin b/Government/20k/ta-si_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..e4468a2950405aa6a30c25dba23b8afad0731f18 --- /dev/null +++ b/Government/20k/ta-si_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90609fe3410387652ac6b7b44b586cd624706d6a527ae1afe9b66bf6d4d39044 +size 4143 diff --git a/Government/20k/tok-en-si/sentencepiece.bpe.model b/Government/20k/tok-en-si/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/20k/tok-en-si/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/20k/tok-en-si/special_tokens_map.json b/Government/20k/tok-en-si/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/20k/tok-en-si/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/20k/tok-en-si/tokenizer_config.json b/Government/20k/tok-en-si/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fd1139979a7c7c4188b26d922af637f4bfb23e1f --- /dev/null +++ b/Government/20k/tok-en-si/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "eng_Latn", + "tgt_lang": "sin_Sinh", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/20k/tok-en-ta/sentencepiece.bpe.model b/Government/20k/tok-en-ta/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/20k/tok-en-ta/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/20k/tok-en-ta/special_tokens_map.json b/Government/20k/tok-en-ta/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/20k/tok-en-ta/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/20k/tok-en-ta/tokenizer_config.json b/Government/20k/tok-en-ta/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ef8f263366a29cf3dd0c97df4b45c41fbbf160b5 --- /dev/null +++ b/Government/20k/tok-en-ta/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "eng_Latn", + "tgt_lang": "tam_Taml", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/20k/tok-si-en/sentencepiece.bpe.model b/Government/20k/tok-si-en/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/20k/tok-si-en/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/20k/tok-si-en/special_tokens_map.json b/Government/20k/tok-si-en/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/20k/tok-si-en/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/20k/tok-si-en/tokenizer_config.json b/Government/20k/tok-si-en/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f80b41ef163bc0a47ebef66aa6c1210dfe0b9792 --- /dev/null +++ b/Government/20k/tok-si-en/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "sin_Sinh", + "tgt_lang": "eng_Latn", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/20k/tok-si-ta/sentencepiece.bpe.model b/Government/20k/tok-si-ta/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/20k/tok-si-ta/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/20k/tok-si-ta/special_tokens_map.json b/Government/20k/tok-si-ta/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/20k/tok-si-ta/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/20k/tok-si-ta/tokenizer_config.json b/Government/20k/tok-si-ta/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..386158eef85de789e966861e91bebf69e9a4f32b --- /dev/null +++ b/Government/20k/tok-si-ta/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "sin_Sinh", + "tgt_lang": "tam_Taml", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/20k/tok-ta-en/sentencepiece.bpe.model b/Government/20k/tok-ta-en/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/20k/tok-ta-en/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/20k/tok-ta-en/special_tokens_map.json b/Government/20k/tok-ta-en/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/20k/tok-ta-en/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/20k/tok-ta-en/tokenizer_config.json b/Government/20k/tok-ta-en/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6deaa799386cb7438bf8a0f96250f961c73dec30 --- /dev/null +++ b/Government/20k/tok-ta-en/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "tam_Taml", + "tgt_lang": "eng_Latn", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/20k/tok-ta-si/sentencepiece.bpe.model b/Government/20k/tok-ta-si/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/20k/tok-ta-si/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/20k/tok-ta-si/special_tokens_map.json b/Government/20k/tok-ta-si/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/20k/tok-ta-si/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/20k/tok-ta-si/tokenizer_config.json b/Government/20k/tok-ta-si/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9526d1cacd39f5ee55eece61fdacbbc6ce50136d --- /dev/null +++ b/Government/20k/tok-ta-si/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "tam_Taml", + "tgt_lang": "sin_Sinh", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/25k/en-si_final_model/config.json b/Government/25k/en-si_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/25k/en-si_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/25k/en-si_final_model/generation_config.json b/Government/25k/en-si_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/25k/en-si_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/25k/en-si_final_model/pytorch_model.bin b/Government/25k/en-si_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..2543b054f29d00df771117cfd7185f6fc3f313ab --- /dev/null +++ b/Government/25k/en-si_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91129c145259a4506006bd1b91da90d2f6bd28aaef2e3bd9b28b13a6ff00faa3 +size 2468870723 diff --git a/Government/25k/en-si_final_model/training_args.bin b/Government/25k/en-si_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..70fd39f50524fcb25a648e91cc02962611085eab --- /dev/null +++ b/Government/25k/en-si_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58367bb2c7cad3127db0062d2e30e5762eb95c05fc9aca2e4eff2de1ef1792f1 +size 4143 diff --git a/Government/25k/en-ta/runs/Oct02_19-10-26_NLPC-Sita-MLM/events.out.tfevents.1759412426.NLPC-Sita-MLM.2581313.0 b/Government/25k/en-ta/runs/Oct02_19-10-26_NLPC-Sita-MLM/events.out.tfevents.1759412426.NLPC-Sita-MLM.2581313.0 new file mode 100644 index 0000000000000000000000000000000000000000..d1520978a636f8cbf46c9f4f1cbcaaa109d67b0a --- /dev/null +++ b/Government/25k/en-ta/runs/Oct02_19-10-26_NLPC-Sita-MLM/events.out.tfevents.1759412426.NLPC-Sita-MLM.2581313.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e7020b96837b84eb4d680ece899467ae22615928fda14356b106f416a4b760f +size 8014 diff --git a/Government/25k/en-ta_final_model/config.json b/Government/25k/en-ta_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/25k/en-ta_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/25k/en-ta_final_model/generation_config.json b/Government/25k/en-ta_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/25k/en-ta_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/25k/en-ta_final_model/pytorch_model.bin b/Government/25k/en-ta_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..f08fc2ef296d48903e6be187462340e8016e699d --- /dev/null +++ b/Government/25k/en-ta_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:926c99452be7a0b065781f79fb0d14d294751b2a60146f6e1c2ca015976b0b6e +size 2468870723 diff --git a/Government/25k/en-ta_final_model/training_args.bin b/Government/25k/en-ta_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..a6d1740984e93ef6141cf4d5e00496430a7084c6 --- /dev/null +++ b/Government/25k/en-ta_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a87cf9b7af34c944e73d25c3389ba6c05d5fde37f18957dca40fdb0e438f8980 +size 4143 diff --git a/Government/25k/nllb_bible_eval_government_en-si_trained_25k.csv b/Government/25k/nllb_bible_eval_government_en-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ade7a844cfab0f2076e20b1ece6e5bf67719403c --- /dev/null +++ b/Government/25k/nllb_bible_eval_government_en-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,34.456340875004464,29.91541262040702,2.927627634611363,16.043010632978344,16.549307363154742,0.7995621707439423 diff --git a/Government/25k/nllb_bible_eval_government_en-ta_trained_25k.csv b/Government/25k/nllb_bible_eval_government_en-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..c3ab644f25f3860a7be4d49356aab30c46eb9257 --- /dev/null +++ b/Government/25k/nllb_bible_eval_government_en-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,37.910253679604686,32.01692922260542,2.4064603779889935,12.961950550378779,11.953366145494822,0.7995613397806883 diff --git a/Government/25k/nllb_bible_eval_government_si-en_trained_25k.csv b/Government/25k/nllb_bible_eval_government_si-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..80f194ba15b83ecb201efc85220d6e76075cb487 --- /dev/null +++ b/Government/25k/nllb_bible_eval_government_si-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,45.44171048157175,44.52892482269925,21.98176525247642,23.20753380936823,22.94907765149802,0.774605580329895 diff --git a/Government/25k/nllb_bible_eval_government_ta-en_trained_25k.csv b/Government/25k/nllb_bible_eval_government_ta-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..d51cbec983c3abe6bf42bc6b38c1ba2a3f82c705 --- /dev/null +++ b/Government/25k/nllb_bible_eval_government_ta-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,38.65940473878675,37.649390484982774,15.81276195468595,17.58532704888579,17.004959992318227,0.7474953307807446 diff --git a/Government/25k/nllb_floresplus_eval_government_en-si_trained_25k.csv b/Government/25k/nllb_floresplus_eval_government_en-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b77940f697a0e39ba8c49326cdf08b74019830aa --- /dev/null +++ b/Government/25k/nllb_floresplus_eval_government_en-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,43.53881729774115,39.32453614210059,9.495034487221888,26.098641019137556,29.645824704158496,0.8745844934886623 diff --git a/Government/25k/nllb_floresplus_eval_government_en-ta_trained_25k.csv b/Government/25k/nllb_floresplus_eval_government_en-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b43b97ac3968fe7bf659953066835f7da4b723fe --- /dev/null +++ b/Government/25k/nllb_floresplus_eval_government_en-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,51.168533619903165,44.988690975617224,10.36678989899248,24.3163674774914,23.88797990096679,0.8646409833148534 diff --git a/Government/25k/nllb_floresplus_eval_government_si-en_trained_25k.csv b/Government/25k/nllb_floresplus_eval_government_si-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..28ff1defd940d0b6ddb7d5b11153c51d0c02dbe0 --- /dev/null +++ b/Government/25k/nllb_floresplus_eval_government_si-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,56.731496713185145,54.60594823245899,27.95596981666909,29.922833678434063,30.64866952047091,0.8645521375854968 diff --git a/Government/25k/nllb_floresplus_eval_government_si-ta_trained_25k.csv b/Government/25k/nllb_floresplus_eval_government_si-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..6e79a7523217b638c0b6c86b0db5bb619d18e881 --- /dev/null +++ b/Government/25k/nllb_floresplus_eval_government_si-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,46.21132146865297,40.03171949968911,6.851872356963542,18.500530348417875,18.308961611100344,0.8455016231348392 diff --git a/Government/25k/nllb_floresplus_eval_government_ta-en_trained_25k.csv b/Government/25k/nllb_floresplus_eval_government_ta-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..a1e67347efe057b6909be622629aba66edce8ee1 --- /dev/null +++ b/Government/25k/nllb_floresplus_eval_government_ta-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,55.05869435825216,52.89229749013958,26.56091591789399,28.275743167021506,29.247876665620574,0.845015874228223 diff --git a/Government/25k/nllb_floresplus_eval_government_ta-si_trained_25k.csv b/Government/25k/nllb_floresplus_eval_government_ta-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..8027547a3cf6690ca2f32cac9b8354ce24f3d483 --- /dev/null +++ b/Government/25k/nllb_floresplus_eval_government_ta-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,38.21677557606723,34.181725386295795,6.884206151628509,20.181455438227566,23.79556439497183,0.8404766332372846 diff --git a/Government/25k/nllb_government_eval_bible_en-si_trained_25k.csv b/Government/25k/nllb_government_eval_bible_en-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ade7a844cfab0f2076e20b1ece6e5bf67719403c --- /dev/null +++ b/Government/25k/nllb_government_eval_bible_en-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,34.456340875004464,29.91541262040702,2.927627634611363,16.043010632978344,16.549307363154742,0.7995621707439423 diff --git a/Government/25k/nllb_government_eval_bible_en-ta_trained_25k.csv b/Government/25k/nllb_government_eval_bible_en-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..c3ab644f25f3860a7be4d49356aab30c46eb9257 --- /dev/null +++ b/Government/25k/nllb_government_eval_bible_en-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,37.910253679604686,32.01692922260542,2.4064603779889935,12.961950550378779,11.953366145494822,0.7995613397806883 diff --git a/Government/25k/nllb_government_eval_bible_si-en_trained_25k.csv b/Government/25k/nllb_government_eval_bible_si-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..80f194ba15b83ecb201efc85220d6e76075cb487 --- /dev/null +++ b/Government/25k/nllb_government_eval_bible_si-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,45.44171048157175,44.52892482269925,21.98176525247642,23.20753380936823,22.94907765149802,0.774605580329895 diff --git a/Government/25k/nllb_government_eval_bible_ta-en_trained_25k.csv b/Government/25k/nllb_government_eval_bible_ta-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..d51cbec983c3abe6bf42bc6b38c1ba2a3f82c705 --- /dev/null +++ b/Government/25k/nllb_government_eval_bible_ta-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,38.65940473878675,37.649390484982774,15.81276195468595,17.58532704888579,17.004959992318227,0.7474953307807446 diff --git a/Government/25k/nllb_government_eval_en-si_trained_25k.csv b/Government/25k/nllb_government_eval_en-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..1d8725642948e62cf62f4710e6fa2bdf4546c716 --- /dev/null +++ b/Government/25k/nllb_government_eval_en-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,57.28053792873068,53.00412598137066,22.52790319592445,44.41770266772205,50.36404431511058,0.9223912889015913 diff --git a/Government/25k/nllb_government_eval_en-ta_trained_25k.csv b/Government/25k/nllb_government_eval_en-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..7b1577a946b859d00cbdbcfc76addb3785304394 --- /dev/null +++ b/Government/25k/nllb_government_eval_en-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,60.46204273966338,54.416082605781426,21.14259715548522,37.136839938510356,37.28464496174482,0.8991987895680081 diff --git a/Government/25k/nllb_government_eval_government_en-si_trained_25k.csv b/Government/25k/nllb_government_eval_government_en-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..2c0d399fc812e7c9bc523f2a6a6b77e633b13309 --- /dev/null +++ b/Government/25k/nllb_government_eval_government_en-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,57.77575685070945,53.38916478804416,22.913657969171457,44.63136119938347,50.79563281416712,0.9236653954982758 diff --git a/Government/25k/nllb_government_eval_government_en-ta_trained_25k.csv b/Government/25k/nllb_government_eval_government_en-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..4900be5a67864c7b985bfdf5ee1798cb1d65fee3 --- /dev/null +++ b/Government/25k/nllb_government_eval_government_en-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,61.33114874800577,55.22221826862441,22.221961258105324,38.15551868828684,38.25538259594189,0.9011772488355636 diff --git a/Government/25k/nllb_government_eval_government_si-en_trained_25k.csv b/Government/25k/nllb_government_eval_government_si-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..56e830c1fa38749e3da3d046e84d9ce188457b71 --- /dev/null +++ b/Government/25k/nllb_government_eval_government_si-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,64.58857041890641,61.87377282714578,37.248262998779424,39.46165914062977,40.52310463151705,0.8774390577673912 diff --git a/Government/25k/nllb_government_eval_government_si-ta_trained_25k.csv b/Government/25k/nllb_government_eval_government_si-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..39849dc56ea38629a1a7a6b156d193f3ad3ff82c --- /dev/null +++ b/Government/25k/nllb_government_eval_government_si-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,59.812702487886824,53.44480979898254,20.33552064617674,35.86678178644539,36.00773367568179,0.8959461055696011 diff --git a/Government/25k/nllb_government_eval_government_ta-en_trained_25k.csv b/Government/25k/nllb_government_eval_government_ta-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..c88738ecc47b3ce1c2f5777bdf3e0b3f4d48c754 --- /dev/null +++ b/Government/25k/nllb_government_eval_government_ta-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,64.04725087499365,61.42962540191179,37.09339503750719,39.02163223194449,39.98901600838739,0.8628479547798633 diff --git a/Government/25k/nllb_government_eval_government_ta-si_trained_25k.csv b/Government/25k/nllb_government_eval_government_ta-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..2505a7686490236d1e8528a8330d455adece7eda --- /dev/null +++ b/Government/25k/nllb_government_eval_government_ta-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,55.164204075164214,50.910692720283855,21.097118075257388,41.11269138796458,47.31720115936583,0.9045515403747558 diff --git a/Government/25k/nllb_government_eval_si-en_trained_25k.csv b/Government/25k/nllb_government_eval_si-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..e46b50c3372e8ea2ca4ed040c80289d44888aae0 --- /dev/null +++ b/Government/25k/nllb_government_eval_si-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,64.58525051167459,61.95056089809234,37.595643518788286,39.65906259231169,40.63070441064639,0.8765090019599394 diff --git a/Government/25k/nllb_government_eval_si-ta_trained_25k.csv b/Government/25k/nllb_government_eval_si-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..697bd7ce649874bdcf81ddc8df639c093702b008 --- /dev/null +++ b/Government/25k/nllb_government_eval_si-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,59.41702632151578,53.17598564145565,20.312903082044933,35.576149014657204,35.808295690838136,0.893279291858844 diff --git a/Government/25k/nllb_government_eval_ta-en_trained_25k.csv b/Government/25k/nllb_government_eval_ta-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..3ff5116de0ca839eb8c5723c062b080e20a2e9ac --- /dev/null +++ b/Government/25k/nllb_government_eval_ta-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,63.57670614520708,61.0046245044047,36.823592331355556,38.49781273522432,39.42112794513708,0.8604383393043817 diff --git a/Government/25k/nllb_government_eval_ta-si_trained_25k.csv b/Government/25k/nllb_government_eval_ta-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..0ca5ba437a7473878af79901e8ee91e304d81cd5 --- /dev/null +++ b/Government/25k/nllb_government_eval_ta-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,54.96755119613592,50.81371091140382,21.15392436002924,41.11194556390115,47.16236270405907,0.9026968573790678 diff --git a/Government/25k/nllb_health_eval_government_en-si_trained_25k.csv b/Government/25k/nllb_health_eval_government_en-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b403898f791f315edcf5f015732a3328da753d74 --- /dev/null +++ b/Government/25k/nllb_health_eval_government_en-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,43.25096129715229,38.92367941876307,10.935473968452124,26.883374289624214,29.927259118058668,0.8349770524361161 diff --git a/Government/25k/nllb_health_eval_government_en-ta_trained_25k.csv b/Government/25k/nllb_health_eval_government_en-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..e381fb113831463a0c82cfff0d00d389b9ac7f4a --- /dev/null +++ b/Government/25k/nllb_health_eval_government_en-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,57.770063459033786,51.13982650874356,13.649049880812218,30.540986543530195,29.445217779048033,0.8380835425704718 diff --git a/Government/25k/nllb_health_eval_government_si-en_trained_25k.csv b/Government/25k/nllb_health_eval_government_si-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..2363f9eb28f605c4c2fc3b610a77a28cf22e0eb7 --- /dev/null +++ b/Government/25k/nllb_health_eval_government_si-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,57.212120300084024,54.15630021537241,23.175948602163185,28.811125019591866,29.158821803333304,0.8477959724289648 diff --git a/Government/25k/nllb_health_eval_government_si-ta_trained_25k.csv b/Government/25k/nllb_health_eval_government_si-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..38cb92ba8d3530a3c83c0e366d44ad1ed5cab58a --- /dev/null +++ b/Government/25k/nllb_health_eval_government_si-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,50.329279921202954,43.48567558700648,7.346455770868597,21.272291935038847,20.46408102851401,0.8077254467168633 diff --git a/Government/25k/nllb_health_eval_government_ta-en_trained_25k.csv b/Government/25k/nllb_health_eval_government_ta-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..dc71912c3871cf8416afaeea9206b6a5f832f094 --- /dev/null +++ b/Government/25k/nllb_health_eval_government_ta-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,58.890632616108704,56.16350397751192,28.459078151482686,31.653800864205223,32.43401341399923,0.8308236742317676 diff --git a/Government/25k/nllb_health_eval_government_ta-si_trained_25k.csv b/Government/25k/nllb_health_eval_government_ta-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..73ac8a42833bddf5069c4a07af01ce68efdfccfa --- /dev/null +++ b/Government/25k/nllb_health_eval_government_ta-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,37.12234504381412,33.219539031380336,7.841096632808772,19.806591952411605,22.352163119496407,0.7895335042659117 diff --git a/Government/25k/nllb_jailbreak_eval_government_en-si_trained_25k.csv b/Government/25k/nllb_jailbreak_eval_government_en-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..cc5f9498944170ab16881592532ddc0673d5e845 --- /dev/null +++ b/Government/25k/nllb_jailbreak_eval_government_en-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,41.94530390315914,38.36738795492778,12.134369099259992,22.955178169349796,25.91187375950136,0.8430828390717506 diff --git a/Government/25k/nllb_jailbreak_eval_government_en-ta_trained_25k.csv b/Government/25k/nllb_jailbreak_eval_government_en-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..c4d99de6f7a685ad188d0236f4b6954790ce1571 --- /dev/null +++ b/Government/25k/nllb_jailbreak_eval_government_en-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,56.147013776750676,51.156393091774014,18.472593105129725,31.029411447858458,29.72512132689668,0.8519543393552304 diff --git a/Government/25k/nllb_jailbreak_eval_government_si-en_trained_25k.csv b/Government/25k/nllb_jailbreak_eval_government_si-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..94f48c8e2e0069ec44263f1b8b966f0d5d514aaf --- /dev/null +++ b/Government/25k/nllb_jailbreak_eval_government_si-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,53.88183375650033,52.08565740531584,28.428531036040155,28.957961328026386,29.46748347959513,0.8550191217660904 diff --git a/Government/25k/nllb_jailbreak_eval_government_si-ta_trained_25k.csv b/Government/25k/nllb_jailbreak_eval_government_si-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b31a359f958c3af879e95582bdc9a06e266d1f92 --- /dev/null +++ b/Government/25k/nllb_jailbreak_eval_government_si-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,50.37771910111668,45.07576589162182,12.36394486911458,22.643750549860602,21.71292236539571,0.8344147505164147 diff --git a/Government/25k/nllb_jailbreak_eval_government_ta-en_trained_25k.csv b/Government/25k/nllb_jailbreak_eval_government_ta-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..d3175db27b45b01cede2d28db9655d110ef8dcea --- /dev/null +++ b/Government/25k/nllb_jailbreak_eval_government_ta-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,54.31364825927708,53.049530758905064,31.48795173143003,29.637699578882415,30.300190451925477,0.840070496827364 diff --git a/Government/25k/nllb_jailbreak_eval_government_ta-si_trained_25k.csv b/Government/25k/nllb_jailbreak_eval_government_ta-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..f857ed0f7ad538981553d79564f6bac38d332c16 --- /dev/null +++ b/Government/25k/nllb_jailbreak_eval_government_ta-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,37.32610532361566,34.27432287596386,10.97646499413233,19.120775618412832,21.580127054022135,0.8094877264499665 diff --git a/Government/25k/nllb_literature-children_eval_government_en-si_trained_25k.csv b/Government/25k/nllb_literature-children_eval_government_en-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b49aba327a772f1d2bf347e607e27de328bcb909 --- /dev/null +++ b/Government/25k/nllb_literature-children_eval_government_en-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,30.537553612782077,27.457684801211464,5.352300825495501,11.776487516391139,13.60866615715853,0.8180207158476114 diff --git a/Government/25k/nllb_literature-children_eval_government_en-ta_trained_25k.csv b/Government/25k/nllb_literature-children_eval_government_en-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..415804feefe9a14b9309e00d2464753f0e8c5783 --- /dev/null +++ b/Government/25k/nllb_literature-children_eval_government_en-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,52.94294062702861,47.48633204257806,12.65209411669922,25.921457143150043,25.42314085711707,0.8307426627380429 diff --git a/Government/25k/nllb_literature-children_eval_government_si-en_trained_25k.csv b/Government/25k/nllb_literature-children_eval_government_si-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..4e7d086580bd987f559c960778c5661795661c01 --- /dev/null +++ b/Government/25k/nllb_literature-children_eval_government_si-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,43.328827400718104,42.21937887996792,18.677081635943036,18.580052705829587,18.800094198809916,0.8038805679678916 diff --git a/Government/25k/nllb_literature-children_eval_government_si-ta_trained_25k.csv b/Government/25k/nllb_literature-children_eval_government_si-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..d4e7f5e5517d977ad555d73a403311c6e40207c3 --- /dev/null +++ b/Government/25k/nllb_literature-children_eval_government_si-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,44.032299218428186,38.542971823818256,6.216689408046166,15.355083659335547,14.624086944787537,0.8002112233466621 diff --git a/Government/25k/nllb_literature-children_eval_government_ta-en_trained_25k.csv b/Government/25k/nllb_literature-children_eval_government_ta-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b3f703393520a787ff9800076adfe393f0e1c079 --- /dev/null +++ b/Government/25k/nllb_literature-children_eval_government_ta-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,50.391050771844135,49.58735372946614,26.358126611234137,25.747113279621463,26.223809152734894,0.8133344068709347 diff --git a/Government/25k/nllb_literature-children_eval_government_ta-si_trained_25k.csv b/Government/25k/nllb_literature-children_eval_government_ta-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ff5b0428930f27ad27d366980622af3bff1bf28a --- /dev/null +++ b/Government/25k/nllb_literature-children_eval_government_ta-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,30.04457413782446,26.872293760071443,3.9983893966060684,10.3177642268497,12.016513765533656,0.7933493303191074 diff --git a/Government/25k/nllb_literature-kathlyn_eval_government_en-si_trained_25k.csv b/Government/25k/nllb_literature-kathlyn_eval_government_en-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b49aba327a772f1d2bf347e607e27de328bcb909 --- /dev/null +++ b/Government/25k/nllb_literature-kathlyn_eval_government_en-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,30.537553612782077,27.457684801211464,5.352300825495501,11.776487516391139,13.60866615715853,0.8180207158476114 diff --git a/Government/25k/nllb_literature-kathlyn_eval_government_en-ta_trained_25k.csv b/Government/25k/nllb_literature-kathlyn_eval_government_en-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..415804feefe9a14b9309e00d2464753f0e8c5783 --- /dev/null +++ b/Government/25k/nllb_literature-kathlyn_eval_government_en-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,52.94294062702861,47.48633204257806,12.65209411669922,25.921457143150043,25.42314085711707,0.8307426627380429 diff --git a/Government/25k/nllb_literature-kathlyn_eval_government_si-en_trained_25k.csv b/Government/25k/nllb_literature-kathlyn_eval_government_si-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..4e7d086580bd987f559c960778c5661795661c01 --- /dev/null +++ b/Government/25k/nllb_literature-kathlyn_eval_government_si-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,43.328827400718104,42.21937887996792,18.677081635943036,18.580052705829587,18.800094198809916,0.8038805679678916 diff --git a/Government/25k/nllb_literature-kathlyn_eval_government_si-ta_trained_25k.csv b/Government/25k/nllb_literature-kathlyn_eval_government_si-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..d4e7f5e5517d977ad555d73a403311c6e40207c3 --- /dev/null +++ b/Government/25k/nllb_literature-kathlyn_eval_government_si-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,44.032299218428186,38.542971823818256,6.216689408046166,15.355083659335547,14.624086944787537,0.8002112233466621 diff --git a/Government/25k/nllb_literature-kathlyn_eval_government_ta-en_trained_25k.csv b/Government/25k/nllb_literature-kathlyn_eval_government_ta-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b3f703393520a787ff9800076adfe393f0e1c079 --- /dev/null +++ b/Government/25k/nllb_literature-kathlyn_eval_government_ta-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,50.391050771844135,49.58735372946614,26.358126611234137,25.747113279621463,26.223809152734894,0.8133344068709347 diff --git a/Government/25k/nllb_literature-kathlyn_eval_government_ta-si_trained_25k.csv b/Government/25k/nllb_literature-kathlyn_eval_government_ta-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ff5b0428930f27ad27d366980622af3bff1bf28a --- /dev/null +++ b/Government/25k/nllb_literature-kathlyn_eval_government_ta-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,30.04457413782446,26.872293760071443,3.9983893966060684,10.3177642268497,12.016513765533656,0.7933493303191074 diff --git a/Government/25k/nllb_literature-old_eval_government_en-si_trained_25k.csv b/Government/25k/nllb_literature-old_eval_government_en-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b49aba327a772f1d2bf347e607e27de328bcb909 --- /dev/null +++ b/Government/25k/nllb_literature-old_eval_government_en-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,30.537553612782077,27.457684801211464,5.352300825495501,11.776487516391139,13.60866615715853,0.8180207158476114 diff --git a/Government/25k/nllb_literature-old_eval_government_en-ta_trained_25k.csv b/Government/25k/nllb_literature-old_eval_government_en-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..415804feefe9a14b9309e00d2464753f0e8c5783 --- /dev/null +++ b/Government/25k/nllb_literature-old_eval_government_en-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,52.94294062702861,47.48633204257806,12.65209411669922,25.921457143150043,25.42314085711707,0.8307426627380429 diff --git a/Government/25k/nllb_literature-old_eval_government_si-en_trained_25k.csv b/Government/25k/nllb_literature-old_eval_government_si-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..4e7d086580bd987f559c960778c5661795661c01 --- /dev/null +++ b/Government/25k/nllb_literature-old_eval_government_si-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,43.328827400718104,42.21937887996792,18.677081635943036,18.580052705829587,18.800094198809916,0.8038805679678916 diff --git a/Government/25k/nllb_literature-old_eval_government_si-ta_trained_25k.csv b/Government/25k/nllb_literature-old_eval_government_si-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..d4e7f5e5517d977ad555d73a403311c6e40207c3 --- /dev/null +++ b/Government/25k/nllb_literature-old_eval_government_si-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,44.032299218428186,38.542971823818256,6.216689408046166,15.355083659335547,14.624086944787537,0.8002112233466621 diff --git a/Government/25k/nllb_literature-old_eval_government_ta-en_trained_25k.csv b/Government/25k/nllb_literature-old_eval_government_ta-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b3f703393520a787ff9800076adfe393f0e1c079 --- /dev/null +++ b/Government/25k/nllb_literature-old_eval_government_ta-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,50.391050771844135,49.58735372946614,26.358126611234137,25.747113279621463,26.223809152734894,0.8133344068709347 diff --git a/Government/25k/nllb_literature-old_eval_government_ta-si_trained_25k.csv b/Government/25k/nllb_literature-old_eval_government_ta-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ff5b0428930f27ad27d366980622af3bff1bf28a --- /dev/null +++ b/Government/25k/nllb_literature-old_eval_government_ta-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,30.04457413782446,26.872293760071443,3.9983893966060684,10.3177642268497,12.016513765533656,0.7933493303191074 diff --git a/Government/25k/nllb_literature_eval_government_en-si_trained_25k.csv b/Government/25k/nllb_literature_eval_government_en-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b49aba327a772f1d2bf347e607e27de328bcb909 --- /dev/null +++ b/Government/25k/nllb_literature_eval_government_en-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,30.537553612782077,27.457684801211464,5.352300825495501,11.776487516391139,13.60866615715853,0.8180207158476114 diff --git a/Government/25k/nllb_literature_eval_government_en-ta_trained_25k.csv b/Government/25k/nllb_literature_eval_government_en-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..415804feefe9a14b9309e00d2464753f0e8c5783 --- /dev/null +++ b/Government/25k/nllb_literature_eval_government_en-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,52.94294062702861,47.48633204257806,12.65209411669922,25.921457143150043,25.42314085711707,0.8307426627380429 diff --git a/Government/25k/nllb_literature_eval_government_si-en_trained_25k.csv b/Government/25k/nllb_literature_eval_government_si-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..4e7d086580bd987f559c960778c5661795661c01 --- /dev/null +++ b/Government/25k/nllb_literature_eval_government_si-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,43.328827400718104,42.21937887996792,18.677081635943036,18.580052705829587,18.800094198809916,0.8038805679678916 diff --git a/Government/25k/nllb_literature_eval_government_si-ta_trained_25k.csv b/Government/25k/nllb_literature_eval_government_si-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..d4e7f5e5517d977ad555d73a403311c6e40207c3 --- /dev/null +++ b/Government/25k/nllb_literature_eval_government_si-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,44.032299218428186,38.542971823818256,6.216689408046166,15.355083659335547,14.624086944787537,0.8002112233466621 diff --git a/Government/25k/nllb_literature_eval_government_ta-en_trained_25k.csv b/Government/25k/nllb_literature_eval_government_ta-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b3f703393520a787ff9800076adfe393f0e1c079 --- /dev/null +++ b/Government/25k/nllb_literature_eval_government_ta-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,50.391050771844135,49.58735372946614,26.358126611234137,25.747113279621463,26.223809152734894,0.8133344068709347 diff --git a/Government/25k/nllb_literature_eval_government_ta-si_trained_25k.csv b/Government/25k/nllb_literature_eval_government_ta-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ff5b0428930f27ad27d366980622af3bff1bf28a --- /dev/null +++ b/Government/25k/nllb_literature_eval_government_ta-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,30.04457413782446,26.872293760071443,3.9983893966060684,10.3177642268497,12.016513765533656,0.7933493303191074 diff --git a/Government/25k/nllb_maths_eval_government_en-si_trained_25k.csv b/Government/25k/nllb_maths_eval_government_en-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..a52451ed501bf9a99360ff3d862a074606273c78 --- /dev/null +++ b/Government/25k/nllb_maths_eval_government_en-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,40.1331593949345,37.9749405806341,16.160581021507667,25.310463847844968,27.7360906142369,0.8429196584969759 diff --git a/Government/25k/nllb_maths_eval_government_en-ta_trained_25k.csv b/Government/25k/nllb_maths_eval_government_en-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..baea6b7bf531396ad471743aa7c04bb6345753ce --- /dev/null +++ b/Government/25k/nllb_maths_eval_government_en-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,50.43671068611862,46.27008156145329,19.451039210384923,25.950664084935074,25.355787433334278,0.8351800081133842 diff --git a/Government/25k/nllb_maths_eval_government_si-en_trained_25k.csv b/Government/25k/nllb_maths_eval_government_si-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..cad20dacbf38f33a5155be7f4a0f7f02cee7a6c8 --- /dev/null +++ b/Government/25k/nllb_maths_eval_government_si-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,57.055757703519596,56.463857046699296,37.12052800347161,37.13924306449029,37.77450764408659,0.8438830208182335 diff --git a/Government/25k/nllb_maths_eval_government_ta-en_trained_25k.csv b/Government/25k/nllb_maths_eval_government_ta-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..c1bb6bfbca8bdab106bdf0b45ed71d912bda63db --- /dev/null +++ b/Government/25k/nllb_maths_eval_government_ta-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,53.866656257456036,53.156347000921436,34.00862873558283,33.6738715465911,34.46886136923999,0.8329405805170536 diff --git a/Government/25k/nllb_news_eval_government_en-si_trained_25k.csv b/Government/25k/nllb_news_eval_government_en-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..111ded500571d68fa6795cf1b8dd1de59d6fb25e --- /dev/null +++ b/Government/25k/nllb_news_eval_government_en-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,45.252665838427404,40.82924968374386,11.472613916243573,27.13425842456199,32.92937573986284,0.8772459758222103 diff --git a/Government/25k/nllb_news_eval_government_en-ta_trained_25k.csv b/Government/25k/nllb_news_eval_government_en-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..a1dff158cba029545602189e5cdc9ee562b99f1d --- /dev/null +++ b/Government/25k/nllb_news_eval_government_en-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,57.96189383266107,52.355247567462946,16.7353650809541,33.13264694861089,32.75953716452201,0.865638548770104 diff --git a/Government/25k/nllb_news_eval_government_si-en_trained_25k.csv b/Government/25k/nllb_news_eval_government_si-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..d2ef5eee8eaf22aa1edf930657d6998a07fe3a59 --- /dev/null +++ b/Government/25k/nllb_news_eval_government_si-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,55.619649765821144,52.95920751394399,23.961089154486878,26.647218304162454,27.498005055406196,0.8542312571704388 diff --git a/Government/25k/nllb_news_eval_government_si-ta_trained_25k.csv b/Government/25k/nllb_news_eval_government_si-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..0e27c17199ea84ad29072369b86a98a335b0bbda --- /dev/null +++ b/Government/25k/nllb_news_eval_government_si-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,49.78500522186812,43.7689835650637,8.20206190883595,21.166895541021454,21.231448498492565,0.8485575679299091 diff --git a/Government/25k/nllb_news_eval_government_ta-en_trained_25k.csv b/Government/25k/nllb_news_eval_government_ta-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..e8b9d2bf59a9c73595d4f5c3e1ac810d9b24bffa --- /dev/null +++ b/Government/25k/nllb_news_eval_government_ta-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,59.55414861647426,57.74703304137459,32.094170005209584,33.25771474480451,34.33577540413442,0.8558003563119705 diff --git a/Government/25k/nllb_news_eval_government_ta-si_trained_25k.csv b/Government/25k/nllb_news_eval_government_ta-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..18420e79009d9f14d612d728af28d043436e04e7 --- /dev/null +++ b/Government/25k/nllb_news_eval_government_ta-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,37.26994610605991,33.210986519051716,6.6850183927221325,19.603112341323836,23.067051210855503,0.822933440081655 diff --git a/Government/25k/nllb_nllb_eval_government_en-si_trained_25k.csv b/Government/25k/nllb_nllb_eval_government_en-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..309b58adc6dce8f032099f108956619c2be049e6 --- /dev/null +++ b/Government/25k/nllb_nllb_eval_government_en-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,42.319887007329434,38.80384792742494,10.908557257224446,23.233545972292767,27.473380399198323,0.901711478390458 diff --git a/Government/25k/nllb_nllb_eval_government_en-ta_trained_25k.csv b/Government/25k/nllb_nllb_eval_government_en-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..c0debb955f2578eff90ba5736bf24fa49faf423c --- /dev/null +++ b/Government/25k/nllb_nllb_eval_government_en-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,55.89077974807722,50.92405687648808,16.77937992038363,29.683968294312578,28.96596565900932,0.9056717679016125 diff --git a/Government/25k/nllb_nllb_eval_government_si-en_trained_25k.csv b/Government/25k/nllb_nllb_eval_government_si-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..bc37cea7e0a3ba77b9353625a2a800b7bba38123 --- /dev/null +++ b/Government/25k/nllb_nllb_eval_government_si-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,61.81986238385338,61.0575686551013,40.87429966556002,40.70235601434973,40.87055641010563,0.8976878566082513 diff --git a/Government/25k/nllb_nllb_eval_government_si-ta_trained_25k.csv b/Government/25k/nllb_nllb_eval_government_si-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..0253f62b5d3364f0c63d9c7d5dfbb707a0ca11bd --- /dev/null +++ b/Government/25k/nllb_nllb_eval_government_si-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,53.72629202724089,48.57216552361423,13.806014017060688,26.541412165796235,25.46648470378095,0.8990643055392573 diff --git a/Government/25k/nllb_nllb_eval_government_ta-en_trained_25k.csv b/Government/25k/nllb_nllb_eval_government_ta-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..da21ecff7ccde4c7a47ef67432830c5e6ca7f479 --- /dev/null +++ b/Government/25k/nllb_nllb_eval_government_ta-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,62.606355219389755,61.935482023287214,42.00175990650263,41.943221614836965,42.1864893150037,0.8881275742368226 diff --git a/Government/25k/nllb_nllb_eval_government_ta-si_trained_25k.csv b/Government/25k/nllb_nllb_eval_government_ta-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..de9270cf93ba1391ee26e1df6df416d38076b087 --- /dev/null +++ b/Government/25k/nllb_nllb_eval_government_ta-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,40.78127075286322,37.45895860642715,10.199632950644968,21.514486477221592,25.243830966859765,0.8861108106996676 diff --git a/Government/25k/nllb_opensubtitles_eval_government_en-si_trained_25k.csv b/Government/25k/nllb_opensubtitles_eval_government_en-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..d2721916d331c90e4ebc0936109f4e768c2cef66 --- /dev/null +++ b/Government/25k/nllb_opensubtitles_eval_government_en-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,34.74392924639648,31.722146731556744,7.22518423376036,19.630955821278032,21.327917510045417,0.8652439583539963 diff --git a/Government/25k/nllb_opensubtitles_eval_government_en-ta_trained_25k.csv b/Government/25k/nllb_opensubtitles_eval_government_en-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ff766a151235b0749e94aea6c62ac8e9997067f7 --- /dev/null +++ b/Government/25k/nllb_opensubtitles_eval_government_en-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,49.96828470037169,46.0653176059842,16.388551708689157,27.76078544768291,27.583351063247836,0.859308678522706 diff --git a/Government/25k/nllb_opensubtitles_eval_government_si-en_trained_25k.csv b/Government/25k/nllb_opensubtitles_eval_government_si-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..06a5e75149d9bb571a9cf0279ccf1d8e6b6e8bd3 --- /dev/null +++ b/Government/25k/nllb_opensubtitles_eval_government_si-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,62.28766815442771,61.51414472315769,39.702805722647234,40.72100110310309,41.010220117202856,0.8896081002503633 diff --git a/Government/25k/nllb_opensubtitles_eval_government_si-ta_trained_25k.csv b/Government/25k/nllb_opensubtitles_eval_government_si-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..eac35b599db066101c219e8165cd5e440cceaa4d --- /dev/null +++ b/Government/25k/nllb_opensubtitles_eval_government_si-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,53.1108460847585,48.70099406890239,16.50568694824706,27.93487100441276,27.014044900074236,0.8846168394088745 diff --git a/Government/25k/nllb_opensubtitles_eval_government_ta-en_trained_25k.csv b/Government/25k/nllb_opensubtitles_eval_government_ta-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..021e57fadd781f7693dac425f1a85d650466a6d1 --- /dev/null +++ b/Government/25k/nllb_opensubtitles_eval_government_ta-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,59.519555151016256,58.830867638406104,36.621497819485775,37.49370983310459,37.777534996640874,0.8696107387244701 diff --git a/Government/25k/nllb_opensubtitles_eval_government_ta-si_trained_25k.csv b/Government/25k/nllb_opensubtitles_eval_government_ta-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..2ee211b60c8503902e152bb3103469b60ecfb1f4 --- /dev/null +++ b/Government/25k/nllb_opensubtitles_eval_government_ta-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,32.115498882583545,29.0352322424891,5.715035714226997,16.597641369098486,18.362433692058854,0.8535309534668922 diff --git a/Government/25k/nllb_wikipedia_eval_government_en-si_trained_25k.csv b/Government/25k/nllb_wikipedia_eval_government_en-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..fc0701764cbc9b9df1738f5060e2e2f62857a859 --- /dev/null +++ b/Government/25k/nllb_wikipedia_eval_government_en-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,42.11352299493802,37.60982662762706,7.471102785714683,23.41732987399879,26.992666872334663,0.8773687633275986 diff --git a/Government/25k/nllb_wikipedia_eval_government_en-ta_trained_25k.csv b/Government/25k/nllb_wikipedia_eval_government_en-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..197cc19889d195ce5933f5729e8c972729392192 --- /dev/null +++ b/Government/25k/nllb_wikipedia_eval_government_en-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,54.66213735077503,48.341725778921536,11.891592515162682,28.933939483948446,28.79674392890101,0.8784477740079164 diff --git a/Government/25k/nllb_wikipedia_eval_government_si-en_trained_25k.csv b/Government/25k/nllb_wikipedia_eval_government_si-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..f7dbcd4b11dea04dd25da6bea4b0f95e93d36134 --- /dev/null +++ b/Government/25k/nllb_wikipedia_eval_government_si-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,54.33205209749793,52.23850858529763,25.520181497053553,28.6899226315677,29.227820042803735,0.843238550812006 diff --git a/Government/25k/nllb_wikipedia_eval_government_si-ta_trained_25k.csv b/Government/25k/nllb_wikipedia_eval_government_si-ta_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..0b0cc984700e11703353f0fc45d921f76a7d2a87 --- /dev/null +++ b/Government/25k/nllb_wikipedia_eval_government_si-ta_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,47.47793495852912,41.258516450053776,7.22054423551598,21.45037334491328,21.22070792686838,0.8479302596896887 diff --git a/Government/25k/nllb_wikipedia_eval_government_ta-en_trained_25k.csv b/Government/25k/nllb_wikipedia_eval_government_ta-en_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..55bb9cb40406756fab5f6a3a338d5a9c587189d0 --- /dev/null +++ b/Government/25k/nllb_wikipedia_eval_government_ta-en_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,58.63879617725423,56.64403548938728,29.95204599338054,33.153971722664544,33.74928818585589,0.8460434170663357 diff --git a/Government/25k/nllb_wikipedia_eval_government_ta-si_trained_25k.csv b/Government/25k/nllb_wikipedia_eval_government_ta-si_trained_25k.csv new file mode 100644 index 0000000000000000000000000000000000000000..2ed7a9e182d00e75a2239807bbb31c8c88f293d4 --- /dev/null +++ b/Government/25k/nllb_wikipedia_eval_government_ta-si_trained_25k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,38.24462187210634,34.101542942234126,6.004275881201997,19.309101197122633,22.61815711724994,0.8478409900069237 diff --git a/Government/25k/si-en/runs/Oct02_21-37-11_NLPC-Sita-MLM/events.out.tfevents.1759421231.NLPC-Sita-MLM.2581313.6 b/Government/25k/si-en/runs/Oct02_21-37-11_NLPC-Sita-MLM/events.out.tfevents.1759421231.NLPC-Sita-MLM.2581313.6 new file mode 100644 index 0000000000000000000000000000000000000000..a941b7ba4b4d5d7336b6dc1bb5f65b83ae8325ea --- /dev/null +++ b/Government/25k/si-en/runs/Oct02_21-37-11_NLPC-Sita-MLM/events.out.tfevents.1759421231.NLPC-Sita-MLM.2581313.6 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de10b95c2b1defceb8ca1a6c2cc7257578deea79949632ab08d28531cfc46a9e +size 8014 diff --git a/Government/25k/si-en_final_model/config.json b/Government/25k/si-en_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/25k/si-en_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/25k/si-en_final_model/generation_config.json b/Government/25k/si-en_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/25k/si-en_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/25k/si-en_final_model/pytorch_model.bin b/Government/25k/si-en_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..bd5dfc795177b94b6b33939c16ce7625192cf45a --- /dev/null +++ b/Government/25k/si-en_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e7fdc6741948dfd3a4ddd5bfd6be67d37fcc1889b2ca5d868fa524c5eb8a50c +size 2468870723 diff --git a/Government/25k/si-en_final_model/training_args.bin b/Government/25k/si-en_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..d5dfd15b570c25d750e64d302dcd69b794f25baf --- /dev/null +++ b/Government/25k/si-en_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16fbaf112e69ffd66ba71700f336cece47b4d87b306e03fc496d72154b8dd64d +size 4143 diff --git a/Government/25k/si-ta/runs/Oct03_02-20-10_NLPC-Sita-MLM/events.out.tfevents.1759438210.NLPC-Sita-MLM.2581313.18 b/Government/25k/si-ta/runs/Oct03_02-20-10_NLPC-Sita-MLM/events.out.tfevents.1759438210.NLPC-Sita-MLM.2581313.18 new file mode 100644 index 0000000000000000000000000000000000000000..33951992fe7f7efef3ca8d1ff5ed6d58db72d6ce --- /dev/null +++ b/Government/25k/si-ta/runs/Oct03_02-20-10_NLPC-Sita-MLM/events.out.tfevents.1759438210.NLPC-Sita-MLM.2581313.18 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1313b6ba8cbfa55d39e3b91ca46cf88b28b31480f218aac55e3cc93ac77ecbad +size 8014 diff --git a/Government/25k/si-ta_final_model/config.json b/Government/25k/si-ta_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/25k/si-ta_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/25k/si-ta_final_model/generation_config.json b/Government/25k/si-ta_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/25k/si-ta_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/25k/si-ta_final_model/pytorch_model.bin b/Government/25k/si-ta_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..327800a0ae7b06aedead634799b552d8eb826ece --- /dev/null +++ b/Government/25k/si-ta_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f83dacb4539d797a3bb5c51042963ecacaba21d11adea3f6210a102ddd14456c +size 2468870723 diff --git a/Government/25k/si-ta_final_model/training_args.bin b/Government/25k/si-ta_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..79bd3b2857c1dfb9f0d0cb4363595a49d35d8ebe --- /dev/null +++ b/Government/25k/si-ta_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3829a38db808d0b4d26aa28435419a734c757d38888f4ffcf989f7cd1a24e817 +size 4143 diff --git a/Government/25k/ta-en/runs/Oct02_23-58-38_NLPC-Sita-MLM/events.out.tfevents.1759429719.NLPC-Sita-MLM.2581313.12 b/Government/25k/ta-en/runs/Oct02_23-58-38_NLPC-Sita-MLM/events.out.tfevents.1759429719.NLPC-Sita-MLM.2581313.12 new file mode 100644 index 0000000000000000000000000000000000000000..20300f6a3d521091c3eaf7197b572ffc03314358 --- /dev/null +++ b/Government/25k/ta-en/runs/Oct02_23-58-38_NLPC-Sita-MLM/events.out.tfevents.1759429719.NLPC-Sita-MLM.2581313.12 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1e3153e14db257b93475277849130238e39a61ee90adabe0e1704434d06a346 +size 8014 diff --git a/Government/25k/ta-en_final_model/config.json b/Government/25k/ta-en_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/25k/ta-en_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/25k/ta-en_final_model/generation_config.json b/Government/25k/ta-en_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/25k/ta-en_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/25k/ta-en_final_model/pytorch_model.bin b/Government/25k/ta-en_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..08ab0e43c2ff5b0dbb5d6d6ef6216cb98d037e20 --- /dev/null +++ b/Government/25k/ta-en_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb6ec0209c67142c32d3d55c407da5e63c50a8796a3dbcda169614153ad8d737 +size 2468870723 diff --git a/Government/25k/ta-en_final_model/training_args.bin b/Government/25k/ta-en_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..5197c1876af0be2d1919e916519c6d138f0b957e --- /dev/null +++ b/Government/25k/ta-en_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bc9250a64c03735ba575fddc99da28f11376e930a06a1ac04f9c629d27150ac +size 4143 diff --git a/Government/25k/ta-si/runs/Oct03_06-51-47_NLPC-Sita-MLM/events.out.tfevents.1759454507.NLPC-Sita-MLM.2581313.24 b/Government/25k/ta-si/runs/Oct03_06-51-47_NLPC-Sita-MLM/events.out.tfevents.1759454507.NLPC-Sita-MLM.2581313.24 new file mode 100644 index 0000000000000000000000000000000000000000..dba5d2ea1db4ca00837ed3adc14ae516750a64d9 --- /dev/null +++ b/Government/25k/ta-si/runs/Oct03_06-51-47_NLPC-Sita-MLM/events.out.tfevents.1759454507.NLPC-Sita-MLM.2581313.24 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5a452271b9c11bbd5f95b3169ce0b5870bdec80256213e7b8c7e4524c54ed1d +size 8014 diff --git a/Government/25k/ta-si_final_model/config.json b/Government/25k/ta-si_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/25k/ta-si_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/25k/ta-si_final_model/generation_config.json b/Government/25k/ta-si_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/25k/ta-si_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/25k/ta-si_final_model/pytorch_model.bin b/Government/25k/ta-si_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..fe9e742e61d352f2841154031ecd703234de7788 --- /dev/null +++ b/Government/25k/ta-si_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e392ae85b71bc82c32088666663c0e80d285a0e7d0242278c33ec6ee5881c52 +size 2468870723 diff --git a/Government/25k/ta-si_final_model/training_args.bin b/Government/25k/ta-si_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..57963ad8dc632468343bf9f3217859b10963aef8 --- /dev/null +++ b/Government/25k/ta-si_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50bc61bbeae001088483d2db11beb61ac724cff21475c01ba112256908ba4c9d +size 4143 diff --git a/Government/25k/tok-en-si/sentencepiece.bpe.model b/Government/25k/tok-en-si/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/25k/tok-en-si/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/25k/tok-en-si/special_tokens_map.json b/Government/25k/tok-en-si/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/25k/tok-en-si/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/25k/tok-en-si/tokenizer_config.json b/Government/25k/tok-en-si/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fd1139979a7c7c4188b26d922af637f4bfb23e1f --- /dev/null +++ b/Government/25k/tok-en-si/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "eng_Latn", + "tgt_lang": "sin_Sinh", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/25k/tok-en-ta/sentencepiece.bpe.model b/Government/25k/tok-en-ta/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/25k/tok-en-ta/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/25k/tok-en-ta/special_tokens_map.json b/Government/25k/tok-en-ta/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/25k/tok-en-ta/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/25k/tok-en-ta/tokenizer_config.json b/Government/25k/tok-en-ta/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ef8f263366a29cf3dd0c97df4b45c41fbbf160b5 --- /dev/null +++ b/Government/25k/tok-en-ta/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "eng_Latn", + "tgt_lang": "tam_Taml", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/25k/tok-si-en/sentencepiece.bpe.model b/Government/25k/tok-si-en/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/25k/tok-si-en/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/25k/tok-si-en/special_tokens_map.json b/Government/25k/tok-si-en/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/25k/tok-si-en/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/25k/tok-si-en/tokenizer_config.json b/Government/25k/tok-si-en/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f80b41ef163bc0a47ebef66aa6c1210dfe0b9792 --- /dev/null +++ b/Government/25k/tok-si-en/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "sin_Sinh", + "tgt_lang": "eng_Latn", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/25k/tok-si-ta/sentencepiece.bpe.model b/Government/25k/tok-si-ta/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/25k/tok-si-ta/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/25k/tok-si-ta/special_tokens_map.json b/Government/25k/tok-si-ta/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/25k/tok-si-ta/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/25k/tok-si-ta/tokenizer_config.json b/Government/25k/tok-si-ta/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..386158eef85de789e966861e91bebf69e9a4f32b --- /dev/null +++ b/Government/25k/tok-si-ta/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "sin_Sinh", + "tgt_lang": "tam_Taml", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/25k/tok-ta-en/sentencepiece.bpe.model b/Government/25k/tok-ta-en/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/25k/tok-ta-en/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/25k/tok-ta-en/special_tokens_map.json b/Government/25k/tok-ta-en/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/25k/tok-ta-en/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/25k/tok-ta-en/tokenizer_config.json b/Government/25k/tok-ta-en/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6deaa799386cb7438bf8a0f96250f961c73dec30 --- /dev/null +++ b/Government/25k/tok-ta-en/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "tam_Taml", + "tgt_lang": "eng_Latn", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/25k/tok-ta-si/sentencepiece.bpe.model b/Government/25k/tok-ta-si/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/25k/tok-ta-si/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/25k/tok-ta-si/special_tokens_map.json b/Government/25k/tok-ta-si/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/25k/tok-ta-si/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/25k/tok-ta-si/tokenizer_config.json b/Government/25k/tok-ta-si/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9526d1cacd39f5ee55eece61fdacbbc6ce50136d --- /dev/null +++ b/Government/25k/tok-ta-si/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "tam_Taml", + "tgt_lang": "sin_Sinh", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/59k/en-si_final_model/config.json b/Government/59k/en-si_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/59k/en-si_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/59k/en-si_final_model/generation_config.json b/Government/59k/en-si_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/59k/en-si_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/59k/en-si_final_model/pytorch_model.bin b/Government/59k/en-si_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..bd3c58059a3ab62a3152c1a0fe104fc8e911808a --- /dev/null +++ b/Government/59k/en-si_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbda2a786ed7fb94e97e06ca5a257cb71088e8afb387280084515f3183a0daf9 +size 2468870723 diff --git a/Government/59k/en-si_final_model/training_args.bin b/Government/59k/en-si_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..a27065cd5394cb912ae0d13dd50cb58d644b67ef --- /dev/null +++ b/Government/59k/en-si_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae5c4534b90164b8cc74a4587cb64c9fcaac3cf3d62c2a01360d91c6f89bac39 +size 4143 diff --git a/Government/59k/nllb_bible_eval_government_en-si_trained_59k.csv b/Government/59k/nllb_bible_eval_government_en-si_trained_59k.csv new file mode 100644 index 0000000000000000000000000000000000000000..d3113d84220ded45a4f1fc4ea0bad9c82dcd76e4 --- /dev/null +++ b/Government/59k/nllb_bible_eval_government_en-si_trained_59k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,32.8439209979711,28.363349865158927,2.2778499715716296,14.374691774850167,14.753576025978443,0.7897823176681995 diff --git a/Government/59k/nllb_government_eval_en-si_trained_59k.csv b/Government/59k/nllb_government_eval_en-si_trained_59k.csv new file mode 100644 index 0000000000000000000000000000000000000000..d5652423d185748d1c8d4fb20edaaf29b2a86070 --- /dev/null +++ b/Government/59k/nllb_government_eval_en-si_trained_59k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,58.63215636177139,54.397978126145276,23.210547408020922,46.32217428726776,52.096225229635294,0.9270260595853944 diff --git a/Government/59k/tok-en-si/sentencepiece.bpe.model b/Government/59k/tok-en-si/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/59k/tok-en-si/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/59k/tok-en-si/special_tokens_map.json b/Government/59k/tok-en-si/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/59k/tok-en-si/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/59k/tok-en-si/tokenizer_config.json b/Government/59k/tok-en-si/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fd1139979a7c7c4188b26d922af637f4bfb23e1f --- /dev/null +++ b/Government/59k/tok-en-si/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "eng_Latn", + "tgt_lang": "sin_Sinh", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/5k/en-si_final_model/config.json b/Government/5k/en-si_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/5k/en-si_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/5k/en-si_final_model/generation_config.json b/Government/5k/en-si_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/5k/en-si_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/5k/en-si_final_model/pytorch_model.bin b/Government/5k/en-si_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..b0bcfcf0f94a0b883067c25523596af61b2e1473 --- /dev/null +++ b/Government/5k/en-si_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b848f2cdb73b43f953e14882e142aac34bc365086c7a836ab8ca8a9a2a0b257 +size 2468870723 diff --git a/Government/5k/en-si_final_model/training_args.bin b/Government/5k/en-si_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..e12533327424529bb070829729cf5c84973e94d4 --- /dev/null +++ b/Government/5k/en-si_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60191a30fe7882398209812d08d93730870b8470ba8d7e634fcfa97d643ae9c7 +size 4143 diff --git a/Government/5k/en-ta/runs/Oct02_21-24-44_NLPC-Sita-MLM/events.out.tfevents.1759420484.NLPC-Sita-MLM.2581313.4 b/Government/5k/en-ta/runs/Oct02_21-24-44_NLPC-Sita-MLM/events.out.tfevents.1759420484.NLPC-Sita-MLM.2581313.4 new file mode 100644 index 0000000000000000000000000000000000000000..81707207f26232a18c6047e5895efc4d73c27564 --- /dev/null +++ b/Government/5k/en-ta/runs/Oct02_21-24-44_NLPC-Sita-MLM/events.out.tfevents.1759420484.NLPC-Sita-MLM.2581313.4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff0a6e8fd0ccb722d93ec4dd83daacc7f151b09e8835772a80233064cd1c61ad +size 6127 diff --git a/Government/5k/en-ta_final_model/config.json b/Government/5k/en-ta_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/5k/en-ta_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/5k/en-ta_final_model/generation_config.json b/Government/5k/en-ta_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/5k/en-ta_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/5k/en-ta_final_model/pytorch_model.bin b/Government/5k/en-ta_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..caa6b75add1518e2bd492231dd93c56a9ef0a814 --- /dev/null +++ b/Government/5k/en-ta_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a9c7d1dc6edaf5015f028152de7fdd1ffe28b137de2f3aee96479dbb8e26b11 +size 2468870723 diff --git a/Government/5k/en-ta_final_model/training_args.bin b/Government/5k/en-ta_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..46a97f74841f3d04205ab47ce1edb5aea73bbcf4 --- /dev/null +++ b/Government/5k/en-ta_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:658cfcbccf05d032c4ad86231880c593f3726159d9aec23b55b6aab00c66f517 +size 4143 diff --git a/Government/5k/nllb_bible_eval_government_en-si_trained_5k.csv b/Government/5k/nllb_bible_eval_government_en-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..1a2f9baf8bb5e53c3320ac9344eb3e9e2d67c0f8 --- /dev/null +++ b/Government/5k/nllb_bible_eval_government_en-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,17.38218305920879,14.720861483249708,0.6110980468231604,4.57594442109759,4.215972757003348,0.5760666883587837 diff --git a/Government/5k/nllb_bible_eval_government_en-ta_trained_5k.csv b/Government/5k/nllb_bible_eval_government_en-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..7d826c0a17e92fc35c587a9b9888cba8b267ca8e --- /dev/null +++ b/Government/5k/nllb_bible_eval_government_en-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,39.104270196186626,33.16444374880192,2.8756653381343487,13.64369627504619,12.35112913731052,0.7828839690685272 diff --git a/Government/5k/nllb_bible_eval_government_si-en_trained_5k.csv b/Government/5k/nllb_bible_eval_government_si-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..d7db2c4ed5d182aa7ae2312bb6f6c436448c6d53 --- /dev/null +++ b/Government/5k/nllb_bible_eval_government_si-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,45.335227345199684,44.4217335563957,21.733600421347013,22.91738522167107,22.648938874611055,0.7763690453767776 diff --git a/Government/5k/nllb_bible_eval_government_ta-en_trained_5k.csv b/Government/5k/nllb_bible_eval_government_ta-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..03d436ddbd927acab344d4921d6d25b094ba6d6b --- /dev/null +++ b/Government/5k/nllb_bible_eval_government_ta-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,39.37151886439601,38.37489519796626,16.399897348626975,18.26562117923812,17.627149209826893,0.7546233857274055 diff --git a/Government/5k/nllb_floresplus_eval_government_en-si_trained_5k.csv b/Government/5k/nllb_floresplus_eval_government_en-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..047ef576bf19585636463dacdf097e398b091f99 --- /dev/null +++ b/Government/5k/nllb_floresplus_eval_government_en-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,8.058227939320835,6.3488511990142,0.027026802552373566,0.17636601956376982,0.2460073296279243,0.4096863102830446 diff --git a/Government/5k/nllb_floresplus_eval_government_en-ta_trained_5k.csv b/Government/5k/nllb_floresplus_eval_government_en-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..476def54ec3f4031a7b659042d07fa9ad958bcbc --- /dev/null +++ b/Government/5k/nllb_floresplus_eval_government_en-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,52.03516812879777,45.77780503134915,10.47506217472233,25.702371506369836,25.19030750123965,0.8678937781705216 diff --git a/Government/5k/nllb_floresplus_eval_government_si-en_trained_5k.csv b/Government/5k/nllb_floresplus_eval_government_si-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..fc01a5302f00ee932706563d45e5d562800a58c2 --- /dev/null +++ b/Government/5k/nllb_floresplus_eval_government_si-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,56.64196178285397,54.44740562310189,28.154022271469724,29.694605434536683,30.57014291561554,0.865584895898231 diff --git a/Government/5k/nllb_floresplus_eval_government_si-ta_trained_5k.csv b/Government/5k/nllb_floresplus_eval_government_si-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..208f7aa1a77cae782fa7d6ff6a003e41aa3149d0 --- /dev/null +++ b/Government/5k/nllb_floresplus_eval_government_si-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,47.23390610823547,41.11550442607534,7.7823378036793915,19.932887956146672,19.798590547136538,0.8459384798149582 diff --git a/Government/5k/nllb_floresplus_eval_government_ta-en_trained_5k.csv b/Government/5k/nllb_floresplus_eval_government_ta-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..5e4ecc1f6eedc63baaf50a17a19e80c1472cfbc0 --- /dev/null +++ b/Government/5k/nllb_floresplus_eval_government_ta-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,55.77564075635998,53.61759194041663,27.478797007832842,28.93272675238803,29.848581440335153,0.8498090575688442 diff --git a/Government/5k/nllb_floresplus_eval_government_ta-si_trained_5k.csv b/Government/5k/nllb_floresplus_eval_government_ta-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b7e165c4802fdbdd89915fc3802001c974877381 --- /dev/null +++ b/Government/5k/nllb_floresplus_eval_government_ta-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,39.12278945002164,35.135536187359804,7.654368709400647,21.329424819398096,24.894369694537282,0.8447011141437787 diff --git a/Government/5k/nllb_government_eval_bible_en-si_trained_5k.csv b/Government/5k/nllb_government_eval_bible_en-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..1a2f9baf8bb5e53c3320ac9344eb3e9e2d67c0f8 --- /dev/null +++ b/Government/5k/nllb_government_eval_bible_en-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,17.38218305920879,14.720861483249708,0.6110980468231604,4.57594442109759,4.215972757003348,0.5760666883587837 diff --git a/Government/5k/nllb_government_eval_bible_en-ta_trained_5k.csv b/Government/5k/nllb_government_eval_bible_en-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..7d826c0a17e92fc35c587a9b9888cba8b267ca8e --- /dev/null +++ b/Government/5k/nllb_government_eval_bible_en-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,39.104270196186626,33.16444374880192,2.8756653381343487,13.64369627504619,12.35112913731052,0.7828839690685272 diff --git a/Government/5k/nllb_government_eval_bible_si-en_trained_5k.csv b/Government/5k/nllb_government_eval_bible_si-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..d7db2c4ed5d182aa7ae2312bb6f6c436448c6d53 --- /dev/null +++ b/Government/5k/nllb_government_eval_bible_si-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,45.335227345199684,44.4217335563957,21.733600421347013,22.91738522167107,22.648938874611055,0.7763690453767776 diff --git a/Government/5k/nllb_government_eval_bible_ta-en_trained_5k.csv b/Government/5k/nllb_government_eval_bible_ta-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..03d436ddbd927acab344d4921d6d25b094ba6d6b --- /dev/null +++ b/Government/5k/nllb_government_eval_bible_ta-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,39.37151886439601,38.37489519796626,16.399897348626975,18.26562117923812,17.627149209826893,0.7546233857274055 diff --git a/Government/5k/nllb_government_eval_en-si_trained_5k.csv b/Government/5k/nllb_government_eval_en-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ac20eb8ca52600e9308fdcafd4ae1c6c90f62b00 --- /dev/null +++ b/Government/5k/nllb_government_eval_en-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,8.479920459754377,6.767447703245162,0.1766950042221418,0.8276832754112516,1.3128873218794352,0.49582474255648096 diff --git a/Government/5k/nllb_government_eval_en-ta_trained_5k.csv b/Government/5k/nllb_government_eval_en-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..6c354f183fbb8df2cc322c6042047b64ee6de00d --- /dev/null +++ b/Government/5k/nllb_government_eval_en-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,57.107298221538684,51.01161552454825,17.322155133080567,32.27767883073434,32.40469014268493,0.8901339866102851 diff --git a/Government/5k/nllb_government_eval_government_en-si_trained_5k.csv b/Government/5k/nllb_government_eval_government_en-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..a29108348c9bf30bfb3c95047fc5bcbf0bc40d62 --- /dev/null +++ b/Government/5k/nllb_government_eval_government_en-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,8.529184619241422,6.817795023193822,0.13680568482507832,0.7307413818879813,1.2547235687803515,0.49993413537740705 diff --git a/Government/5k/nllb_government_eval_government_en-ta_trained_5k.csv b/Government/5k/nllb_government_eval_government_en-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..1ee900e27ccf25f2c61b968090ab4199ec20ee32 --- /dev/null +++ b/Government/5k/nllb_government_eval_government_en-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,57.488707523186044,51.33326923292132,17.451089018577015,32.797904822354056,32.88863987169818,0.892828518897295 diff --git a/Government/5k/nllb_government_eval_government_si-en_trained_5k.csv b/Government/5k/nllb_government_eval_government_si-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..0853788a61cd5f773d7ffdf0b06f09777b971808 --- /dev/null +++ b/Government/5k/nllb_government_eval_government_si-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,60.99733266989447,58.223940529836106,32.76016680774736,34.19087842338811,35.339936404370995,0.8665921680033207 diff --git a/Government/5k/nllb_government_eval_government_si-ta_trained_5k.csv b/Government/5k/nllb_government_eval_government_si-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..73f4724e73641e959c59e356f544cd23fc64532f --- /dev/null +++ b/Government/5k/nllb_government_eval_government_si-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,56.12097833718678,49.733543358075515,17.030684140516204,30.7426792607728,30.896783690821394,0.8861886198818684 diff --git a/Government/5k/nllb_government_eval_government_ta-en_trained_5k.csv b/Government/5k/nllb_government_eval_government_ta-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..1251698594531e1257d41750a7100ad0e3f693dc --- /dev/null +++ b/Government/5k/nllb_government_eval_government_ta-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,60.107155518614285,57.4457710791638,32.311082846928464,33.55085234615088,34.4590611198238,0.8479743801653385 diff --git a/Government/5k/nllb_government_eval_government_ta-si_trained_5k.csv b/Government/5k/nllb_government_eval_government_ta-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..e2a826bbab4ef08025daa96307ad0285867edd68 --- /dev/null +++ b/Government/5k/nllb_government_eval_government_ta-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,51.41926015585591,47.17738904375178,17.444572340694904,35.27541311412441,41.78334972760448,0.8926729651689529 diff --git a/Government/5k/nllb_government_eval_si-en_trained_5k.csv b/Government/5k/nllb_government_eval_si-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..dff6a479183efafc7513a2f666fc5a5580974bb7 --- /dev/null +++ b/Government/5k/nllb_government_eval_si-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,61.2033090165022,58.47500490804066,33.0392515898939,34.40713092993954,35.461120832915796,0.8663766804253291 diff --git a/Government/5k/nllb_government_eval_si-ta_trained_5k.csv b/Government/5k/nllb_government_eval_si-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..a222ae1ea055c8563332418823d2f58cc190a34c --- /dev/null +++ b/Government/5k/nllb_government_eval_si-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,55.90315071039969,49.655906360944726,16.826009480293255,30.5146046040742,30.722127131519127,0.883553880543482 diff --git a/Government/5k/nllb_government_eval_ta-en_trained_5k.csv b/Government/5k/nllb_government_eval_ta-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..929b1610fc048ccd44f9e49dbe2b11dcb75b236d --- /dev/null +++ b/Government/5k/nllb_government_eval_ta-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,60.052901197436405,57.430911231944634,32.385516827127226,33.435562149471906,34.37252797484691,0.8469337616128646 diff --git a/Government/5k/nllb_government_eval_ta-si_trained_5k.csv b/Government/5k/nllb_government_eval_ta-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..48db4a9228e83e01c0ee6e762d1cb26ce7039080 --- /dev/null +++ b/Government/5k/nllb_government_eval_ta-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,50.856976770411855,46.67090345828602,17.16727990082564,34.968591220000306,41.373373435238754,0.8909372211652817 diff --git a/Government/5k/nllb_health_eval_government_en-si_trained_5k.csv b/Government/5k/nllb_health_eval_government_en-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..4abcd739b128c29870fc65b25f52ec293c67aa37 --- /dev/null +++ b/Government/5k/nllb_health_eval_government_en-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,6.1429329243101805,4.977345281893618,0.0372134062417059,0.11139531895325058,0.21950967703787327,0.39680020181315667 diff --git a/Government/5k/nllb_health_eval_government_en-ta_trained_5k.csv b/Government/5k/nllb_health_eval_government_en-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..81bae54ae438fb04de3f64cd9d52ec8389778ecd --- /dev/null +++ b/Government/5k/nllb_health_eval_government_en-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,58.66871165732608,52.11839595334895,13.722466517727376,31.967876341824944,30.735447764073637,0.8376800656765699 diff --git a/Government/5k/nllb_health_eval_government_si-en_trained_5k.csv b/Government/5k/nllb_health_eval_government_si-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..b9a31f23661000f931288d54e84445d92e719a0e --- /dev/null +++ b/Government/5k/nllb_health_eval_government_si-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,57.73196066859465,54.65553290904695,23.68447635480341,29.445175969422657,29.92273280986862,0.8502491591686613 diff --git a/Government/5k/nllb_health_eval_government_si-ta_trained_5k.csv b/Government/5k/nllb_health_eval_government_si-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..7f202f6910b302bcb788feba45d6c844ec8e7e77 --- /dev/null +++ b/Government/5k/nllb_health_eval_government_si-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,51.37994416681284,44.600702669565,7.689568469864875,22.39699110248858,21.570591767918586,0.8083506729263736 diff --git a/Government/5k/nllb_health_eval_government_ta-en_trained_5k.csv b/Government/5k/nllb_health_eval_government_ta-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..4ea27bae70da17da508deb93c251dc1c15a7a54c --- /dev/null +++ b/Government/5k/nllb_health_eval_government_ta-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,59.18970070926514,56.47359668928438,28.929360390084657,32.009640327426226,32.956460565560505,0.8328773201704025 diff --git a/Government/5k/nllb_health_eval_government_ta-si_trained_5k.csv b/Government/5k/nllb_health_eval_government_ta-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..60a2d796eb7113cd10bb9d0b38aa3b190e2df58f --- /dev/null +++ b/Government/5k/nllb_health_eval_government_ta-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,37.65115327528594,33.743185841828364,8.574490363016755,20.69555348481924,23.218023579085937,0.7880995444269204 diff --git a/Government/5k/nllb_jailbreak_eval_government_en-si_trained_5k.csv b/Government/5k/nllb_jailbreak_eval_government_en-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..18980903692b97b54d956f7ef3bfb7a9cfe14a08 --- /dev/null +++ b/Government/5k/nllb_jailbreak_eval_government_en-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,6.527991317587816,5.434436017464176,0.1679797515935076,0.7485159688817504,0.9217368558347369,0.4593092997223139 diff --git a/Government/5k/nllb_jailbreak_eval_government_en-ta_trained_5k.csv b/Government/5k/nllb_jailbreak_eval_government_en-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..58086a51f57abf905e105487d1e146945c18cb85 --- /dev/null +++ b/Government/5k/nllb_jailbreak_eval_government_en-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,56.912620918833454,51.79601398850566,19.88472608153632,32.20857214956527,31.37671701700715,0.8543761656731367 diff --git a/Government/5k/nllb_jailbreak_eval_government_si-en_trained_5k.csv b/Government/5k/nllb_jailbreak_eval_government_si-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..3be58d2d01d4bba97f232caf2bef73b650e23a59 --- /dev/null +++ b/Government/5k/nllb_jailbreak_eval_government_si-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,54.680471231588754,52.920082556983104,30.350335192013162,30.535097787680858,31.254736434782973,0.8590239198207855 diff --git a/Government/5k/nllb_jailbreak_eval_government_si-ta_trained_5k.csv b/Government/5k/nllb_jailbreak_eval_government_si-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..4786bf97245574acea27e3d00485f4520ba8eaaf --- /dev/null +++ b/Government/5k/nllb_jailbreak_eval_government_si-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,51.81668421812679,46.51265806217224,14.118832177970003,23.904679503172375,23.24996088975882,0.8370880727469922 diff --git a/Government/5k/nllb_jailbreak_eval_government_ta-en_trained_5k.csv b/Government/5k/nllb_jailbreak_eval_government_ta-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..aaf281ffbbffca0273b25ad0fe720fe71d007c3b --- /dev/null +++ b/Government/5k/nllb_jailbreak_eval_government_ta-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,55.30634053524792,54.06575575883763,32.67612800144492,30.587048687078873,31.414799560176046,0.8421250923871993 diff --git a/Government/5k/nllb_jailbreak_eval_government_ta-si_trained_5k.csv b/Government/5k/nllb_jailbreak_eval_government_ta-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..bcd4bf7feaed9b4f29a885040214a7491b663ca4 --- /dev/null +++ b/Government/5k/nllb_jailbreak_eval_government_ta-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,38.58916766639129,35.548020932245,12.016463336897296,20.18377192507685,22.706979516016947,0.8178093786835671 diff --git a/Government/5k/nllb_literature-children_eval_government_en-si_trained_5k.csv b/Government/5k/nllb_literature-children_eval_government_en-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..998d3a736082ae30cf51832cb479c74950d90414 --- /dev/null +++ b/Government/5k/nllb_literature-children_eval_government_en-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,8.063824155188607,6.555031221691862,0.1819747397416745,0.3778547201497972,0.3628863984872553,0.46793258693814277 diff --git a/Government/5k/nllb_literature-children_eval_government_en-ta_trained_5k.csv b/Government/5k/nllb_literature-children_eval_government_en-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..56270473bbfcb444b73e0e6182e40ed33ee87111 --- /dev/null +++ b/Government/5k/nllb_literature-children_eval_government_en-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,52.73975783803249,47.36067825923689,12.806093452131401,26.16715572899859,25.169424200290603,0.8350815981867965 diff --git a/Government/5k/nllb_literature-children_eval_government_si-en_trained_5k.csv b/Government/5k/nllb_literature-children_eval_government_si-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..f7ba7abe70a1d1064b00deac419f4832dcb7b905 --- /dev/null +++ b/Government/5k/nllb_literature-children_eval_government_si-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,43.69492188032749,42.61634074514172,19.82485043746595,19.36560532468245,19.55606985774085,0.807296268671751 diff --git a/Government/5k/nllb_literature-children_eval_government_si-ta_trained_5k.csv b/Government/5k/nllb_literature-children_eval_government_si-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..4af639bddfa41964c5ff4e7c396dc2e0aaa6a2e9 --- /dev/null +++ b/Government/5k/nllb_literature-children_eval_government_si-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,44.14071777997267,38.70461289488636,6.745391082329602,16.030624195019897,15.502823721584761,0.8027440217356102 diff --git a/Government/5k/nllb_literature-children_eval_government_ta-en_trained_5k.csv b/Government/5k/nllb_literature-children_eval_government_ta-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..5474a6b6fb1aec8f55eafeace5b8aec4f7f7243a --- /dev/null +++ b/Government/5k/nllb_literature-children_eval_government_ta-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,50.115983991974566,49.31229876069414,26.280567187594468,25.576055242195256,26.052869267090234,0.8162053002182532 diff --git a/Government/5k/nllb_literature-children_eval_government_ta-si_trained_5k.csv b/Government/5k/nllb_literature-children_eval_government_ta-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..452cc5193f45b9b6063bbad56e34db4ae0c2d15e --- /dev/null +++ b/Government/5k/nllb_literature-children_eval_government_ta-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,31.32002834430054,28.134170682083813,4.480587464149585,11.491425934951907,13.227056488225278,0.799190194723695 diff --git a/Government/5k/nllb_literature-kathlyn_eval_government_en-si_trained_5k.csv b/Government/5k/nllb_literature-kathlyn_eval_government_en-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..998d3a736082ae30cf51832cb479c74950d90414 --- /dev/null +++ b/Government/5k/nllb_literature-kathlyn_eval_government_en-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,8.063824155188607,6.555031221691862,0.1819747397416745,0.3778547201497972,0.3628863984872553,0.46793258693814277 diff --git a/Government/5k/nllb_literature-kathlyn_eval_government_en-ta_trained_5k.csv b/Government/5k/nllb_literature-kathlyn_eval_government_en-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..56270473bbfcb444b73e0e6182e40ed33ee87111 --- /dev/null +++ b/Government/5k/nllb_literature-kathlyn_eval_government_en-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,52.73975783803249,47.36067825923689,12.806093452131401,26.16715572899859,25.169424200290603,0.8350815981867965 diff --git a/Government/5k/nllb_literature-kathlyn_eval_government_si-en_trained_5k.csv b/Government/5k/nllb_literature-kathlyn_eval_government_si-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..f7ba7abe70a1d1064b00deac419f4832dcb7b905 --- /dev/null +++ b/Government/5k/nllb_literature-kathlyn_eval_government_si-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,43.69492188032749,42.61634074514172,19.82485043746595,19.36560532468245,19.55606985774085,0.807296268671751 diff --git a/Government/5k/nllb_literature-kathlyn_eval_government_si-ta_trained_5k.csv b/Government/5k/nllb_literature-kathlyn_eval_government_si-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..4af639bddfa41964c5ff4e7c396dc2e0aaa6a2e9 --- /dev/null +++ b/Government/5k/nllb_literature-kathlyn_eval_government_si-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,44.14071777997267,38.70461289488636,6.745391082329602,16.030624195019897,15.502823721584761,0.8027440217356102 diff --git a/Government/5k/nllb_literature-kathlyn_eval_government_ta-en_trained_5k.csv b/Government/5k/nllb_literature-kathlyn_eval_government_ta-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..5474a6b6fb1aec8f55eafeace5b8aec4f7f7243a --- /dev/null +++ b/Government/5k/nllb_literature-kathlyn_eval_government_ta-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,50.115983991974566,49.31229876069414,26.280567187594468,25.576055242195256,26.052869267090234,0.8162053002182532 diff --git a/Government/5k/nllb_literature-kathlyn_eval_government_ta-si_trained_5k.csv b/Government/5k/nllb_literature-kathlyn_eval_government_ta-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..452cc5193f45b9b6063bbad56e34db4ae0c2d15e --- /dev/null +++ b/Government/5k/nllb_literature-kathlyn_eval_government_ta-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,31.32002834430054,28.134170682083813,4.480587464149585,11.491425934951907,13.227056488225278,0.799190194723695 diff --git a/Government/5k/nllb_literature-old_eval_government_en-si_trained_5k.csv b/Government/5k/nllb_literature-old_eval_government_en-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..998d3a736082ae30cf51832cb479c74950d90414 --- /dev/null +++ b/Government/5k/nllb_literature-old_eval_government_en-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,8.063824155188607,6.555031221691862,0.1819747397416745,0.3778547201497972,0.3628863984872553,0.46793258693814277 diff --git a/Government/5k/nllb_literature-old_eval_government_en-ta_trained_5k.csv b/Government/5k/nllb_literature-old_eval_government_en-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..56270473bbfcb444b73e0e6182e40ed33ee87111 --- /dev/null +++ b/Government/5k/nllb_literature-old_eval_government_en-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,52.73975783803249,47.36067825923689,12.806093452131401,26.16715572899859,25.169424200290603,0.8350815981867965 diff --git a/Government/5k/nllb_literature-old_eval_government_si-en_trained_5k.csv b/Government/5k/nllb_literature-old_eval_government_si-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..f7ba7abe70a1d1064b00deac419f4832dcb7b905 --- /dev/null +++ b/Government/5k/nllb_literature-old_eval_government_si-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,43.69492188032749,42.61634074514172,19.82485043746595,19.36560532468245,19.55606985774085,0.807296268671751 diff --git a/Government/5k/nllb_literature-old_eval_government_si-ta_trained_5k.csv b/Government/5k/nllb_literature-old_eval_government_si-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..4af639bddfa41964c5ff4e7c396dc2e0aaa6a2e9 --- /dev/null +++ b/Government/5k/nllb_literature-old_eval_government_si-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,44.14071777997267,38.70461289488636,6.745391082329602,16.030624195019897,15.502823721584761,0.8027440217356102 diff --git a/Government/5k/nllb_literature-old_eval_government_ta-en_trained_5k.csv b/Government/5k/nllb_literature-old_eval_government_ta-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..5474a6b6fb1aec8f55eafeace5b8aec4f7f7243a --- /dev/null +++ b/Government/5k/nllb_literature-old_eval_government_ta-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,50.115983991974566,49.31229876069414,26.280567187594468,25.576055242195256,26.052869267090234,0.8162053002182532 diff --git a/Government/5k/nllb_literature-old_eval_government_ta-si_trained_5k.csv b/Government/5k/nllb_literature-old_eval_government_ta-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..452cc5193f45b9b6063bbad56e34db4ae0c2d15e --- /dev/null +++ b/Government/5k/nllb_literature-old_eval_government_ta-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,31.32002834430054,28.134170682083813,4.480587464149585,11.491425934951907,13.227056488225278,0.799190194723695 diff --git a/Government/5k/nllb_literature_eval_government_en-si_trained_5k.csv b/Government/5k/nllb_literature_eval_government_en-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..998d3a736082ae30cf51832cb479c74950d90414 --- /dev/null +++ b/Government/5k/nllb_literature_eval_government_en-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,8.063824155188607,6.555031221691862,0.1819747397416745,0.3778547201497972,0.3628863984872553,0.46793258693814277 diff --git a/Government/5k/nllb_literature_eval_government_en-ta_trained_5k.csv b/Government/5k/nllb_literature_eval_government_en-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..56270473bbfcb444b73e0e6182e40ed33ee87111 --- /dev/null +++ b/Government/5k/nllb_literature_eval_government_en-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,52.73975783803249,47.36067825923689,12.806093452131401,26.16715572899859,25.169424200290603,0.8350815981867965 diff --git a/Government/5k/nllb_literature_eval_government_si-en_trained_5k.csv b/Government/5k/nllb_literature_eval_government_si-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..f7ba7abe70a1d1064b00deac419f4832dcb7b905 --- /dev/null +++ b/Government/5k/nllb_literature_eval_government_si-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,43.69492188032749,42.61634074514172,19.82485043746595,19.36560532468245,19.55606985774085,0.807296268671751 diff --git a/Government/5k/nllb_literature_eval_government_si-ta_trained_5k.csv b/Government/5k/nllb_literature_eval_government_si-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..4af639bddfa41964c5ff4e7c396dc2e0aaa6a2e9 --- /dev/null +++ b/Government/5k/nllb_literature_eval_government_si-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,44.14071777997267,38.70461289488636,6.745391082329602,16.030624195019897,15.502823721584761,0.8027440217356102 diff --git a/Government/5k/nllb_literature_eval_government_ta-en_trained_5k.csv b/Government/5k/nllb_literature_eval_government_ta-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..5474a6b6fb1aec8f55eafeace5b8aec4f7f7243a --- /dev/null +++ b/Government/5k/nllb_literature_eval_government_ta-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,50.115983991974566,49.31229876069414,26.280567187594468,25.576055242195256,26.052869267090234,0.8162053002182532 diff --git a/Government/5k/nllb_literature_eval_government_ta-si_trained_5k.csv b/Government/5k/nllb_literature_eval_government_ta-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..452cc5193f45b9b6063bbad56e34db4ae0c2d15e --- /dev/null +++ b/Government/5k/nllb_literature_eval_government_ta-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,31.32002834430054,28.134170682083813,4.480587464149585,11.491425934951907,13.227056488225278,0.799190194723695 diff --git a/Government/5k/nllb_maths_eval_government_en-si_trained_5k.csv b/Government/5k/nllb_maths_eval_government_en-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..63e05d152885d3aed86ad78669034a56ad4fd41c --- /dev/null +++ b/Government/5k/nllb_maths_eval_government_en-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,6.519839470122508,5.185081036326267,0.04893371406007849,0.20115199082009322,0.18642439578313189,0.44491713312268255 diff --git a/Government/5k/nllb_maths_eval_government_en-ta_trained_5k.csv b/Government/5k/nllb_maths_eval_government_en-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..45da5e2d439c794c98355bba685019413249a8af --- /dev/null +++ b/Government/5k/nllb_maths_eval_government_en-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,50.6870615642952,46.644072448648,20.39240768504329,26.561548371984117,25.92481296568236,0.8300952911973 diff --git a/Government/5k/nllb_maths_eval_government_si-en_trained_5k.csv b/Government/5k/nllb_maths_eval_government_si-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..59e0f13b9b016f379b51c60d2e493f0c51392502 --- /dev/null +++ b/Government/5k/nllb_maths_eval_government_si-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,57.493680580337156,56.93484539279011,38.38131159782538,37.96777084280895,38.54294766248619,0.8448873843252659 diff --git a/Government/5k/nllb_maths_eval_government_ta-en_trained_5k.csv b/Government/5k/nllb_maths_eval_government_ta-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..cd3bae06abf1f231e30ba39c3db3f2790a5dedf7 --- /dev/null +++ b/Government/5k/nllb_maths_eval_government_ta-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,54.71448645869131,53.98212664331717,34.865633675762915,34.49562297253713,35.2138555042198,0.8373731119334697 diff --git a/Government/5k/nllb_news_eval_government_en-si_trained_5k.csv b/Government/5k/nllb_news_eval_government_en-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ec6574d22cbff26caf30ab9c59dae4767217c55d --- /dev/null +++ b/Government/5k/nllb_news_eval_government_en-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,8.064029365713953,6.348901418473853,0.02969855668144618,0.29900276509853085,0.4486387708986559,0.4363476236760616 diff --git a/Government/5k/nllb_news_eval_government_en-ta_trained_5k.csv b/Government/5k/nllb_news_eval_government_en-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..ac5a333e3017cfcf9502e803f112771e0a5e6ada --- /dev/null +++ b/Government/5k/nllb_news_eval_government_en-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,58.92559041177843,53.330231600017186,17.836532112029595,34.421406579635025,33.976456445227754,0.8648977399351128 diff --git a/Government/5k/nllb_news_eval_government_si-en_trained_5k.csv b/Government/5k/nllb_news_eval_government_si-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..86a3d0a316c7260be8f50a3cb3ad88805c7ad52b --- /dev/null +++ b/Government/5k/nllb_news_eval_government_si-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,55.00139455274541,52.31023841866155,23.658658442202828,25.794596124102387,26.69403926211541,0.8537135955691337 diff --git a/Government/5k/nllb_news_eval_government_si-ta_trained_5k.csv b/Government/5k/nllb_news_eval_government_si-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..453dec123971f8d3df1b4bf7e4f323b7cc769bb6 --- /dev/null +++ b/Government/5k/nllb_news_eval_government_si-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,50.95998031662033,44.90239311813765,8.83243198122357,22.702569083306024,22.694441463061317,0.8536119359966007 diff --git a/Government/5k/nllb_news_eval_government_ta-en_trained_5k.csv b/Government/5k/nllb_news_eval_government_ta-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..fc2f58ca91c7b11a2fca79d87e79021a067558a2 --- /dev/null +++ b/Government/5k/nllb_news_eval_government_ta-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,60.047573839598925,58.19651120554607,33.17435801105844,33.95815999261179,35.077209258460485,0.8596431332299987 diff --git a/Government/5k/nllb_news_eval_government_ta-si_trained_5k.csv b/Government/5k/nllb_news_eval_government_ta-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..5695b2a543f163aae6728c14fb7d65465675701e --- /dev/null +++ b/Government/5k/nllb_news_eval_government_ta-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,38.06765279142419,33.94830309744808,7.201166739204406,20.49284853547799,23.954984625606958,0.8293779508601273 diff --git a/Government/5k/nllb_nllb_eval_government_en-si_trained_5k.csv b/Government/5k/nllb_nllb_eval_government_en-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..99520433fc30706125a144ecebdc0e42295a6c42 --- /dev/null +++ b/Government/5k/nllb_nllb_eval_government_en-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,8.980534923916675,7.5253723406132105,0.4358489858308041,1.0720427279427547,1.0162517113668066,0.5183496097743482 diff --git a/Government/5k/nllb_nllb_eval_government_en-ta_trained_5k.csv b/Government/5k/nllb_nllb_eval_government_en-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..dae9cedcb64d38396a1453f63e18eed3325e752c --- /dev/null +++ b/Government/5k/nllb_nllb_eval_government_en-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,56.013767992105265,51.10331545240099,17.49474037237022,30.74849893729091,29.937989572443307,0.9035753657594062 diff --git a/Government/5k/nllb_nllb_eval_government_si-en_trained_5k.csv b/Government/5k/nllb_nllb_eval_government_si-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..2b324d2d14297681d82521818f71391b186c2984 --- /dev/null +++ b/Government/5k/nllb_nllb_eval_government_si-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,62.78110346546808,62.05912069488389,42.211733156761916,41.70838449001414,41.91348074095811,0.9005061542755713 diff --git a/Government/5k/nllb_nllb_eval_government_si-ta_trained_5k.csv b/Government/5k/nllb_nllb_eval_government_si-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..d502fbd59e51ab3621249287cd660f2fed676e5d --- /dev/null +++ b/Government/5k/nllb_nllb_eval_government_si-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,54.10965653462958,48.94280558343474,14.383224952538336,27.135943356000563,26.12773441010943,0.8999825488074803 diff --git a/Government/5k/nllb_nllb_eval_government_ta-en_trained_5k.csv b/Government/5k/nllb_nllb_eval_government_ta-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..44ce7768e2ec4355b39a384bed2eb93ddb23df26 --- /dev/null +++ b/Government/5k/nllb_nllb_eval_government_ta-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,62.8475124059938,62.144966584928895,42.70043472874664,42.00573936028934,42.24314774074809,0.8903522048958289 diff --git a/Government/5k/nllb_nllb_eval_government_ta-si_trained_5k.csv b/Government/5k/nllb_nllb_eval_government_ta-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..06a7bf8885d5e091dbbb05fd498e4f5f730998eb --- /dev/null +++ b/Government/5k/nllb_nllb_eval_government_ta-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,42.03424671487618,38.715827452288764,11.287307485401328,22.42073353637314,26.325389675886214,0.8902426442085833 diff --git a/Government/5k/nllb_opensubtitles_eval_government_en-si_trained_5k.csv b/Government/5k/nllb_opensubtitles_eval_government_en-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..3534d646f253b867bcc1f3e4e3249d1c4cae178c --- /dev/null +++ b/Government/5k/nllb_opensubtitles_eval_government_en-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,18.823833403766027,17.25520904752903,2.8231739289631452,6.079735336389598,5.4618216841043665,0.6699169592484832 diff --git a/Government/5k/nllb_opensubtitles_eval_government_en-ta_trained_5k.csv b/Government/5k/nllb_opensubtitles_eval_government_en-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..7a19bc85f423817c20589cb07bbe2afbeca7625e --- /dev/null +++ b/Government/5k/nllb_opensubtitles_eval_government_en-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,48.270452924617146,44.54937233725686,15.715368181882,27.295537890775297,27.037730056270444,0.8486921544075012 diff --git a/Government/5k/nllb_opensubtitles_eval_government_si-en_trained_5k.csv b/Government/5k/nllb_opensubtitles_eval_government_si-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..57d5c60eda36ca9525f50d314d0a3a496aeb928c --- /dev/null +++ b/Government/5k/nllb_opensubtitles_eval_government_si-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,63.40878654318058,62.604126471230536,41.30336470327033,42.09400841117228,42.33976466887083,0.8930004394203425 diff --git a/Government/5k/nllb_opensubtitles_eval_government_si-ta_trained_5k.csv b/Government/5k/nllb_opensubtitles_eval_government_si-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..4c9d45b11f4906d78075d7dbc18da1d892a79233 --- /dev/null +++ b/Government/5k/nllb_opensubtitles_eval_government_si-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,54.432059902914645,50.075506543199296,17.528247788305706,30.09849940414254,29.195537045212998,0.8908426854386926 diff --git a/Government/5k/nllb_opensubtitles_eval_government_ta-en_trained_5k.csv b/Government/5k/nllb_opensubtitles_eval_government_ta-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..cda6734dec666613d63ac1f42c7e7d03e1c72ad2 --- /dev/null +++ b/Government/5k/nllb_opensubtitles_eval_government_ta-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,60.2375088708884,59.542621070701095,37.73203593706005,38.06462375547255,38.33088629383616,0.8731431738883257 diff --git a/Government/5k/nllb_opensubtitles_eval_government_ta-si_trained_5k.csv b/Government/5k/nllb_opensubtitles_eval_government_ta-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..cccdf3fa5d4e58ef1b53fdcb671382fb4a14dab7 --- /dev/null +++ b/Government/5k/nllb_opensubtitles_eval_government_ta-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,34.33365229924564,31.362802703379582,7.442684778858739,18.426800660151997,20.06852154162444,0.8642446266561746 diff --git a/Government/5k/nllb_wikipedia_eval_government_en-si_trained_5k.csv b/Government/5k/nllb_wikipedia_eval_government_en-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..094303a039f4f5443ef42315c42eb16565749d48 --- /dev/null +++ b/Government/5k/nllb_wikipedia_eval_government_en-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,8.871693845189547,6.993505824594849,0.027742975346973384,0.16451259301635496,0.22485960079918635,0.44386809903383256 diff --git a/Government/5k/nllb_wikipedia_eval_government_en-ta_trained_5k.csv b/Government/5k/nllb_wikipedia_eval_government_en-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..f6b8bc4c5910b978c0524e79dab50bfb8c26f992 --- /dev/null +++ b/Government/5k/nllb_wikipedia_eval_government_en-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,54.76977526202764,48.428196118589256,11.978872103962386,29.1431958843411,28.692988400969938,0.874132488772273 diff --git a/Government/5k/nllb_wikipedia_eval_government_si-en_trained_5k.csv b/Government/5k/nllb_wikipedia_eval_government_si-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..5aea92cc831668d8da5185dc98b890e42e4db9bf --- /dev/null +++ b/Government/5k/nllb_wikipedia_eval_government_si-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,54.013321807431225,51.992516773370546,25.638065555853593,28.404409789625227,28.94878056957761,0.8408416794836521 diff --git a/Government/5k/nllb_wikipedia_eval_government_si-ta_trained_5k.csv b/Government/5k/nllb_wikipedia_eval_government_si-ta_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..5298efe20a5a8808b29187a22f0bfac067f72dd2 --- /dev/null +++ b/Government/5k/nllb_wikipedia_eval_government_si-ta_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-ta_final_model,47.54393797687138,41.262209610796106,6.82105375829823,21.135531898017184,20.775951029697076,0.8490756225883961 diff --git a/Government/5k/nllb_wikipedia_eval_government_ta-en_trained_5k.csv b/Government/5k/nllb_wikipedia_eval_government_ta-en_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..fb6f83f78165f85028605fe4865c2fb0743534e4 --- /dev/null +++ b/Government/5k/nllb_wikipedia_eval_government_ta-en_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,59.11444411843373,57.148035802254526,31.35505151525585,34.01561366893348,34.56963405352514,0.8486373663544655 diff --git a/Government/5k/nllb_wikipedia_eval_government_ta-si_trained_5k.csv b/Government/5k/nllb_wikipedia_eval_government_ta-si_trained_5k.csv new file mode 100644 index 0000000000000000000000000000000000000000..5bf7376ac1c5f1bc8b5fd2409295ce64d9e83495 --- /dev/null +++ b/Government/5k/nllb_wikipedia_eval_government_ta-si_trained_5k.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-si_final_model,38.86199731494143,34.62355952934438,6.433818298061117,19.883944774417436,23.10354666443403,0.8495119138360023 diff --git a/Government/5k/si-en/runs/Oct02_23-46-55_NLPC-Sita-MLM/events.out.tfevents.1759429015.NLPC-Sita-MLM.2581313.10 b/Government/5k/si-en/runs/Oct02_23-46-55_NLPC-Sita-MLM/events.out.tfevents.1759429015.NLPC-Sita-MLM.2581313.10 new file mode 100644 index 0000000000000000000000000000000000000000..1db771be367fb27f4dd634cd57722b2fa232273a --- /dev/null +++ b/Government/5k/si-en/runs/Oct02_23-46-55_NLPC-Sita-MLM/events.out.tfevents.1759429015.NLPC-Sita-MLM.2581313.10 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:722f44a83a76b4904bfe452025ad7e68a3fc0f36c0a78b88396c04b1a56e636d +size 6127 diff --git a/Government/5k/si-en_final_model/config.json b/Government/5k/si-en_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/5k/si-en_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/5k/si-en_final_model/generation_config.json b/Government/5k/si-en_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/5k/si-en_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/5k/si-en_final_model/pytorch_model.bin b/Government/5k/si-en_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..482d704cd65340727acf804aa645ffefaeb39e27 --- /dev/null +++ b/Government/5k/si-en_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb4e9b337e86073f1f8eb5bfbe242b1f309e8252452ef9b839729f6800bf2ecf +size 2468870723 diff --git a/Government/5k/si-en_final_model/training_args.bin b/Government/5k/si-en_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..d4ea0cc1b0124b4194b28de47fce545e5abfcacc --- /dev/null +++ b/Government/5k/si-en_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5b91cf152db3e08387ef274a97acad5746c068f4b58f6e9ac9c16d1870355e5 +size 4143 diff --git a/Government/5k/si-ta/runs/Oct03_06-38-47_NLPC-Sita-MLM/events.out.tfevents.1759453727.NLPC-Sita-MLM.2581313.22 b/Government/5k/si-ta/runs/Oct03_06-38-47_NLPC-Sita-MLM/events.out.tfevents.1759453727.NLPC-Sita-MLM.2581313.22 new file mode 100644 index 0000000000000000000000000000000000000000..a2787f8d969cbe24086f397fc7413c1d7ffdaf91 --- /dev/null +++ b/Government/5k/si-ta/runs/Oct03_06-38-47_NLPC-Sita-MLM/events.out.tfevents.1759453727.NLPC-Sita-MLM.2581313.22 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a940f6b1973fee29c3537c32188f959023a2eff32bf9e31d3b524d6a60014479 +size 6127 diff --git a/Government/5k/si-ta_final_model/config.json b/Government/5k/si-ta_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/5k/si-ta_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/5k/si-ta_final_model/generation_config.json b/Government/5k/si-ta_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/5k/si-ta_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/5k/si-ta_final_model/pytorch_model.bin b/Government/5k/si-ta_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..7858561873017d7033650b54655f41d9b535e840 --- /dev/null +++ b/Government/5k/si-ta_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6830d574e91e825665f65a45c5d58cbcfe5bd2f164394f6e9ee007c4c3e2fff2 +size 2468870723 diff --git a/Government/5k/si-ta_final_model/training_args.bin b/Government/5k/si-ta_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..a28901ca31eb4b0da5d41791509653859b03561e --- /dev/null +++ b/Government/5k/si-ta_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0ecc618845dddc7cc2167f7378bbf36d4fa03a523e1b3018063a4d64e0d9213 +size 4143 diff --git a/Government/5k/ta-en/runs/Oct03_02-08-26_NLPC-Sita-MLM/events.out.tfevents.1759437506.NLPC-Sita-MLM.2581313.16 b/Government/5k/ta-en/runs/Oct03_02-08-26_NLPC-Sita-MLM/events.out.tfevents.1759437506.NLPC-Sita-MLM.2581313.16 new file mode 100644 index 0000000000000000000000000000000000000000..07c7833b8357a6a537ccee2061c3d8d92982dc90 --- /dev/null +++ b/Government/5k/ta-en/runs/Oct03_02-08-26_NLPC-Sita-MLM/events.out.tfevents.1759437506.NLPC-Sita-MLM.2581313.16 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc53a2ba8efb3bad2dc388971d332ea237869dba7b0422c8a1ae0ab016c7c15e +size 6127 diff --git a/Government/5k/ta-en_final_model/config.json b/Government/5k/ta-en_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/5k/ta-en_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/5k/ta-en_final_model/generation_config.json b/Government/5k/ta-en_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/5k/ta-en_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/5k/ta-en_final_model/pytorch_model.bin b/Government/5k/ta-en_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..576a8a78b5bd456c09deb03ad7440a529fc18855 --- /dev/null +++ b/Government/5k/ta-en_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03b3e336c431f43865454b1801d0dfcc3454f1ec904059607dfb6cb764fc40b8 +size 2468870723 diff --git a/Government/5k/ta-en_final_model/training_args.bin b/Government/5k/ta-en_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..cc02f1280e7194883bd37f6d11ab9c30fba3cd3c --- /dev/null +++ b/Government/5k/ta-en_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0538feb48dd4d15c34034d1029ce11720c6c2c458ed51ab0c9d33b37451a59e8 +size 4143 diff --git a/Government/5k/ta-si/runs/Oct03_09-09-37_NLPC-Sita-MLM/events.out.tfevents.1759462777.NLPC-Sita-MLM.2581313.28 b/Government/5k/ta-si/runs/Oct03_09-09-37_NLPC-Sita-MLM/events.out.tfevents.1759462777.NLPC-Sita-MLM.2581313.28 new file mode 100644 index 0000000000000000000000000000000000000000..0ddae69aeb38841223e4421ed4f8e49ce7654144 --- /dev/null +++ b/Government/5k/ta-si/runs/Oct03_09-09-37_NLPC-Sita-MLM/events.out.tfevents.1759462777.NLPC-Sita-MLM.2581313.28 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f695a15c8d1781a7fb88317e933409fd2b1261dffbe2570344bb00e3044aab5f +size 6127 diff --git a/Government/5k/ta-si_final_model/config.json b/Government/5k/ta-si_final_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fa89f369317ba0bef43f7848043bc668edfd8510 --- /dev/null +++ b/Government/5k/ta-si_final_model/config.json @@ -0,0 +1,35 @@ +{ + "_name_or_path": "facebook/nllb-200-distilled-600M", + "activation_dropout": 0.0, + "activation_function": "relu", + "architectures": [ + "M2M100ForConditionalGeneration" + ], + "attention_dropout": 0.1, + "bos_token_id": 0, + "d_model": 1024, + "decoder_attention_heads": 16, + "decoder_ffn_dim": 4096, + "decoder_layerdrop": 0, + "decoder_layers": 12, + "decoder_start_token_id": 2, + "dropout": 0.1, + "encoder_attention_heads": 16, + "encoder_ffn_dim": 4096, + "encoder_layerdrop": 0, + "encoder_layers": 12, + "eos_token_id": 2, + "init_std": 0.02, + "is_encoder_decoder": true, + "max_length": 200, + "max_position_embeddings": 1024, + "model_type": "m2m_100", + "num_hidden_layers": 12, + "pad_token_id": 1, + "scale_embedding": true, + "tokenizer_class": "NllbTokenizer", + "torch_dtype": "float32", + "transformers_version": "4.30.2", + "use_cache": true, + "vocab_size": 256206 +} diff --git a/Government/5k/ta-si_final_model/generation_config.json b/Government/5k/ta-si_final_model/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..78d28ece999cde5a7c6b42f25fecffd56faa769f --- /dev/null +++ b/Government/5k/ta-si_final_model/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 0, + "decoder_start_token_id": 2, + "eos_token_id": 2, + "max_length": 200, + "pad_token_id": 1, + "transformers_version": "4.30.2" +} diff --git a/Government/5k/ta-si_final_model/pytorch_model.bin b/Government/5k/ta-si_final_model/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..190231679cadb37558800ee597ff208439cf67ec --- /dev/null +++ b/Government/5k/ta-si_final_model/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:514b4b13aa09eb0057bf0a96e6b01ecce5bab7c76216bdfd6a55180bab94a19a +size 2468870723 diff --git a/Government/5k/ta-si_final_model/training_args.bin b/Government/5k/ta-si_final_model/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..55e1b6ee7021cbc842d03b39e76ec0a56db7c65f --- /dev/null +++ b/Government/5k/ta-si_final_model/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de159f5472d5ac5a3eff3e344f32e6192cc0638b889a1d45777df360241fcb71 +size 4143 diff --git a/Government/5k/tok-en-si/sentencepiece.bpe.model b/Government/5k/tok-en-si/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/5k/tok-en-si/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/5k/tok-en-si/special_tokens_map.json b/Government/5k/tok-en-si/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/5k/tok-en-si/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/5k/tok-en-si/tokenizer_config.json b/Government/5k/tok-en-si/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fd1139979a7c7c4188b26d922af637f4bfb23e1f --- /dev/null +++ b/Government/5k/tok-en-si/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "eng_Latn", + "tgt_lang": "sin_Sinh", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/5k/tok-en-ta/sentencepiece.bpe.model b/Government/5k/tok-en-ta/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/5k/tok-en-ta/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/5k/tok-en-ta/special_tokens_map.json b/Government/5k/tok-en-ta/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/5k/tok-en-ta/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/5k/tok-en-ta/tokenizer_config.json b/Government/5k/tok-en-ta/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ef8f263366a29cf3dd0c97df4b45c41fbbf160b5 --- /dev/null +++ b/Government/5k/tok-en-ta/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "eng_Latn", + "tgt_lang": "tam_Taml", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/5k/tok-si-en/sentencepiece.bpe.model b/Government/5k/tok-si-en/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/5k/tok-si-en/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/5k/tok-si-en/special_tokens_map.json b/Government/5k/tok-si-en/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/5k/tok-si-en/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/5k/tok-si-en/tokenizer_config.json b/Government/5k/tok-si-en/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f80b41ef163bc0a47ebef66aa6c1210dfe0b9792 --- /dev/null +++ b/Government/5k/tok-si-en/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "sin_Sinh", + "tgt_lang": "eng_Latn", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/5k/tok-si-ta/sentencepiece.bpe.model b/Government/5k/tok-si-ta/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/5k/tok-si-ta/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/5k/tok-si-ta/special_tokens_map.json b/Government/5k/tok-si-ta/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/5k/tok-si-ta/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/5k/tok-si-ta/tokenizer_config.json b/Government/5k/tok-si-ta/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..386158eef85de789e966861e91bebf69e9a4f32b --- /dev/null +++ b/Government/5k/tok-si-ta/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "sin_Sinh", + "tgt_lang": "tam_Taml", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/5k/tok-ta-en/sentencepiece.bpe.model b/Government/5k/tok-ta-en/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/5k/tok-ta-en/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/5k/tok-ta-en/special_tokens_map.json b/Government/5k/tok-ta-en/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/5k/tok-ta-en/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/5k/tok-ta-en/tokenizer_config.json b/Government/5k/tok-ta-en/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6deaa799386cb7438bf8a0f96250f961c73dec30 --- /dev/null +++ b/Government/5k/tok-ta-en/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "tam_Taml", + "tgt_lang": "eng_Latn", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/5k/tok-ta-si/sentencepiece.bpe.model b/Government/5k/tok-ta-si/sentencepiece.bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..dc2262d3e1d375b235eb71c24119c8e73f85d4ad --- /dev/null +++ b/Government/5k/tok-ta-si/sentencepiece.bpe.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a +size 4852054 diff --git a/Government/5k/tok-ta-si/special_tokens_map.json b/Government/5k/tok-ta-si/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..02fba84e401dd2017999a39cac2c1278591c3a03 --- /dev/null +++ b/Government/5k/tok-ta-si/special_tokens_map.json @@ -0,0 +1,219 @@ +{ + "additional_special_tokens": [ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "ajp_Arab", + "aka_Latn", + "amh_Ethi", + "apc_Arab", + "arb_Arab", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "ceb_Latn", + "ces_Latn", + "cjk_Latn", + "ckb_Arab", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "est_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "pes_Arab", + "fij_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "grn_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "knc_Arab", + "knc_Latn", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kmb_Latn", + "kon_Latn", + "kor_Hang", + "kmr_Latn", + "lao_Laoo", + "lvs_Latn", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "min_Latn", + "mkd_Cyrl", + "plt_Latn", + "mlt_Latn", + "mni_Beng", + "khk_Cyrl", + "mos_Latn", + "mri_Latn", + "zsm_Latn", + "mya_Mymr", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "npi_Deva", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "gaz_Latn", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "pbt_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Beng", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "als_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tgl_Latn", + "tha_Thai", + "tir_Ethi", + "taq_Latn", + "taq_Tfng", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn", + "tzm_Tfng", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zho_Hans", + "zho_Hant", + "zul_Latn" + ], + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/Government/5k/tok-ta-si/tokenizer_config.json b/Government/5k/tok-ta-si/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9526d1cacd39f5ee55eece61fdacbbc6ce50136d --- /dev/null +++ b/Government/5k/tok-ta-si/tokenizer_config.json @@ -0,0 +1,25 @@ +{ + "additional_special_tokens": null, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "eos_token": "", + "legacy_behaviour": false, + "mask_token": { + "__type": "AddedToken", + "content": "", + "lstrip": true, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "model_max_length": 1024, + "pad_token": "", + "sep_token": "", + "sp_model_kwargs": {}, + "src_lang": "tam_Taml", + "tgt_lang": "sin_Sinh", + "tokenizer_class": "NllbTokenizer", + "tokenizer_file": null, + "unk_token": "" +} diff --git a/Government/combined results/government_crossdomain_all_directions_summary.csv b/Government/combined results/government_crossdomain_all_directions_summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..dc9489251a601657c6eb94fbe4d68f00c1cb1495 --- /dev/null +++ b/Government/combined results/government_crossdomain_all_directions_summary.csv @@ -0,0 +1,43 @@ +direction,data_size,Bible_chrf_sacrebleuapi,Bible_chrfpp_sacrebleuapi,Bible_bleu_sacrebleuapi,Bible_spBLEU-flores101,Bible_spBLEU-flores200,Bible_comet,Government_chrf_sacrebleuapi,Government_chrfpp_sacrebleuapi,Government_bleu_sacrebleuapi,Government_spBLEU-flores101,Government_spBLEU-flores200,Government_comet,OpenSubtitles_chrf_sacrebleuapi,OpenSubtitles_chrfpp_sacrebleuapi,OpenSubtitles_bleu_sacrebleuapi,OpenSubtitles_spBLEU-flores101,OpenSubtitles_spBLEU-flores200,OpenSubtitles_comet,NLLB_chrf_sacrebleuapi,NLLB_chrfpp_sacrebleuapi,NLLB_bleu_sacrebleuapi,NLLB_spBLEU-flores101,NLLB_spBLEU-flores200,NLLB_comet,Wikipedia_chrf_sacrebleuapi,Wikipedia_chrfpp_sacrebleuapi,Wikipedia_bleu_sacrebleuapi,Wikipedia_spBLEU-flores101,Wikipedia_spBLEU-flores200,Wikipedia_comet,Literature_chrf_sacrebleuapi,Literature_chrfpp_sacrebleuapi,Literature_bleu_sacrebleuapi,Literature_spBLEU-flores101,Literature_spBLEU-flores200,Literature_comet,Literature-Children_chrf_sacrebleuapi,Literature-Children_chrfpp_sacrebleuapi,Literature-Children_bleu_sacrebleuapi,Literature-Children_spBLEU-flores101,Literature-Children_spBLEU-flores200,Literature-Children_comet,Literature-Kathlyn_chrf_sacrebleuapi,Literature-Kathlyn_chrfpp_sacrebleuapi,Literature-Kathlyn_bleu_sacrebleuapi,Literature-Kathlyn_spBLEU-flores101,Literature-Kathlyn_spBLEU-flores200,Literature-Kathlyn_comet,Literature-Old_chrf_sacrebleuapi,Literature-Old_chrfpp_sacrebleuapi,Literature-Old_bleu_sacrebleuapi,Literature-Old_spBLEU-flores101,Literature-Old_spBLEU-flores200,Literature-Old_comet,News_chrf_sacrebleuapi,News_chrfpp_sacrebleuapi,News_bleu_sacrebleuapi,News_spBLEU-flores101,News_spBLEU-flores200,News_comet,Maths_chrf_sacrebleuapi,Maths_chrfpp_sacrebleuapi,Maths_bleu_sacrebleuapi,Maths_spBLEU-flores101,Maths_spBLEU-flores200,Maths_comet,Health_chrf_sacrebleuapi,Health_chrfpp_sacrebleuapi,Health_bleu_sacrebleuapi,Health_spBLEU-flores101,Health_spBLEU-flores200,Health_comet,FloresPlus_chrf_sacrebleuapi,FloresPlus_chrfpp_sacrebleuapi,FloresPlus_bleu_sacrebleuapi,FloresPlus_spBLEU-flores101,FloresPlus_spBLEU-flores200,FloresPlus_comet,Jailbreak_chrf_sacrebleuapi,Jailbreak_chrfpp_sacrebleuapi,Jailbreak_bleu_sacrebleuapi,Jailbreak_spBLEU-flores101,Jailbreak_spBLEU-flores200,Jailbreak_comet +en-si,baseline,0.2074466786941936,0.992404644366044,0.0114184838480725,0.0021871822261416,0.0023078106301359,0.351035983428359,1.9957824955315877,2.499977992441443,1.6800513225163616,1.212937758336134,1.055154151403294,0.5387619452029466,0.847093121279806,2.4684363097189745,1.396130818258411,0.1335446333113439,0.1565595240682873,0.6057251301407814,0.7585231071916326,2.0418635894125736,0.311198372233723,0.0900186221881229,0.1051094510834449,0.5752390958219766,0.7443504702963069,1.597579383453706,0.2952394162547146,0.1644145649375478,0.1569865791110672,0.4998222410529852,0.5405674952727352,1.72281406584829,0.2106596891674983,0.0213847448284255,0.0202065602410734,0.5059475341588259,0.5405674952727352,1.72281406584829,0.2106596891674983,0.0213847448284255,0.0202065602410734,0.5059475341588259,0.5405674952727352,1.72281406584829,0.2106596891674983,0.0213847448284255,0.0202065602410734,0.5059475341588259,0.5405674952727352,1.72281406584829,0.2106596891674983,0.0213847448284255,0.0202065602410734,0.5059475341588259,1.1743217071010077,1.713707163740193,0.7760985154863841,0.4451472768674177,0.3925480474445332,0.5711525277197361,3.386902647626311,5.396413898661497,7.043350945997881,4.407736007891258,4.186116810472018,0.5152128710597753,2.1293772294034072,2.6482080421096397,1.177958280471317,1.235578054683879,1.073537231998086,0.4134376797113526,0.783503884977641,1.7245734048828818,0.5241960228541523,0.2973223112210829,0.3139251616408782,0.4667224814533716,2.7299557155007927,3.4342239238448125,3.070198561059441,1.377379113964754,1.4588223666474471,0.519491742938757 +en-si,1k,23.516304960748695,20.362706332865574,1.6177223328136954,7.698804887228647,7.142460377516814,0.6451798568218946,7.693363479451183,6.136089531064441,0.0203424558411158,0.1361625082452787,0.2708677194690141,0.3770179857909679,32.830743778102054,30.83796375018137,8.46478022121088,15.040401102261876,14.104804439363615,0.7946870295628905,12.324043069958456,10.868875229575847,0.8902522416379338,1.8066512013725395,1.8088916748363504,0.5654888506523086,8.11906180769877,6.549857745501184,0.1237060414889934,0.433680415977373,0.5140921731137955,0.3466547068357468,10.20057431150489,8.942477418219665,0.835031192198746,1.6339000097540557,1.521072472920624,0.5377916059345007,10.20057431150489,8.942477418219665,0.835031192198746,1.6339000097540557,1.521072472920624,0.5377916059345007,10.20057431150489,8.942477418219665,0.835031192198746,1.6339000097540557,1.521072472920624,0.5377916059345007,10.20057431150489,8.942477418219665,0.835031192198746,1.6339000097540557,1.521072472920624,0.5377916059345007,9.31829164832508,7.602404261012937,0.2071728739300219,0.7506842504471134,0.9476541691597596,0.4086479433476925,7.135553179408386,5.990287751321683,0.2387540998454511,0.5156325949052365,0.4899032462176094,0.3933246617615223,7.943758973222321,6.379120539858632,0.0276695009220651,0.1918128396954742,0.2125325925109905,0.3151403463288639,7.215223944683759,5.744608434304767,0.048936397246092,0.1648504019946428,0.2229497306164265,0.320197635400201,9.44752151854744,8.10370681971603,0.5929600023807652,1.2132553605616745,1.272400283286709,0.4513409103006124 +en-si,5k,17.38218305920879,14.720861483249708,0.6110980468231604,4.57594442109759,4.215972757003348,0.5760666883587837,8.529184619241422,6.817795023193822,0.1368056848250783,0.7307413818879813,1.2547235687803515,0.499934135377407,18.823833403766027,17.25520904752903,2.8231739289631452,6.079735336389598,5.4618216841043665,0.6699169592484832,8.980534923916675,7.525372340613211,0.4358489858308041,1.072042727942755,1.0162517113668066,0.5183496097743482,8.871693845189547,6.993505824594849,0.0277429753469733,0.1645125930163549,0.2248596007991863,0.4438680990338325,8.063824155188607,6.555031221691862,0.1819747397416745,0.3778547201497972,0.3628863984872553,0.4679325869381427,8.063824155188607,6.555031221691862,0.1819747397416745,0.3778547201497972,0.3628863984872553,0.4679325869381427,8.063824155188607,6.555031221691862,0.1819747397416745,0.3778547201497972,0.3628863984872553,0.4679325869381427,8.063824155188607,6.555031221691862,0.1819747397416745,0.3778547201497972,0.3628863984872553,0.4679325869381427,8.064029365713953,6.348901418473853,0.0296985566814461,0.2990027650985308,0.4486387708986559,0.4363476236760616,6.519839470122508,5.185081036326267,0.0489337140600784,0.2011519908200932,0.1864243957831318,0.4449171331226825,6.1429329243101805,4.977345281893618,0.0372134062417059,0.1113953189532505,0.2195096770378732,0.3968002018131566,8.058227939320835,6.3488511990142,0.0270268025523735,0.1763660195637698,0.2460073296279243,0.4096863102830446,6.527991317587816,5.434436017464176,0.1679797515935076,0.7485159688817504,0.9217368558347367,0.4593092997223139 +en-si,10k,9.883167308842296,8.08164852593724,0.1224908082681523,1.6782909910263053,1.51688499860891,0.4502922486960888,7.931510211422728,6.177940517062402,0.1438249418127693,0.6302031145633212,1.0312510931564078,0.4728857491314411,9.800694865178825,8.375756621578411,0.5730870903662663,1.4827913789729208,1.4035582820664598,0.5102573437988758,7.869460496102654,6.09056504503559,0.1053689439449828,0.4985236041396916,0.5363278084194655,0.4601839078451714,7.275620508536446,5.622158052913327,0.0214232608019175,0.1101172691185725,0.1595470926438534,0.4203639453500509,6.376891953361408,4.851865039223764,0.0180506833167269,0.0442598824851941,0.0643229995316287,0.42568679946661,6.376891953361408,4.851865039223764,0.0180506833167269,0.0442598824851941,0.0643229995316287,0.42568679946661,6.376891953361408,4.851865039223764,0.0180506833167269,0.0442598824851941,0.0643229995316287,0.42568679946661,6.376891953361408,4.851865039223764,0.0180506833167269,0.0442598824851941,0.0643229995316287,0.42568679946661,6.783172487448554,5.22918019278509,0.0140679334862007,0.1930584845597379,0.3257971104827631,0.4174291704893112,6.786315824711309,5.199195420389427,0.0153743647953712,0.0221255940232247,0.0442699082295149,0.4291429179757833,5.863895093672576,4.60535828879025,0.0095053759705964,0.0314906975713339,0.0741187821838944,0.3987277927693356,7.101437099437691,5.433916452424211,0.0060960765752617,0.0651761491160575,0.1203964169392692,0.4025747983646487,6.071189768294846,4.833782795198674,0.041475688531638,0.1170244111598882,0.1268903339098027,0.4328604847192764 +en-si,15k,10.333521780820629,8.453661202177699,0.0665625178426419,1.5401320162594443,1.3016910398397168,0.4509024264812469,9.731813599282969,7.8013772560248285,0.1314581386354406,0.3657227959232438,0.4982705083406589,0.3988617490530014,6.577743041669929,5.300362655175006,0.1003228485111686,0.3516569502132318,0.3133280497660109,0.4609458845481276,6.324312311847597,4.99984908358139,0.0638513895650396,0.2349258100220535,0.2378402586714434,0.4581684144181248,9.591036118587716,7.478943564953437,0.0147339518621636,0.0934519961296255,0.0877371244631593,0.374054478764534,5.80325024079758,4.518580409004698,0.0262864967808605,0.0270903484808721,0.0435175861915902,0.4377163580656051,5.80325024079758,4.518580409004698,0.0262864967808605,0.0270903484808721,0.0435175861915902,0.4377163580656051,5.80325024079758,4.518580409004698,0.0262864967808605,0.0270903484808721,0.0435175861915902,0.4377163580656051,5.80325024079758,4.518580409004698,0.0262864967808605,0.0270903484808721,0.0435175861915902,0.4377163580656051,7.475587226092283,5.872985418064176,0.0132616503047819,0.093916072142558,0.1147211509808602,0.396491152331233,6.837063431902609,5.473992424430646,0.0169743394047959,0.0240623100054295,0.016119390622961,0.4090483323037624,9.423835346437755,7.569232714787168,0.0166801104087271,0.1045040222430923,0.1066884802702877,0.3727596676751469,8.110053421354792,6.406640947709287,0.0468008003620898,0.134156015523987,0.1478120657477304,0.3853278315880082,5.374077312754306,4.392153280587568,0.0609208628011329,0.0885308924521573,0.1160557479024545,0.4336004293262958 +en-si,20k,31.761604881464233,27.54796001571456,2.7451206165921933,13.882638289577836,13.98031568061058,0.7690197389423847,57.21169238265881,52.83729260161385,22.43693673311732,43.73262113530742,50.00083444741132,0.9225092524886132,35.12391260428374,32.105489417981794,7.502054254124334,19.84046279426906,22.19210912077999,0.8655002781748772,42.8673764543524,39.32278272621688,11.347460755796227,23.681411463770186,27.9130563227134,0.9010286902821903,41.55216385298722,37.118928968163544,7.435175493487085,22.869206294691708,26.457512815265737,0.8730469604730606,30.323146952610823,27.219078742914693,4.997742728382483,11.572875707289988,13.355103424191404,0.8145392687618732,30.323146952610823,27.219078742914693,4.997742728382483,11.572875707289988,13.355103424191404,0.8145392687618732,30.323146952610823,27.219078742914693,4.997742728382483,11.572875707289988,13.355103424191404,0.8145392687618732,30.323146952610823,27.219078742914693,4.997742728382483,11.572875707289988,13.355103424191404,0.8145392687618732,45.4274593270867,41.001988715490064,11.590726452242302,27.283072483750267,32.96372236158503,0.879407027348876,40.16798911664159,38.07374908008929,16.656258371139497,25.488915980680275,27.846527606251858,0.8411055938601494,43.19499688941104,38.93219459599893,11.245618379250637,26.743125840871944,29.755853396932068,0.8339593650082524,43.34636755091586,39.07131672687825,9.227334131840172,25.83282090100737,29.605976140051983,0.8745417469604448,41.75732488247362,38.2163434368176,12.26004639904115,22.983043562325175,25.83915085648592,0.8406499291956425 +en-si,25k,34.456340875004464,29.91541262040702,2.927627634611363,16.043010632978344,16.549307363154742,0.7995621707439423,57.77575685070945,53.38916478804416,22.913657969171457,44.63136119938347,50.79563281416712,0.9236653954982758,34.74392924639648,31.722146731556744,7.22518423376036,19.630955821278032,21.327917510045417,0.8652439583539963,42.319887007329434,38.80384792742494,10.908557257224446,23.233545972292767,27.473380399198323,0.901711478390458,42.11352299493802,37.60982662762706,7.471102785714683,23.41732987399879,26.992666872334663,0.8773687633275986,30.537553612782077,27.457684801211464,5.352300825495501,11.77648751639114,13.60866615715853,0.8180207158476114,30.537553612782077,27.457684801211464,5.352300825495501,11.77648751639114,13.60866615715853,0.8180207158476114,30.537553612782077,27.457684801211464,5.352300825495501,11.77648751639114,13.60866615715853,0.8180207158476114,30.537553612782077,27.457684801211464,5.352300825495501,11.77648751639114,13.60866615715853,0.8180207158476114,45.25266583842741,40.82924968374386,11.472613916243573,27.13425842456199,32.92937573986284,0.8772459758222103,40.1331593949345,37.9749405806341,16.160581021507667,25.310463847844968,27.7360906142369,0.8429196584969759,43.25096129715229,38.92367941876307,10.935473968452124,26.883374289624214,29.927259118058668,0.8349770524361161,43.53881729774115,39.32453614210059,9.495034487221888,26.098641019137556,29.645824704158496,0.8745844934886623,41.94530390315914,38.36738795492778,12.134369099259992,22.955178169349796,25.91187375950136,0.8430828390717506 +si-en,baseline,45.7982751634026,44.93582368126017,22.655268358949783,24.23433589434345,23.81285028233166,0.7768540764153004,32.43705692240101,30.84695416771855,14.55282284869292,14.241602139556662,13.844908765044012,0.6612490080296993,0.2093695928524757,0.4610373422708847,0.2172444536953513,0.0348241300282007,0.0354425516877704,0.5909831962287426,4.342796789294033,4.749983204532853,2.807257100878493,2.5871081712100445,2.45253724891628,0.6258792190551757,29.78788601544511,28.904571784948065,14.614483505858551,15.328936597680402,15.164980042762892,0.65542942070961,12.68270519652462,12.506382621891632,4.647287447640901,4.748886789179166,4.84982681362838,0.5844122405648231,12.68270519652462,12.506382621891632,4.647287447640901,4.748886789179166,4.84982681362838,0.5844122405648231,12.68270519652462,12.506382621891632,4.647287447640901,4.748886789179166,4.84982681362838,0.5844122405648231,12.68270519652462,12.506382621891632,4.647287447640901,4.748886789179166,4.84982681362838,0.5844122405648231,28.476586976813845,27.171542359315197,12.62598973960596,12.65714651911421,12.389014001011189,0.6620795749127865,11.317448157669336,12.51008597255433,10.716065134149002,9.704364195477536,9.059869661837732,0.5851124048382044,44.92138874126114,42.792813526751225,20.948990193440864,24.18940878364083,23.40765789077685,0.7220131568694382,22.55840081391738,22.048393660536043,11.962472512396154,11.919058023999186,11.377242895806102,0.6375348256687402,13.70091286495427,13.301510063094474,7.9429792134790675,7.060747562369329,7.125536393775505,0.5843882857114077 +si-en,1k,45.519679436405895,44.56427531232765,21.64082810028898,23.12556477445832,22.756168684278343,0.7835294347405434,57.7801523647287,54.89723004749681,28.536492739972218,28.88016835917535,30.227912618064472,0.8545789907276631,64.47308072896288,63.74329046003354,43.67043920882924,43.98564205774316,44.25832162381529,0.8969421654343604,63.41284019658625,62.61060550849287,42.91050967755352,42.32787525245691,42.58545376539262,0.9012708021769568,55.132224891054925,53.208937399375245,27.568990711729228,29.601467860214452,30.229356099978883,0.8444438032507896,44.48527430359607,43.45372079058745,20.5791049450655,19.968255129646845,20.22726305440647,0.8111989430785179,44.48527430359607,43.45372079058745,20.5791049450655,19.968255129646845,20.22726305440647,0.8111989430785179,44.48527430359607,43.45372079058745,20.5791049450655,19.968255129646845,20.22726305440647,0.8111989430785179,44.48527430359607,43.45372079058745,20.5791049450655,19.968255129646845,20.22726305440647,0.8111989430785179,54.5944648728102,51.89513492854887,23.597437634350108,25.583373745829206,26.626821242478208,0.8546181761026382,58.57429503758378,57.988276525550745,39.17008812369143,39.31751521238576,39.553216712025744,0.8474531008601188,58.823952962234195,55.76889243030728,24.96873919761974,31.08439877453402,31.533984745001874,0.85347719641214,58.104840438076465,56.060526082433626,30.543987665322973,32.04481816309639,32.80023559608612,0.8685261862786863,54.9625739402548,53.21327625787566,30.63372234457116,30.852209509922588,31.487794552918498,0.8601958205401897 +si-en,5k,45.33522734519968,44.4217335563957,21.733600421347013,22.91738522167107,22.648938874611055,0.7763690453767776,60.99733266989447,58.223940529836106,32.76016680774736,34.19087842338811,35.339936404370995,0.8665921680033207,63.40878654318058,62.604126471230536,41.30336470327033,42.09400841117228,42.33976466887083,0.8930004394203425,62.78110346546808,62.05912069488389,42.21173315676192,41.70838449001414,41.91348074095811,0.9005061542755713,54.013321807431225,51.992516773370546,25.638065555853593,28.404409789625227,28.94878056957761,0.8408416794836521,43.69492188032749,42.61634074514172,19.82485043746595,19.36560532468245,19.55606985774085,0.807296268671751,43.69492188032749,42.61634074514172,19.82485043746595,19.36560532468245,19.55606985774085,0.807296268671751,43.69492188032749,42.61634074514172,19.82485043746595,19.36560532468245,19.55606985774085,0.807296268671751,43.69492188032749,42.61634074514172,19.82485043746595,19.36560532468245,19.55606985774085,0.807296268671751,55.00139455274541,52.31023841866155,23.658658442202828,25.794596124102387,26.69403926211541,0.8537135955691337,57.49368058033715,56.93484539279011,38.38131159782538,37.96777084280895,38.54294766248619,0.8448873843252659,57.73196066859465,54.65553290904695,23.68447635480341,29.445175969422657,29.92273280986862,0.8502491591686613,56.64196178285397,54.44740562310189,28.154022271469724,29.694605434536683,30.57014291561554,0.865584895898231,54.680471231588754,52.920082556983104,30.35033519201316,30.535097787680854,31.254736434782973,0.8590239198207855 +si-en,10k,45.07330718302318,44.13769071062262,21.65309928276407,22.884942868586275,22.55774563190717,0.7763391099870205,62.73844567044389,59.981579805973475,34.628357862719746,36.654224341305095,37.66515942097653,0.8708139849305153,62.57068956655811,61.76668331962446,40.21677890085725,41.20475903534689,41.44644800048119,0.8913131302297115,62.2963419600587,61.51956040246825,41.345476841818595,41.07059044087541,41.2644213431573,0.8979451639677248,54.56450179765024,52.47018091510572,25.835169532912634,28.99903907364859,29.498197853102567,0.8424103186130524,43.69342098316437,42.546686819581005,19.19400950178123,18.807084984445588,18.98680461063105,0.8059120564162732,43.69342098316437,42.546686819581005,19.19400950178123,18.807084984445588,18.98680461063105,0.8059120564162732,43.69342098316437,42.546686819581005,19.19400950178123,18.807084984445588,18.98680461063105,0.8059120564162732,43.69342098316437,42.546686819581005,19.19400950178123,18.807084984445588,18.98680461063105,0.8059120564162732,55.16912437882716,52.41452548941682,23.225707785715844,25.69047415193137,26.629105316212765,0.8533352153301239,57.016858143481265,56.42857018784528,37.39970757061199,37.218307322272175,37.74691390025642,0.8421808927953244,57.48438223269277,54.35155986906376,23.155730792464308,29.116973964187466,29.53171280944268,0.8489664266953307,56.99668379741255,54.809779080789056,28.151436901598544,29.99779167953794,30.877600166083894,0.8655534790085239,53.9237987618724,52.13672088503674,28.302042608214855,29.449375428703405,30.324157627931157,0.8554237298518419 +si-en,15k,45.43490212145097,44.461716905870546,21.836615165017935,23.113368684205422,22.79028610179947,0.7760936993658543,63.66743657611551,60.907282231187885,35.963847478597884,38.062048194354155,39.14898865638359,0.8755780427753925,62.309569067666494,61.51447385810135,39.85246405730881,40.53007397285569,40.78251697321025,0.8903046472668648,62.13359523376011,61.36838945744591,41.138863850499035,40.83376787968107,41.02992900037648,0.8981701793280233,54.596528187164026,52.52762694790201,25.839339896954755,29.11411895865148,29.585157945833636,0.8438672468662262,43.36910379797703,42.289734486998555,19.171156780144635,18.83634896985832,19.03263358047044,0.8026190398931503,43.36910379797703,42.289734486998555,19.171156780144635,18.83634896985832,19.03263358047044,0.8026190398931503,43.36910379797703,42.289734486998555,19.171156780144635,18.83634896985832,19.03263358047044,0.8026190398931503,43.36910379797703,42.289734486998555,19.171156780144635,18.83634896985832,19.03263358047044,0.8026190398931503,54.79661168252055,52.118872979463774,23.385029031710246,25.404756406612243,26.521867470756305,0.8521652072072029,57.310069958060105,56.76185730321125,38.43488474377299,38.07576167493571,38.69220960171209,0.8414851614534855,57.449860202434664,54.331743280233425,23.193473569032133,28.956658292795417,29.306935414676797,0.8493524404054278,56.66471171297911,54.55280182410841,28.147612954372494,29.996577761750903,30.809015410443397,0.8639266135782121,54.27109503581603,52.4599949261298,29.295788397067184,29.68282584631609,30.324293926971475,0.8559750660508871 +si-en,20k,44.91420789245757,43.873439137375634,21.301441819365863,22.351071106075548,22.206533266775025,0.7740990107506514,64.29823445582777,61.601305651445806,36.75204629114596,38.70142850164052,39.84540391536018,0.8764319905340672,61.747554780003625,60.91758251594824,38.6864159002422,39.59130143461301,39.86300529902638,0.8888738882243633,62.05599317037146,61.280281004093354,40.91603177497263,40.8178257183554,41.027606741137376,0.8979786402037664,54.349890181366554,52.24562779681402,25.14899465512415,28.50412595867878,28.93093362503844,0.8428374643325806,43.40740277127494,42.345012525977396,19.0703681106256,18.888681230286217,19.042522371038515,0.8063090334832669,43.40740277127494,42.345012525977396,19.0703681106256,18.888681230286217,19.042522371038515,0.8063090334832669,43.40740277127494,42.345012525977396,19.0703681106256,18.888681230286217,19.042522371038515,0.8063090334832669,43.40740277127494,42.345012525977396,19.0703681106256,18.888681230286217,19.042522371038515,0.8063090334832669,55.56985871516454,52.88673440348383,23.521531094749506,26.347645725152898,27.213331193706768,0.8544390945136547,57.20566080895111,56.58499485696334,37.49051482258888,37.57353683685339,38.15952997638359,0.844492649435997,57.10297271238651,54.03252746431237,22.972128657325474,28.48771281175676,28.890848978998367,0.8467513275280427,56.516526222256225,54.31503846883062,27.52217835877474,29.48182997817969,30.23545759180062,0.864110741985174,53.41263845863003,51.60699080241575,27.49977320692965,28.729020323413017,29.376364538435467,0.8524583500027657 +si-en,25k,45.44171048157175,44.52892482269925,21.98176525247642,23.20753380936823,22.94907765149802,0.774605580329895,64.58857041890641,61.87377282714578,37.248262998779424,39.46165914062977,40.52310463151705,0.8774390577673912,62.28766815442771,61.51414472315769,39.702805722647234,40.72100110310309,41.010220117202856,0.8896081002503633,61.81986238385338,61.0575686551013,40.87429966556002,40.70235601434973,40.87055641010563,0.8976878566082513,54.33205209749793,52.23850858529763,25.520181497053553,28.6899226315677,29.227820042803735,0.843238550812006,43.328827400718104,42.21937887996792,18.677081635943036,18.580052705829587,18.800094198809916,0.8038805679678916,43.328827400718104,42.21937887996792,18.677081635943036,18.580052705829587,18.800094198809916,0.8038805679678916,43.328827400718104,42.21937887996792,18.677081635943036,18.580052705829587,18.800094198809916,0.8038805679678916,43.328827400718104,42.21937887996792,18.677081635943036,18.580052705829587,18.800094198809916,0.8038805679678916,55.619649765821144,52.95920751394399,23.96108915448688,26.647218304162458,27.498005055406196,0.8542312571704388,57.0557577035196,56.463857046699296,37.12052800347161,37.13924306449029,37.77450764408659,0.8438830208182335,57.212120300084024,54.15630021537241,23.175948602163185,28.811125019591863,29.158821803333304,0.8477959724289648,56.731496713185145,54.60594823245899,27.95596981666909,29.922833678434063,30.64866952047091,0.8645521375854968,53.88183375650033,52.08565740531584,28.428531036040155,28.957961328026386,29.46748347959513,0.8550191217660904 +en-ta,baseline,0.1820428517840199,1.1067003517608265,0.0110823711907671,0.0025371006574629,0.0027115722959845,0.3555766829550266,2.7326091482853245,3.4402962062567553,1.8190494396216712,1.5257398406012643,1.6493600666293629,0.533683074772358,0.7106221154527631,2.570921700813326,1.4788269145982211,0.1892647228888548,0.2536608589994819,0.6126294483244419,0.8396751780130931,2.4111867421786233,0.3066246178883174,0.2813822906072251,0.3096680563225658,0.5835503396391869,3.864874288629729,4.624060876786521,1.448036721709419,2.567352063547686,2.708872369094082,0.5320055771023035,0.4308148131801461,2.023331196074034,0.1512175048726044,0.0215280175368612,0.0231630885510583,0.4852443312432755,0.4308148131801461,2.023331196074034,0.1512175048726044,0.0215280175368612,0.0231630885510583,0.4852443312432755,0.4308148131801461,2.023331196074034,0.1512175048726044,0.0215280175368612,0.0231630885510583,0.4852443312432755,0.4308148131801461,2.023331196074034,0.1512175048726044,0.0215280175368612,0.0231630885510583,0.4852443312432755,1.998127703773364,3.3012853643443663,1.101227424070257,1.5735222160523783,1.6449114728899694,0.5120277152841829,2.8081650520647368,5.5836312752642145,6.501583604993177,4.206994549527542,4.512365149250633,0.5298982553035021,1.4038432938609309,2.4884344633513176,1.4751634938100298,0.8742299437387782,1.0600629870636833,0.4197459498494864,2.3375772397461256,3.339293286068496,1.0137212343306958,1.5814583844970087,1.5903384046591815,0.4869961341614779,0.950902745891338,2.5102398659453193,2.870091446825767,0.8270928935658232,1.170507674721423,0.5279000070542097 +en-ta,1k,34.9595922976334,30.016010590575693,3.6955114039946846,11.425554236738714,10.234295696241674,0.7152103856056928,51.49403723919621,45.43089824989166,13.736301489950655,24.093670097241016,24.023475896842484,0.8820261645913124,23.543217403368963,22.582984314652695,8.67274438400228,14.710985334183531,14.895692831927844,0.6994686460494995,52.48340779674099,48.03652080074238,17.20047536975647,29.440567616607588,28.722471108002285,0.8617566767791719,56.255478335036344,49.83306218734586,13.02195300864623,30.373405621589413,30.195715254109636,0.8780179094821214,53.89895728855406,48.89007726591565,14.649069178790135,28.643382408052908,28.11198441735683,0.8140736782785367,53.89895728855406,48.89007726591565,14.649069178790135,28.643382408052908,28.11198441735683,0.8140736782785367,53.89895728855406,48.89007726591565,14.649069178790135,28.643382408052908,28.11198441735683,0.8140736782785367,53.89895728855406,48.89007726591565,14.649069178790135,28.643382408052908,28.11198441735683,0.8140736782785367,61.18633229487469,55.994558658194705,21.79529355470753,39.77194115398926,39.28750945693206,0.852673879768475,52.00734110365251,48.036037740621786,21.279910114693163,28.573803977208566,28.00619078972203,0.8331779134869576,60.68331352514558,54.275432087342246,16.359838256117694,35.505708648653545,34.3043864590842,0.8473187069296837,53.865656846497664,47.80073113672015,12.633814796264634,28.59163422478019,28.399945483860275,0.8676608577665132,58.10779627770626,53.1788446442861,21.707498859768968,35.19240880440027,34.731288781745924,0.8380884910523891 +en-ta,5k,39.104270196186626,33.16444374880192,2.8756653381343487,13.64369627504619,12.35112913731052,0.7828839690685272,57.48870752318605,51.33326923292132,17.451089018577015,32.797904822354056,32.88863987169818,0.892828518897295,48.270452924617146,44.54937233725686,15.715368181882,27.295537890775297,27.037730056270444,0.8486921544075012,56.013767992105265,51.10331545240099,17.49474037237022,30.74849893729091,29.937989572443307,0.9035753657594062,54.76977526202764,48.428196118589256,11.978872103962386,29.1431958843411,28.692988400969934,0.874132488772273,52.73975783803249,47.36067825923689,12.8060934521314,26.16715572899859,25.169424200290603,0.8350815981867965,52.73975783803249,47.36067825923689,12.8060934521314,26.16715572899859,25.169424200290603,0.8350815981867965,52.73975783803249,47.36067825923689,12.8060934521314,26.16715572899859,25.169424200290603,0.8350815981867965,52.73975783803249,47.36067825923689,12.8060934521314,26.16715572899859,25.169424200290603,0.8350815981867965,58.92559041177843,53.330231600017186,17.836532112029595,34.421406579635025,33.976456445227754,0.8648977399351128,50.6870615642952,46.644072448648,20.39240768504329,26.561548371984117,25.92481296568236,0.8300952911973,58.66871165732608,52.11839595334895,13.722466517727376,31.967876341824944,30.735447764073637,0.8376800656765699,52.03516812879777,45.77780503134915,10.47506217472233,25.702371506369836,25.19030750123965,0.8678937781705216,56.912620918833454,51.79601398850566,19.88472608153632,32.20857214956527,31.37671701700715,0.8543761656731367 +en-ta,10k,39.04016452320649,33.15214888174151,2.8184624261362243,13.751787694216905,12.653757012332289,0.7956526825428009,59.06764975646131,52.913556093013966,18.8233242367348,34.91316090158198,34.99057592315347,0.8964461959302426,44.84399057895997,41.46688671504414,14.827865536541932,25.05746381065306,25.058395301870657,0.826336010709405,55.701610270706645,50.770739016004974,16.868370570975276,29.73773902223746,29.018614315651423,0.902689752738534,53.76769626572674,47.42992356678965,11.011035596320127,27.81871954823268,27.688381316325337,0.878192524343729,52.80011373193968,47.35622145271364,12.741313369670015,26.446102922675102,25.894374314433485,0.8346316877890516,52.80011373193968,47.35622145271364,12.741313369670015,26.446102922675102,25.894374314433485,0.8346316877890516,52.80011373193968,47.35622145271364,12.741313369670015,26.446102922675102,25.894374314433485,0.8346316877890516,52.80011373193968,47.35622145271364,12.741313369670015,26.446102922675102,25.894374314433485,0.8346316877890516,58.12530632693921,52.442792795931496,16.683589966304222,33.32834972613715,33.03293022848524,0.8653240330876356,50.45288363552336,46.509719373423394,20.235962579515416,26.069489072072415,25.576828778799833,0.8329620827734471,58.33440706989346,51.67512736768906,13.453431672446712,31.75238413859764,30.682964623161613,0.8420447525680065,51.33106002932208,45.11555555810329,10.204760623961306,24.73892903928464,24.451190328689275,0.8639418881519039,56.21916360382115,51.056636792979745,18.72396709640088,30.6603151104876,29.946766608740884,0.8520219075381756 +en-ta,15k,39.35146130939026,33.3893609747557,2.6919493432304913,14.209230774547496,12.92935046957307,0.798079612314701,60.32710475528184,54.11043139111106,20.73495912890604,36.29463437793894,36.41417193218862,0.9002067874670029,51.053005804755614,47.20893438933282,17.80236231837347,29.0229461422151,28.7448018857421,0.8637542776614427,55.95602263572816,50.96604446578231,16.97995308261609,29.940204717375405,29.06211337216374,0.9060601958381016,54.147304763835265,47.77457877302795,11.134996124969732,27.911489519257888,27.664231157728317,0.8771339770406484,53.330662605200885,47.977488579298054,13.7430360924881,27.0984453930422,26.13129626962484,0.8352127524989622,53.330662605200885,47.977488579298054,13.7430360924881,27.0984453930422,26.13129626962484,0.8352127524989622,53.330662605200885,47.977488579298054,13.7430360924881,27.0984453930422,26.13129626962484,0.8352127524989622,53.330662605200885,47.977488579298054,13.7430360924881,27.0984453930422,26.13129626962484,0.8352127524989622,58.18240480822938,52.583530136995606,16.883673473235667,33.380973106643324,33.202182667126806,0.8685271259711449,50.41074236607987,46.39616223416936,19.92487952308717,26.499396868002357,25.886182013385874,0.8344578335285187,58.100641671237,51.51679561098018,13.729242014213668,30.92716962659006,29.867780694387307,0.8373897092342377,51.603214168549805,45.46102073550337,10.678829148020034,24.980858657267618,24.55441496505266,0.8630498663886733,57.19502017847633,52.106862659615615,20.146883816765214,32.03345035713335,31.151251845438047,0.8521424018740654 +en-ta,20k,38.28284821810499,32.467935193315014,2.620634215953889,12.909023254023582,11.71730023876448,0.7924355041086674,60.832506487062155,54.728394037524666,21.977958296988536,37.40296049724705,37.44162196006756,0.901136607259512,48.443547247449175,44.6555521776317,15.59176727548575,27.03275688100976,26.801265810086395,0.85416314214468,56.04076681501125,51.08520502581324,16.989963735299348,30.094701617954517,29.162302264301612,0.905110799828828,53.91530090818076,47.48323717505256,10.894641087837911,28.106499127369172,27.98331556305807,0.8761209879219533,52.47163916860495,47.0332261198968,12.643277181350124,25.68446427296771,24.678714971778387,0.8305922095077457,52.47163916860495,47.0332261198968,12.643277181350124,25.68446427296771,24.678714971778387,0.8305922095077457,52.47163916860495,47.0332261198968,12.643277181350124,25.68446427296771,24.678714971778387,0.8305922095077457,52.47163916860495,47.0332261198968,12.643277181350124,25.68446427296771,24.678714971778387,0.8305922095077457,57.74674327800203,52.02965375140137,15.87429821987729,32.689456054430885,32.341695161283745,0.8664364232475977,50.17759637654757,46.10077286825023,19.552887677839095,25.944749033962783,25.191118400261335,0.8320967829525471,57.864730475266526,51.2403734299827,13.537007166607967,30.4877518725062,29.57607711885231,0.8412932092249393,50.54670262782469,44.38126135159692,9.807197413498848,23.720731513652108,23.272251376965823,0.8616439145161465,56.40288106711744,51.35929482121719,19.36945799493403,31.485539332141823,30.495628128187413,0.8543994006514549 +en-ta,25k,37.910253679604686,32.01692922260542,2.4064603779889935,12.96195055037878,11.953366145494822,0.7995613397806883,61.33114874800577,55.22221826862441,22.221961258105324,38.15551868828684,38.25538259594189,0.9011772488355636,49.96828470037169,46.0653176059842,16.388551708689157,27.76078544768291,27.583351063247836,0.859308678522706,55.89077974807722,50.92405687648808,16.77937992038363,29.683968294312574,28.96596565900932,0.9056717679016124,54.66213735077503,48.341725778921536,11.891592515162682,28.933939483948446,28.79674392890101,0.8784477740079164,52.94294062702861,47.48633204257806,12.65209411669922,25.921457143150043,25.42314085711707,0.8307426627380429,52.94294062702861,47.48633204257806,12.65209411669922,25.921457143150043,25.42314085711707,0.8307426627380429,52.94294062702861,47.48633204257806,12.65209411669922,25.921457143150043,25.42314085711707,0.8307426627380429,52.94294062702861,47.48633204257806,12.65209411669922,25.921457143150043,25.42314085711707,0.8307426627380429,57.96189383266107,52.355247567462946,16.7353650809541,33.13264694861089,32.75953716452201,0.865638548770104,50.43671068611862,46.27008156145329,19.451039210384923,25.95066408493507,25.35578743333428,0.8351800081133842,57.770063459033786,51.13982650874356,13.649049880812218,30.540986543530195,29.445217779048033,0.8380835425704718,51.168533619903165,44.988690975617224,10.36678989899248,24.3163674774914,23.88797990096679,0.8646409833148534,56.14701377675067,51.156393091774014,18.472593105129725,31.029411447858458,29.72512132689668,0.8519543393552304 +ta-en,baseline,39.47622083734604,38.535686343285626,16.80805320253603,18.917533482605823,18.198721079028463,0.7528944167643785,52.3529295476339,49.38957776608853,22.983161463191124,22.78860494341881,23.888948978622665,0.8210956664085388,23.287211611003936,22.38927432915715,11.122744297292002,9.407024275001389,10.92931256708631,0.6389645711779595,42.07568030691544,40.858521039377784,26.18871728655078,23.279777518510983,25.91722171225571,0.7494174346327782,59.68646735470296,57.866382438661454,33.204412644274505,34.89017215436297,35.637236082588075,0.8468928661346435,33.532064508931796,32.15883836468728,14.273128624048406,14.123921514861046,15.284138938637296,0.6710098042256303,33.532064508931796,32.15883836468728,14.273128624048406,14.123921514861046,15.284138938637296,0.6710098042256303,33.532064508931796,32.15883836468728,14.273128624048406,14.123921514861046,15.284138938637296,0.6710098042256303,33.532064508931796,32.15883836468728,14.273128624048406,14.123921514861046,15.284138938637296,0.6710098042256303,55.95338991980899,54.10241056867392,32.40037756016967,33.277017412245605,34.37400931232348,0.8045579167195114,48.55646207985666,47.844840747445275,30.852783835081933,30.937461958666827,31.65396680699585,0.7837131512463092,60.18050958026944,57.52052470528929,30.45956583501814,33.85661078371377,34.56109868596526,0.8350872081518174,55.356985866580544,53.27514272756617,28.884337353160277,30.380139274020152,31.23775259989815,0.843577752648135,42.33599917092071,40.69509676471764,23.927384098520275,22.840365209874985,23.52094887359705,0.7381379562616348 +ta-en,1k,39.75161387490352,38.70065033403796,16.618038541811128,18.634568567182622,18.002394763247366,0.756501064568758,56.05655518747233,53.31425305044379,27.395680142968754,27.31640954628235,28.43727114616367,0.8340802385210991,60.51562192009797,59.771888856269854,38.24495304092191,38.30404615626526,38.57233271486981,0.8747753919959068,63.30388362788709,62.561316987392054,43.1055615067307,42.21331735764612,42.484750301588114,0.8913898070157318,59.45726298732943,57.526204765255976,32.25733837627613,33.865652765902965,34.71708040021201,0.8478907163143158,50.85534112089745,50.03031690407106,26.34812694098741,25.71520616359661,26.279905137766537,0.8200640871568963,50.85534112089745,50.03031690407106,26.34812694098741,25.71520616359661,26.279905137766537,0.8200640871568963,50.85534112089745,50.03031690407106,26.34812694098741,25.71520616359661,26.279905137766537,0.8200640871568963,50.85534112089745,50.03031690407106,26.34812694098741,25.71520616359661,26.279905137766537,0.8200640871568963,60.6933361148328,58.92145564017714,34.55620099741434,34.9155807710764,35.97092486853162,0.8604976444179753,54.64784196754577,53.89762744806972,34.78873357474227,34.48074447431171,35.23792224710771,0.8342977927327156,59.45104570223155,56.60695794787922,28.579661522093712,32.17800497697126,32.881421889606045,0.8324152611792087,56.0838536046274,53.98404543363633,28.32553559027332,29.500632062307492,30.417167497105098,0.8509534669546742,55.75054509800249,54.428314269123,33.533878099037544,30.933041977866377,31.898920036378996,0.8466845436096191 +ta-en,5k,39.37151886439601,38.37489519796626,16.399897348626975,18.26562117923812,17.627149209826893,0.7546233857274055,60.107155518614285,57.4457710791638,32.311082846928464,33.55085234615088,34.4590611198238,0.8479743801653385,60.2375088708884,59.542621070701095,37.73203593706005,38.06462375547255,38.33088629383616,0.8731431738883257,62.8475124059938,62.144966584928895,42.70043472874664,42.00573936028934,42.24314774074809,0.8903522048958289,59.11444411843373,57.148035802254526,31.35505151525585,34.01561366893348,34.56963405352514,0.8486373663544655,50.115983991974566,49.31229876069414,26.280567187594468,25.576055242195256,26.052869267090234,0.8162053002182532,50.115983991974566,49.31229876069414,26.280567187594468,25.576055242195256,26.052869267090234,0.8162053002182532,50.115983991974566,49.31229876069414,26.280567187594468,25.576055242195256,26.052869267090234,0.8162053002182532,50.115983991974566,49.31229876069414,26.280567187594468,25.576055242195256,26.052869267090234,0.8162053002182532,60.047573839598925,58.19651120554607,33.17435801105844,33.95815999261179,35.077209258460485,0.8596431332299987,54.71448645869131,53.98212664331717,34.865633675762915,34.49562297253713,35.2138555042198,0.8373731119334697,59.18970070926514,56.47359668928438,28.929360390084657,32.009640327426226,32.956460565560505,0.8328773201704025,55.77564075635998,53.61759194041663,27.47879700783284,28.93272675238803,29.848581440335153,0.8498090575688442,55.30634053524792,54.06575575883763,32.67612800144492,30.587048687078873,31.414799560176046,0.8421250923871993 +ta-en,10k,39.66184981994951,38.70144410559059,16.635038725941627,18.45114127947077,17.833644803220327,0.7544466648101806,61.75798035193796,59.12903021473232,34.39499240839612,35.92470564198692,36.87249579225605,0.8535116041898727,59.37888756243647,58.611080366930466,36.011471758769666,36.921736269966054,37.18628782193747,0.8704207756072283,62.68994539190883,62.00439961885627,42.1481485980801,41.777717693770626,41.99095634978417,0.8899459793635505,58.71705205081543,56.79024100743819,30.6912228651309,33.38481512896798,33.970360670851335,0.844701763510704,50.17644250494791,49.375202104535795,26.14306719178659,25.516428252417697,26.009296617616663,0.8162300244870561,50.17644250494791,49.375202104535795,26.14306719178659,25.516428252417697,26.009296617616663,0.8162300244870561,50.17644250494791,49.375202104535795,26.14306719178659,25.516428252417697,26.009296617616663,0.8162300244870561,50.17644250494791,49.375202104535795,26.14306719178659,25.516428252417697,26.009296617616663,0.8162300244870561,59.44241851014225,57.543245973599454,32.02068197304198,32.989606765740824,34.080175890389235,0.8565062239287369,53.84290157280306,53.064500963953186,33.947287247661755,33.50291595801113,34.28204636472693,0.8322957644760609,58.86040077530651,56.06605342114281,28.22128878884421,31.735624558849164,32.58044952164305,0.831830090880394,55.43089314826211,53.2716331352053,26.94205797497817,28.53422766568191,29.344144666730266,0.8467663534780736,54.54161727059267,53.27600517263037,30.879667285745647,29.763568275179793,30.61953628467286,0.8417990252375602 +ta-en,15k,38.93958526618946,37.9423666899648,15.89971395202742,17.724165475640607,17.08788519688558,0.7514738410115241,63.04912140181778,60.405523019129504,35.798060148238854,37.48408588690264,38.46184102871189,0.8601772767901421,58.97133468965048,58.25553140519777,35.8433465718555,36.49129606679034,36.78553642188729,0.868731607362628,62.122531248298095,61.423738860976584,41.175402204866195,40.927687728202656,41.18346186472626,0.8877022689095867,58.65983465313128,56.6802895135369,30.142395620650245,33.11914588724891,33.70708416313059,0.8469915004372597,50.190185691728914,49.424318295275064,26.457919601300983,25.555387915175853,25.992960120207965,0.8157399566844106,50.190185691728914,49.424318295275064,26.457919601300983,25.555387915175853,25.992960120207965,0.8157399566844106,50.190185691728914,49.424318295275064,26.457919601300983,25.555387915175853,25.992960120207965,0.8157399566844106,50.190185691728914,49.424318295275064,26.457919601300983,25.555387915175853,25.992960120207965,0.8157399566844106,59.30171072542917,57.48852277318919,32.08427537674978,33.049149792465755,34.08395652798429,0.8557109833661333,53.84658011107133,53.045838570945094,33.67318492445719,33.17528266750687,33.94058941032087,0.8335989883840084,59.067049911385126,56.3283334193523,28.487133101658152,31.97818434390073,32.75584316497059,0.8321624001860619,55.68135527981136,53.55910697731873,27.455037344067588,29.32706115143437,30.20855458604444,0.8470760650432156,55.045920882965426,53.797912454365495,32.82592941773117,30.46379047561909,31.30141057598678,0.8432262063622474 +ta-en,20k,39.352881441156754,38.407028109180736,16.471995156082222,18.271513709309875,17.61234070563459,0.7510786295533181,63.70925685695593,61.05296545304575,36.695646360709205,38.62036185696437,39.57677341749939,0.8613841069936752,59.737405956280995,59.04830807240404,37.091242067017184,37.87127352387377,38.09516634037666,0.8707491488307715,62.539739987267374,61.82136271659,41.69018052840806,41.77167609895496,41.98730022828941,0.8877965231954724,59.11743537461197,57.25481268525855,31.142725749303025,33.95221240983595,34.52268266906763,0.8468463129401207,50.505420243857714,49.75731011942967,26.53612539809372,25.79033326888846,26.262142190743685,0.815742032567936,50.505420243857714,49.75731011942967,26.53612539809372,25.79033326888846,26.262142190743685,0.815742032567936,50.505420243857714,49.75731011942967,26.53612539809372,25.79033326888846,26.262142190743685,0.815742032567936,50.505420243857714,49.75731011942967,26.53612539809372,25.79033326888846,26.262142190743685,0.815742032567936,59.56161636541984,57.769991001503406,32.340413891555414,33.55668401317413,34.61805405095608,0.8567443510973789,54.390175800976536,53.56059851807995,34.23743796064806,33.83403558143353,34.5719096302564,0.8334979208707809,58.8758507333153,56.191984827088895,28.76203338344732,31.58639392202739,32.41846059697566,0.8308889925479889,55.37387878212861,53.254707170996426,26.958480768793823,28.634835686816626,29.578988533306852,0.8464345760614034,54.974062519610186,53.75881317245834,31.844467583229925,30.594213601152543,31.40440967247881,0.8428259556293487 +ta-en,25k,38.65940473878675,37.649390484982774,15.81276195468595,17.58532704888579,17.004959992318227,0.7474953307807446,64.04725087499365,61.42962540191179,37.09339503750719,39.02163223194449,39.98901600838739,0.8628479547798633,59.519555151016256,58.830867638406104,36.621497819485775,37.49370983310459,37.777534996640874,0.8696107387244701,62.606355219389755,61.935482023287214,42.00175990650263,41.943221614836965,42.1864893150037,0.8881275742368226,58.63879617725423,56.64403548938728,29.95204599338054,33.153971722664544,33.74928818585589,0.8460434170663357,50.391050771844135,49.58735372946614,26.358126611234137,25.747113279621463,26.223809152734898,0.8133344068709347,50.391050771844135,49.58735372946614,26.358126611234137,25.747113279621463,26.223809152734898,0.8133344068709347,50.391050771844135,49.58735372946614,26.358126611234137,25.747113279621463,26.223809152734898,0.8133344068709347,50.391050771844135,49.58735372946614,26.358126611234137,25.747113279621463,26.223809152734898,0.8133344068709347,59.55414861647426,57.74703304137459,32.094170005209584,33.25771474480451,34.33577540413442,0.8558003563119705,53.86665625745603,53.15634700092144,34.00862873558283,33.6738715465911,34.46886136923999,0.8329405805170536,58.890632616108704,56.16350397751192,28.45907815148269,31.653800864205223,32.43401341399923,0.8308236742317676,55.05869435825216,52.89229749013958,26.56091591789399,28.275743167021503,29.247876665620577,0.845015874228223,54.31364825927708,53.049530758905064,31.48795173143003,29.637699578882415,30.300190451925477,0.840070496827364 +si-ta,baseline,,,,,,,1.506499672099712,2.4557752982830663,1.4632295174138816,0.8168146703604148,0.9846654424814892,0.5261105810403823,0.1483256577480597,0.4502203040744262,0.2168377791627246,0.0158720623622216,0.0192236979810407,0.5707517316937447,0.3991315937132054,1.2016165396211462,0.1672614921158257,0.0492256066085154,0.0612625807307166,0.5897180299162865,0.6226172276234757,1.569811801568743,0.3599066800468254,0.1444285046708023,0.1507449763486991,0.5247272368520498,0.2009138710399402,0.9862013885203196,0.0439269115867961,0.0149505862470798,0.0160947403179194,0.5033425367050652,0.2009138710399402,0.9862013885203196,0.0439269115867961,0.0149505862470798,0.0160947403179194,0.5033425367050652,0.2009138710399402,0.9862013885203196,0.0439269115867961,0.0149505862470798,0.0160947403179194,0.5033425367050652,0.2009138710399402,0.9862013885203196,0.0439269115867961,0.0149505862470798,0.0160947403179194,0.5033425367050652,0.5369858883410602,1.5497125670514913,0.2322594182936931,0.0536642255607852,0.0773424208521478,0.5412610062968827,,,,,,,1.013064442321414,2.1382657200712902,0.9734489037312932,0.6359235764087909,0.7222283891295722,0.5001927641961648,0.5628607944577833,1.5334494611820455,0.601637909996467,0.2290556112313874,0.3074346832828144,0.5284592901382286,0.7098839047708805,1.6801276275033277,2.236073717290074,0.5717536472337825,0.808304947689558,0.5240307556092739 +si-ta,1k,,,,,,,50.2443796188764,44.10493824421842,12.538969053246612,22.42627877247282,22.439774320757984,0.8743745173215867,36.323185021540496,33.21552307918757,11.559699054015397,20.3981987484158,20.47530392566763,0.7550140042155981,53.82644072837088,48.96399499026332,15.64400772655998,28.21661867520769,27.593187279545745,0.8876809190942,48.56815297697707,42.42238610194449,7.914428956367575,22.110250492175208,21.96760787336064,0.8537849247455597,45.13866684662315,40.07546567521024,8.198705487657577,18.005897082619562,17.594096148416693,0.8022552307798257,45.13866684662315,40.07546567521024,8.198705487657577,18.005897082619562,17.594096148416693,0.8022552307798257,45.13866684662315,40.07546567521024,8.198705487657577,18.005897082619562,17.594096148416693,0.8022552307798257,45.13866684662315,40.07546567521024,8.198705487657577,18.005897082619562,17.594096148416693,0.8022552307798257,53.2617833571283,47.44848569038954,11.42307905567309,26.463722638642164,26.46904198918585,0.859532229016505,,,,,,,53.78217793027862,47.105039169606194,9.66112278327546,25.42321036216343,24.746129997956235,0.8144897919300259,49.377805890374155,43.35548016326561,9.269762772435334,23.17183827724759,23.25917858100641,0.855572857316068,53.71652084735038,48.50413378360178,16.66258277987578,27.250011859180137,26.361991957852965,0.8376016582548619 +si-ta,5k,,,,,,,56.12097833718678,49.733543358075515,17.030684140516204,30.7426792607728,30.89678369082139,0.8861886198818684,54.432059902914645,50.075506543199296,17.528247788305706,30.09849940414254,29.195537045213,0.8908426854386926,54.10965653462958,48.94280558343474,14.383224952538336,27.135943356000563,26.12773441010943,0.8999825488074803,47.54393797687138,41.262209610796106,6.82105375829823,21.135531898017184,20.775951029697076,0.8490756225883961,44.14071777997267,38.70461289488636,6.745391082329602,16.030624195019897,15.50282372158476,0.8027440217356102,44.14071777997267,38.70461289488636,6.745391082329602,16.030624195019897,15.50282372158476,0.8027440217356102,44.14071777997267,38.70461289488636,6.745391082329602,16.030624195019897,15.50282372158476,0.8027440217356102,44.14071777997267,38.70461289488636,6.745391082329602,16.030624195019897,15.50282372158476,0.8027440217356102,50.95998031662033,44.90239311813765,8.83243198122357,22.702569083306024,22.694441463061317,0.8536119359966007,,,,,,,51.37994416681284,44.600702669565,7.689568469864875,22.39699110248858,21.570591767918582,0.8083506729263736,47.23390610823547,41.11550442607534,7.782337803679392,19.932887956146672,19.798590547136538,0.8459384798149582,51.81668421812679,46.51265806217224,14.118832177970004,23.904679503172375,23.24996088975882,0.8370880727469922 +si-ta,10k,,,,,,,58.14078422125692,51.75968329106598,18.738581491878676,33.452466220048635,33.60204293320304,0.8897950460314751,53.154454788185866,48.712022861518555,15.884186660493718,28.40768736031095,27.62017506581964,0.8876451987326145,53.303848750785654,48.15741744708899,13.72785644479278,26.326082651119886,25.30305922974426,0.89893731930169,47.526336143455275,41.23071711558105,6.9203346734527695,20.920052776866168,20.688363725891698,0.850786651223898,44.03344124736107,38.56380996498546,6.175457135375574,15.423270454345603,14.782094050787254,0.8010131987495399,44.03344124736107,38.56380996498546,6.175457135375574,15.423270454345603,14.782094050787254,0.8010131987495399,44.03344124736107,38.56380996498546,6.175457135375574,15.423270454345603,14.782094050787254,0.8010131987495399,44.03344124736107,38.56380996498546,6.175457135375574,15.423270454345603,14.782094050787254,0.8010131987495399,50.5543703039938,44.53653984259175,8.793252120251214,21.94804983747638,22.03497418894363,0.8499003647255802,,,,,,,51.05445562150454,44.1282584335198,7.43661377510804,22.47372435594028,21.762093358602204,0.8153357454480565,46.836618790209656,40.6531192195075,7.401129420771832,19.285564207292083,19.20115461267066,0.8505266855651211,51.49816428271053,46.14675934942884,14.048096092055278,23.686153244821465,22.950627944652982,0.8389946821331978 +si-ta,15k,,,,,,,58.51502500815897,52.10340591896176,18.71465430045571,33.94814988996791,34.16824189172424,0.892300804167986,54.127066085395946,49.69329220502086,17.16696700590091,29.26894214159221,28.652638825484704,0.8884524757713079,53.51653697424845,48.41602536871634,14.103769750493685,26.635338195111228,25.807689005505196,0.8983319042863336,47.63386011934315,41.36829753464779,6.992413270637968,21.42097035492968,21.228218568403623,0.8501114219129086,44.42504437023244,39.027253448053614,6.704736604280777,16.12462739360897,15.357443866101152,0.80511938022936,44.42504437023244,39.027253448053614,6.704736604280777,16.12462739360897,15.357443866101152,0.80511938022936,44.42504437023244,39.027253448053614,6.704736604280777,16.12462739360897,15.357443866101152,0.80511938022936,44.42504437023244,39.027253448053614,6.704736604280777,16.12462739360897,15.357443866101152,0.80511938022936,50.44887726763213,44.51688307506956,9.174720208715458,22.10982496645205,22.061340830915096,0.849138046314399,,,,,,,50.641585133776026,43.91872731729796,7.56702385722543,21.66294710805543,20.83484070879369,0.8097094015731495,46.62147952798226,40.48548461848681,7.200193895231319,19.35224094154367,19.205403285057585,0.8485663407463802,51.00509178274327,45.801115742436075,13.660645461567997,23.874330078442313,23.09008631877295,0.8354445858299733 +si-ta,20k,,,,,,,59.14881079657924,52.82544950369273,20.215442354544713,35.25547044957205,35.461619164613,0.894321031153202,53.08459688059487,48.65732276542112,16.262591401317337,28.4537268730661,27.756742230218972,0.8849549402892589,53.24051678377969,48.073056935221835,13.568906081074187,26.30433318149175,25.29748470040835,0.896335870414392,47.74047028670271,41.385700049697526,7.188332326652838,21.410848685094138,21.39304268462389,0.8521688247025013,43.93742835498326,38.49243562653017,6.303843648287157,15.444236178288651,14.804014853927448,0.7984641714718654,43.93742835498326,38.49243562653017,6.303843648287157,15.444236178288651,14.804014853927448,0.7984641714718654,43.93742835498326,38.49243562653017,6.303843648287157,15.444236178288651,14.804014853927448,0.7984641714718654,43.93742835498326,38.49243562653017,6.303843648287157,15.444236178288651,14.804014853927448,0.7984641714718654,49.75691223157466,43.71992192067232,8.308092514389523,21.38375516081867,21.43980144304131,0.8479972150056593,,,,,,,50.260324332065466,43.4422812064727,7.505271716002482,21.512867888862676,20.674334748665892,0.8072064611110457,46.487538553989125,40.31292580462015,6.9582131221492585,19.146769036928447,18.933340558617992,0.8471615392965055,50.91020212917918,45.596668896008374,13.316260018099728,23.26058858982264,22.461570864255275,0.8351807125210762 +si-ta,25k,,,,,,,59.812702487886824,53.44480979898254,20.33552064617674,35.86678178644539,36.00773367568179,0.8959461055696011,53.1108460847585,48.70099406890239,16.50568694824706,27.93487100441276,27.014044900074236,0.8846168394088745,53.72629202724089,48.57216552361423,13.806014017060688,26.541412165796235,25.46648470378095,0.8990643055392573,47.47793495852912,41.258516450053776,7.22054423551598,21.45037334491328,21.22070792686838,0.8479302596896887,44.032299218428186,38.542971823818256,6.216689408046166,15.355083659335548,14.624086944787535,0.8002112233466621,44.032299218428186,38.542971823818256,6.216689408046166,15.355083659335548,14.624086944787535,0.8002112233466621,44.032299218428186,38.542971823818256,6.216689408046166,15.355083659335548,14.624086944787535,0.8002112233466621,44.032299218428186,38.542971823818256,6.216689408046166,15.355083659335548,14.624086944787535,0.8002112233466621,49.78500522186812,43.7689835650637,8.20206190883595,21.166895541021457,21.231448498492565,0.8485575679299091,,,,,,,50.329279921202954,43.48567558700648,7.346455770868597,21.272291935038847,20.46408102851401,0.8077254467168633,46.21132146865297,40.03171949968911,6.851872356963542,18.500530348417875,18.308961611100344,0.8455016231348392,50.37771910111668,45.07576589162182,12.36394486911458,22.6437505498606,21.71292236539571,0.8344147505164147 +ta-si,baseline,,,,,,,2.1264790250606924,2.8059466038175693,1.6139690857599398,1.1365380107647691,0.963688045159638,0.5284281494617462,0.7973949256386279,2.406827458170219,1.0616449149333549,0.2178005764513419,0.2316938084927182,0.5833025423586369,0.77098643919795,1.998452512735229,0.3758215093238002,0.1157819799315809,0.1505558957515414,0.5912962484657764,0.8516461597485614,1.8100795334610311,0.3236137104946885,0.1423561347192973,0.1365804452582944,0.5140674581825733,0.4109079281980246,1.4658801399815615,0.1070041023726659,0.0221457000919698,0.0201868617670006,0.5111609247796659,0.4109079281980246,1.4658801399815615,0.1070041023726659,0.0221457000919698,0.0201868617670006,0.5111609247796659,0.4109079281980246,1.4658801399815615,0.1070041023726659,0.0221457000919698,0.0201868617670006,0.5111609247796659,0.4109079281980246,1.4658801399815615,0.1070041023726659,0.0221457000919698,0.0201868617670006,0.5111609247796659,2.028414304903063,2.5912661213586685,0.9266677410931268,0.4042689190115334,0.3726255123294969,0.5168055228949073,,,,,,,2.623521733310948,3.1080422505838414,1.0873166695415593,1.1456848084903108,0.9758022354772898,0.4921825256730829,0.8482613919797445,1.87764022307322,0.6389513759492589,0.2639885277142922,0.2856255702972314,0.5167262381303451,2.155647341058514,3.0479886610603177,2.7193614809686384,0.836017800841948,0.8711490692234946,0.5428231197297573 +ta-si,1k,,,,,,,46.95468711618753,42.71863536497049,13.946403489944109,28.67080157940711,35.66790765231362,0.8795496428310872,41.19187592870378,38.419747113569834,13.543599122251214,23.98665182406473,26.149743096018017,0.8803802770078182,43.44096000550498,40.04578032668024,12.345930720666471,23.55773478102019,27.371812657261128,0.8940775383131777,39.239179268723184,35.08154507375792,7.089033852800834,20.430535571924903,23.64374864390406,0.8513488554060459,32.21343052357557,29.07701645656392,5.4597256404869,12.505948053229249,14.27471265084141,0.8002993582459851,32.21343052357557,29.07701645656392,5.4597256404869,12.505948053229249,14.27471265084141,0.8002993582459851,32.21343052357557,29.07701645656392,5.4597256404869,12.505948053229249,14.27471265084141,0.8002993582459851,32.21343052357557,29.07701645656392,5.4597256404869,12.505948053229249,14.27471265084141,0.8002993582459851,38.72222656917083,34.64803824339464,7.588713776478428,21.290457189445423,24.660552995976907,0.8313557970968466,,,,,,,38.80244151460313,35.118936902723284,9.915471801860118,22.41068104302556,25.31076371035167,0.7955503560108493,40.185809049862115,36.195736062588395,8.655722843861811,22.222520270610033,25.833272205353627,0.8500500880329034,40.2176191045428,37.05086886311183,13.14964171911592,22.21999833121419,25.24383735605519,0.820767950668931 +ta-si,5k,,,,,,,51.41926015585591,47.17738904375178,17.444572340694904,35.27541311412441,41.78334972760448,0.8926729651689529,34.33365229924564,31.362802703379582,7.442684778858739,18.426800660151997,20.06852154162444,0.8642446266561746,42.03424671487618,38.71582745228877,11.287307485401328,22.42073353637314,26.325389675886218,0.8902426442085833,38.86199731494143,34.62355952934438,6.433818298061117,19.883944774417436,23.10354666443403,0.8495119138360023,31.32002834430054,28.134170682083813,4.480587464149585,11.491425934951907,13.227056488225278,0.799190194723695,31.32002834430054,28.134170682083813,4.480587464149585,11.491425934951907,13.227056488225278,0.799190194723695,31.32002834430054,28.134170682083813,4.480587464149585,11.491425934951907,13.227056488225278,0.799190194723695,31.32002834430054,28.134170682083813,4.480587464149585,11.491425934951907,13.227056488225278,0.799190194723695,38.06765279142419,33.94830309744808,7.201166739204406,20.49284853547799,23.95498462560696,0.8293779508601273,,,,,,,37.65115327528594,33.743185841828364,8.574490363016755,20.69555348481924,23.218023579085937,0.7880995444269204,39.12278945002164,35.135536187359804,7.654368709400647,21.329424819398096,24.89436969453728,0.8447011141437787,38.58916766639129,35.548020932245,12.016463336897296,20.18377192507685,22.706979516016947,0.8178093786835671 +ta-si,10k,,,,,,,53.003758933588315,48.821356943977854,19.31417763140684,37.860609529253814,44.2256042868842,0.8984571205079556,32.779991853044535,29.769163253223105,6.336988867612174,17.075376725173953,18.95436559347672,0.8551582052856683,41.14962374342426,37.83151456057326,10.654342943989796,21.77550293245442,25.623770716708265,0.8870724192334603,38.4952395569435,34.30218847145008,6.345584298933479,19.66453846144426,22.87273195578639,0.8488303147852421,30.75703384311096,27.58530232637968,4.349458365486436,10.994093247594384,12.700897148810842,0.7969094352745121,30.75703384311096,27.58530232637968,4.349458365486436,10.994093247594384,12.700897148810842,0.7969094352745121,30.75703384311096,27.58530232637968,4.349458365486436,10.994093247594384,12.700897148810842,0.7969094352745121,30.75703384311096,27.58530232637968,4.349458365486436,10.994093247594384,12.700897148810842,0.7969094352745121,37.45298774264474,33.35210096895649,6.719988257674699,19.863863864849968,23.370670517369696,0.825703306333913,,,,,,,37.03803416828515,33.12868511463382,8.343706010206716,20.31863699567055,22.899490207344897,0.7829769134749563,38.55509522752336,34.58953090015751,7.345789897949188,20.66906789337079,24.236483807724387,0.8423951678567018,38.24081710579901,35.136434247784166,11.725754619794358,20.08262685955402,22.570575210159387,0.8129719280898571 +ta-si,15k,,,,,,,53.96876394544066,49.68875362882517,19.57129556993449,39.16798179399488,45.52770375688494,0.9014797831177712,32.52286357531791,29.487865443464617,6.248544947911525,17.09394300575521,18.70591304753473,0.8551399844735861,40.78065473084989,37.457620865710304,10.342019107119588,21.709779587946738,25.558254074279883,0.8867994582004328,38.44667851910203,34.258587708481436,6.186473063416697,19.75736103493816,23.017708074232512,0.8492549396157265,30.847155564452667,27.666299764563313,4.727389743288037,11.18376805290996,12.876095051785802,0.799267442432982,30.847155564452667,27.666299764563313,4.727389743288037,11.18376805290996,12.876095051785802,0.799267442432982,30.847155564452667,27.666299764563313,4.727389743288037,11.18376805290996,12.876095051785802,0.799267442432982,30.847155564452667,27.666299764563313,4.727389743288037,11.18376805290996,12.876095051785802,0.799267442432982,37.39422991381425,33.25335478871849,6.7076017167597035,19.666608785867624,23.247194308801653,0.8248296706306357,,,,,,,36.94074923703173,33.023939801820724,7.766407385787166,19.567224746683117,22.092532643274968,0.7912866747652998,38.45302325056041,34.450685923333786,7.005155614085869,20.396857388683156,23.95739109565358,0.8433839754595351,37.70430986761613,34.67567508096997,11.514139456675752,19.386135763634236,21.834840330552364,0.8098987563550473 +ta-si,20k,,,,,,,55.04726933015037,50.79367153843363,21.29440586973716,40.66185853738951,46.80437621858133,0.904481227517128,32.37243219819934,29.3066901510309,5.980697619581673,16.681144463590545,18.476464714958414,0.8546403640657664,40.70668509577528,37.42139152547206,10.40911683528101,21.36231324881069,25.230424046965325,0.8851708894780559,38.315255110238624,34.157239328495564,6.343790929618099,19.59889799872615,22.762480237872943,0.8501061606109143,30.15441975702176,27.009932319452407,4.1769704779573775,10.643300021881029,12.345638717868718,0.7941161549406935,30.15441975702176,27.009932319452407,4.1769704779573775,10.643300021881029,12.345638717868718,0.7941161549406935,30.15441975702176,27.009932319452407,4.1769704779573775,10.643300021881029,12.345638717868718,0.7941161549406935,30.15441975702176,27.009932319452407,4.1769704779573775,10.643300021881029,12.345638717868718,0.7941161549406935,37.28571821610571,33.188914491541766,6.5162986350236345,19.537762120428734,22.92513616951124,0.8252781428276531,,,,,,,37.08153859266698,33.15436499964011,7.982183313326316,19.493380535411852,21.932104589943624,0.7907066747964341,38.394709144731394,34.40134327692656,7.262813656482166,20.372393786070468,23.936931551113577,0.8433336493643847,37.12786168243185,34.14130791151598,10.769647874978968,19.216753233688237,21.49082430021167,0.8091989916861058 +ta-si,25k,,,,,,,55.164204075164214,50.910692720283855,21.097118075257388,41.11269138796458,47.31720115936583,0.9045515403747558,32.115498882583545,29.0352322424891,5.715035714226997,16.597641369098486,18.362433692058858,0.8535309534668922,40.78127075286322,37.45895860642715,10.199632950644968,21.514486477221592,25.243830966859765,0.8861108106996676,38.24462187210634,34.101542942234126,6.004275881201997,19.309101197122633,22.61815711724994,0.8478409900069237,30.04457413782446,26.872293760071443,3.998389396606069,10.3177642268497,12.016513765533656,0.7933493303191074,30.04457413782446,26.872293760071443,3.998389396606069,10.3177642268497,12.016513765533656,0.7933493303191074,30.04457413782446,26.872293760071443,3.998389396606069,10.3177642268497,12.016513765533656,0.7933493303191074,30.04457413782446,26.872293760071443,3.998389396606069,10.3177642268497,12.016513765533656,0.7933493303191074,37.26994610605991,33.210986519051716,6.6850183927221325,19.603112341323836,23.067051210855503,0.822933440081655,,,,,,,37.12234504381412,33.219539031380336,7.841096632808772,19.806591952411605,22.352163119496407,0.7895335042659117,38.21677557606723,34.181725386295795,6.884206151628509,20.18145543822757,23.79556439497183,0.8404766332372846,37.32610532361566,34.27432287596386,10.97646499413233,19.120775618412832,21.580127054022135,0.8094877264499665 diff --git a/Government/combined results/government_crossdomain_en-si_summary.csv b/Government/combined results/government_crossdomain_en-si_summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..dd027099b436f94b1526e1aad97711cffaed5cdd --- /dev/null +++ b/Government/combined results/government_crossdomain_en-si_summary.csv @@ -0,0 +1,8 @@ +data_size,Bible_chrf_sacrebleuapi,Bible_chrfpp_sacrebleuapi,Bible_bleu_sacrebleuapi,Bible_spBLEU-flores101,Bible_spBLEU-flores200,Bible_comet,Government_chrf_sacrebleuapi,Government_chrfpp_sacrebleuapi,Government_bleu_sacrebleuapi,Government_spBLEU-flores101,Government_spBLEU-flores200,Government_comet,OpenSubtitles_chrf_sacrebleuapi,OpenSubtitles_chrfpp_sacrebleuapi,OpenSubtitles_bleu_sacrebleuapi,OpenSubtitles_spBLEU-flores101,OpenSubtitles_spBLEU-flores200,OpenSubtitles_comet,NLLB_chrf_sacrebleuapi,NLLB_chrfpp_sacrebleuapi,NLLB_bleu_sacrebleuapi,NLLB_spBLEU-flores101,NLLB_spBLEU-flores200,NLLB_comet,Wikipedia_chrf_sacrebleuapi,Wikipedia_chrfpp_sacrebleuapi,Wikipedia_bleu_sacrebleuapi,Wikipedia_spBLEU-flores101,Wikipedia_spBLEU-flores200,Wikipedia_comet,Literature_chrf_sacrebleuapi,Literature_chrfpp_sacrebleuapi,Literature_bleu_sacrebleuapi,Literature_spBLEU-flores101,Literature_spBLEU-flores200,Literature_comet,Literature-Children_chrf_sacrebleuapi,Literature-Children_chrfpp_sacrebleuapi,Literature-Children_bleu_sacrebleuapi,Literature-Children_spBLEU-flores101,Literature-Children_spBLEU-flores200,Literature-Children_comet,Literature-Kathlyn_chrf_sacrebleuapi,Literature-Kathlyn_chrfpp_sacrebleuapi,Literature-Kathlyn_bleu_sacrebleuapi,Literature-Kathlyn_spBLEU-flores101,Literature-Kathlyn_spBLEU-flores200,Literature-Kathlyn_comet,Literature-Old_chrf_sacrebleuapi,Literature-Old_chrfpp_sacrebleuapi,Literature-Old_bleu_sacrebleuapi,Literature-Old_spBLEU-flores101,Literature-Old_spBLEU-flores200,Literature-Old_comet,News_chrf_sacrebleuapi,News_chrfpp_sacrebleuapi,News_bleu_sacrebleuapi,News_spBLEU-flores101,News_spBLEU-flores200,News_comet,Maths_chrf_sacrebleuapi,Maths_chrfpp_sacrebleuapi,Maths_bleu_sacrebleuapi,Maths_spBLEU-flores101,Maths_spBLEU-flores200,Maths_comet,Health_chrf_sacrebleuapi,Health_chrfpp_sacrebleuapi,Health_bleu_sacrebleuapi,Health_spBLEU-flores101,Health_spBLEU-flores200,Health_comet,FloresPlus_chrf_sacrebleuapi,FloresPlus_chrfpp_sacrebleuapi,FloresPlus_bleu_sacrebleuapi,FloresPlus_spBLEU-flores101,FloresPlus_spBLEU-flores200,FloresPlus_comet,Jailbreak_chrf_sacrebleuapi,Jailbreak_chrfpp_sacrebleuapi,Jailbreak_bleu_sacrebleuapi,Jailbreak_spBLEU-flores101,Jailbreak_spBLEU-flores200,Jailbreak_comet +baseline,0.2074466786941936,0.992404644366044,0.0114184838480725,0.0021871822261416,0.0023078106301359,0.351035983428359,1.9957824955315877,2.499977992441443,1.6800513225163616,1.212937758336134,1.055154151403294,0.5387619452029466,0.847093121279806,2.4684363097189745,1.396130818258411,0.1335446333113439,0.1565595240682873,0.6057251301407814,0.7585231071916326,2.0418635894125736,0.311198372233723,0.0900186221881229,0.1051094510834449,0.5752390958219766,0.7443504702963069,1.597579383453706,0.2952394162547146,0.1644145649375478,0.1569865791110672,0.4998222410529852,0.5405674952727352,1.72281406584829,0.2106596891674983,0.0213847448284255,0.0202065602410734,0.5059475341588259,0.5405674952727352,1.72281406584829,0.2106596891674983,0.0213847448284255,0.0202065602410734,0.5059475341588259,0.5405674952727352,1.72281406584829,0.2106596891674983,0.0213847448284255,0.0202065602410734,0.5059475341588259,0.5405674952727352,1.72281406584829,0.2106596891674983,0.0213847448284255,0.0202065602410734,0.5059475341588259,1.1743217071010077,1.713707163740193,0.7760985154863841,0.4451472768674177,0.3925480474445332,0.5711525277197361,3.386902647626311,5.396413898661497,7.043350945997881,4.407736007891258,4.186116810472018,0.5152128710597753,2.1293772294034072,2.6482080421096397,1.177958280471317,1.235578054683879,1.073537231998086,0.4134376797113526,0.783503884977641,1.7245734048828818,0.5241960228541523,0.2973223112210829,0.3139251616408782,0.4667224814533716,2.7299557155007927,3.4342239238448125,3.070198561059441,1.377379113964754,1.4588223666474471,0.519491742938757 +1k,23.516304960748695,20.362706332865574,1.6177223328136954,7.698804887228647,7.142460377516814,0.6451798568218946,7.693363479451183,6.136089531064441,0.0203424558411158,0.1361625082452787,0.2708677194690141,0.3770179857909679,32.830743778102054,30.83796375018137,8.46478022121088,15.040401102261876,14.104804439363615,0.7946870295628905,12.324043069958456,10.868875229575847,0.8902522416379338,1.8066512013725395,1.8088916748363504,0.5654888506523086,8.11906180769877,6.549857745501184,0.1237060414889934,0.433680415977373,0.5140921731137955,0.3466547068357468,10.20057431150489,8.942477418219665,0.835031192198746,1.6339000097540557,1.521072472920624,0.5377916059345007,10.20057431150489,8.942477418219665,0.835031192198746,1.6339000097540557,1.521072472920624,0.5377916059345007,10.20057431150489,8.942477418219665,0.835031192198746,1.6339000097540557,1.521072472920624,0.5377916059345007,10.20057431150489,8.942477418219665,0.835031192198746,1.6339000097540557,1.521072472920624,0.5377916059345007,9.31829164832508,7.602404261012937,0.2071728739300219,0.7506842504471134,0.9476541691597596,0.4086479433476925,7.135553179408386,5.990287751321683,0.2387540998454511,0.5156325949052365,0.4899032462176094,0.3933246617615223,7.943758973222321,6.379120539858632,0.0276695009220651,0.1918128396954742,0.2125325925109905,0.3151403463288639,7.215223944683759,5.744608434304767,0.048936397246092,0.1648504019946428,0.2229497306164265,0.320197635400201,9.44752151854744,8.10370681971603,0.5929600023807652,1.2132553605616745,1.272400283286709,0.4513409103006124 +5k,17.38218305920879,14.720861483249708,0.6110980468231604,4.57594442109759,4.215972757003348,0.5760666883587837,8.529184619241422,6.817795023193822,0.1368056848250783,0.7307413818879813,1.2547235687803515,0.499934135377407,18.823833403766027,17.25520904752903,2.8231739289631452,6.079735336389598,5.4618216841043665,0.6699169592484832,8.980534923916675,7.525372340613211,0.4358489858308041,1.072042727942755,1.0162517113668066,0.5183496097743482,8.871693845189547,6.993505824594849,0.0277429753469733,0.1645125930163549,0.2248596007991863,0.4438680990338325,8.063824155188607,6.555031221691862,0.1819747397416745,0.3778547201497972,0.3628863984872553,0.4679325869381427,8.063824155188607,6.555031221691862,0.1819747397416745,0.3778547201497972,0.3628863984872553,0.4679325869381427,8.063824155188607,6.555031221691862,0.1819747397416745,0.3778547201497972,0.3628863984872553,0.4679325869381427,8.063824155188607,6.555031221691862,0.1819747397416745,0.3778547201497972,0.3628863984872553,0.4679325869381427,8.064029365713953,6.348901418473853,0.0296985566814461,0.2990027650985308,0.4486387708986559,0.4363476236760616,6.519839470122508,5.185081036326267,0.0489337140600784,0.2011519908200932,0.1864243957831318,0.4449171331226825,6.1429329243101805,4.977345281893618,0.0372134062417059,0.1113953189532505,0.2195096770378732,0.3968002018131566,8.058227939320835,6.3488511990142,0.0270268025523735,0.1763660195637698,0.2460073296279243,0.4096863102830446,6.527991317587816,5.434436017464176,0.1679797515935076,0.7485159688817504,0.9217368558347367,0.4593092997223139 +10k,9.883167308842296,8.08164852593724,0.1224908082681523,1.6782909910263053,1.51688499860891,0.4502922486960888,7.931510211422728,6.177940517062402,0.1438249418127693,0.6302031145633212,1.0312510931564078,0.4728857491314411,9.800694865178825,8.375756621578411,0.5730870903662663,1.4827913789729208,1.4035582820664598,0.5102573437988758,7.869460496102654,6.09056504503559,0.1053689439449828,0.4985236041396916,0.5363278084194655,0.4601839078451714,7.275620508536446,5.622158052913327,0.0214232608019175,0.1101172691185725,0.1595470926438534,0.4203639453500509,6.376891953361408,4.851865039223764,0.0180506833167269,0.0442598824851941,0.0643229995316287,0.42568679946661,6.376891953361408,4.851865039223764,0.0180506833167269,0.0442598824851941,0.0643229995316287,0.42568679946661,6.376891953361408,4.851865039223764,0.0180506833167269,0.0442598824851941,0.0643229995316287,0.42568679946661,6.376891953361408,4.851865039223764,0.0180506833167269,0.0442598824851941,0.0643229995316287,0.42568679946661,6.783172487448554,5.22918019278509,0.0140679334862007,0.1930584845597379,0.3257971104827631,0.4174291704893112,6.786315824711309,5.199195420389427,0.0153743647953712,0.0221255940232247,0.0442699082295149,0.4291429179757833,5.863895093672576,4.60535828879025,0.0095053759705964,0.0314906975713339,0.0741187821838944,0.3987277927693356,7.101437099437691,5.433916452424211,0.0060960765752617,0.0651761491160575,0.1203964169392692,0.4025747983646487,6.071189768294846,4.833782795198674,0.041475688531638,0.1170244111598882,0.1268903339098027,0.4328604847192764 +15k,10.333521780820629,8.453661202177699,0.0665625178426419,1.5401320162594443,1.3016910398397168,0.4509024264812469,9.731813599282969,7.8013772560248285,0.1314581386354406,0.3657227959232438,0.4982705083406589,0.3988617490530014,6.577743041669929,5.300362655175006,0.1003228485111686,0.3516569502132318,0.3133280497660109,0.4609458845481276,6.324312311847597,4.99984908358139,0.0638513895650396,0.2349258100220535,0.2378402586714434,0.4581684144181248,9.591036118587716,7.478943564953437,0.0147339518621636,0.0934519961296255,0.0877371244631593,0.374054478764534,5.80325024079758,4.518580409004698,0.0262864967808605,0.0270903484808721,0.0435175861915902,0.4377163580656051,5.80325024079758,4.518580409004698,0.0262864967808605,0.0270903484808721,0.0435175861915902,0.4377163580656051,5.80325024079758,4.518580409004698,0.0262864967808605,0.0270903484808721,0.0435175861915902,0.4377163580656051,5.80325024079758,4.518580409004698,0.0262864967808605,0.0270903484808721,0.0435175861915902,0.4377163580656051,7.475587226092283,5.872985418064176,0.0132616503047819,0.093916072142558,0.1147211509808602,0.396491152331233,6.837063431902609,5.473992424430646,0.0169743394047959,0.0240623100054295,0.016119390622961,0.4090483323037624,9.423835346437755,7.569232714787168,0.0166801104087271,0.1045040222430923,0.1066884802702877,0.3727596676751469,8.110053421354792,6.406640947709287,0.0468008003620898,0.134156015523987,0.1478120657477304,0.3853278315880082,5.374077312754306,4.392153280587568,0.0609208628011329,0.0885308924521573,0.1160557479024545,0.4336004293262958 +20k,31.761604881464233,27.54796001571456,2.7451206165921933,13.882638289577836,13.98031568061058,0.7690197389423847,57.21169238265881,52.83729260161385,22.43693673311732,43.73262113530742,50.00083444741132,0.9225092524886132,35.12391260428374,32.105489417981794,7.502054254124334,19.84046279426906,22.19210912077999,0.8655002781748772,42.8673764543524,39.32278272621688,11.347460755796227,23.681411463770186,27.9130563227134,0.9010286902821903,41.55216385298722,37.118928968163544,7.435175493487085,22.869206294691708,26.457512815265737,0.8730469604730606,30.323146952610823,27.219078742914693,4.997742728382483,11.572875707289988,13.355103424191404,0.8145392687618732,30.323146952610823,27.219078742914693,4.997742728382483,11.572875707289988,13.355103424191404,0.8145392687618732,30.323146952610823,27.219078742914693,4.997742728382483,11.572875707289988,13.355103424191404,0.8145392687618732,30.323146952610823,27.219078742914693,4.997742728382483,11.572875707289988,13.355103424191404,0.8145392687618732,45.4274593270867,41.001988715490064,11.590726452242302,27.283072483750267,32.96372236158503,0.879407027348876,40.16798911664159,38.07374908008929,16.656258371139497,25.488915980680275,27.846527606251858,0.8411055938601494,43.19499688941104,38.93219459599893,11.245618379250637,26.743125840871944,29.755853396932068,0.8339593650082524,43.34636755091586,39.07131672687825,9.227334131840172,25.83282090100737,29.605976140051983,0.8745417469604448,41.75732488247362,38.2163434368176,12.26004639904115,22.983043562325175,25.83915085648592,0.8406499291956425 +25k,34.456340875004464,29.91541262040702,2.927627634611363,16.043010632978344,16.549307363154742,0.7995621707439423,57.77575685070945,53.38916478804416,22.913657969171457,44.63136119938347,50.79563281416712,0.9236653954982758,34.74392924639648,31.722146731556744,7.22518423376036,19.630955821278032,21.327917510045417,0.8652439583539963,42.319887007329434,38.80384792742494,10.908557257224446,23.233545972292767,27.473380399198323,0.901711478390458,42.11352299493802,37.60982662762706,7.471102785714683,23.41732987399879,26.992666872334663,0.8773687633275986,30.537553612782077,27.457684801211464,5.352300825495501,11.77648751639114,13.60866615715853,0.8180207158476114,30.537553612782077,27.457684801211464,5.352300825495501,11.77648751639114,13.60866615715853,0.8180207158476114,30.537553612782077,27.457684801211464,5.352300825495501,11.77648751639114,13.60866615715853,0.8180207158476114,30.537553612782077,27.457684801211464,5.352300825495501,11.77648751639114,13.60866615715853,0.8180207158476114,45.25266583842741,40.82924968374386,11.472613916243573,27.13425842456199,32.92937573986284,0.8772459758222103,40.1331593949345,37.9749405806341,16.160581021507667,25.310463847844968,27.7360906142369,0.8429196584969759,43.25096129715229,38.92367941876307,10.935473968452124,26.883374289624214,29.927259118058668,0.8349770524361161,43.53881729774115,39.32453614210059,9.495034487221888,26.098641019137556,29.645824704158496,0.8745844934886623,41.94530390315914,38.36738795492778,12.134369099259992,22.955178169349796,25.91187375950136,0.8430828390717506 diff --git a/Government/combined results/government_crossdomain_en-ta_summary.csv b/Government/combined results/government_crossdomain_en-ta_summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..d5400f58d1b16c9566135b19d46cc590608f181e --- /dev/null +++ b/Government/combined results/government_crossdomain_en-ta_summary.csv @@ -0,0 +1,8 @@ +data_size,Bible_chrf_sacrebleuapi,Bible_chrfpp_sacrebleuapi,Bible_bleu_sacrebleuapi,Bible_spBLEU-flores101,Bible_spBLEU-flores200,Bible_comet,Government_chrf_sacrebleuapi,Government_chrfpp_sacrebleuapi,Government_bleu_sacrebleuapi,Government_spBLEU-flores101,Government_spBLEU-flores200,Government_comet,OpenSubtitles_chrf_sacrebleuapi,OpenSubtitles_chrfpp_sacrebleuapi,OpenSubtitles_bleu_sacrebleuapi,OpenSubtitles_spBLEU-flores101,OpenSubtitles_spBLEU-flores200,OpenSubtitles_comet,NLLB_chrf_sacrebleuapi,NLLB_chrfpp_sacrebleuapi,NLLB_bleu_sacrebleuapi,NLLB_spBLEU-flores101,NLLB_spBLEU-flores200,NLLB_comet,Wikipedia_chrf_sacrebleuapi,Wikipedia_chrfpp_sacrebleuapi,Wikipedia_bleu_sacrebleuapi,Wikipedia_spBLEU-flores101,Wikipedia_spBLEU-flores200,Wikipedia_comet,Literature_chrf_sacrebleuapi,Literature_chrfpp_sacrebleuapi,Literature_bleu_sacrebleuapi,Literature_spBLEU-flores101,Literature_spBLEU-flores200,Literature_comet,Literature-Children_chrf_sacrebleuapi,Literature-Children_chrfpp_sacrebleuapi,Literature-Children_bleu_sacrebleuapi,Literature-Children_spBLEU-flores101,Literature-Children_spBLEU-flores200,Literature-Children_comet,Literature-Kathlyn_chrf_sacrebleuapi,Literature-Kathlyn_chrfpp_sacrebleuapi,Literature-Kathlyn_bleu_sacrebleuapi,Literature-Kathlyn_spBLEU-flores101,Literature-Kathlyn_spBLEU-flores200,Literature-Kathlyn_comet,Literature-Old_chrf_sacrebleuapi,Literature-Old_chrfpp_sacrebleuapi,Literature-Old_bleu_sacrebleuapi,Literature-Old_spBLEU-flores101,Literature-Old_spBLEU-flores200,Literature-Old_comet,News_chrf_sacrebleuapi,News_chrfpp_sacrebleuapi,News_bleu_sacrebleuapi,News_spBLEU-flores101,News_spBLEU-flores200,News_comet,Maths_chrf_sacrebleuapi,Maths_chrfpp_sacrebleuapi,Maths_bleu_sacrebleuapi,Maths_spBLEU-flores101,Maths_spBLEU-flores200,Maths_comet,Health_chrf_sacrebleuapi,Health_chrfpp_sacrebleuapi,Health_bleu_sacrebleuapi,Health_spBLEU-flores101,Health_spBLEU-flores200,Health_comet,FloresPlus_chrf_sacrebleuapi,FloresPlus_chrfpp_sacrebleuapi,FloresPlus_bleu_sacrebleuapi,FloresPlus_spBLEU-flores101,FloresPlus_spBLEU-flores200,FloresPlus_comet,Jailbreak_chrf_sacrebleuapi,Jailbreak_chrfpp_sacrebleuapi,Jailbreak_bleu_sacrebleuapi,Jailbreak_spBLEU-flores101,Jailbreak_spBLEU-flores200,Jailbreak_comet +baseline,0.1820428517840199,1.1067003517608265,0.0110823711907671,0.0025371006574629,0.0027115722959845,0.3555766829550266,2.7326091482853245,3.4402962062567553,1.8190494396216712,1.5257398406012643,1.6493600666293629,0.533683074772358,0.7106221154527631,2.570921700813326,1.4788269145982211,0.1892647228888548,0.2536608589994819,0.6126294483244419,0.8396751780130931,2.4111867421786233,0.3066246178883174,0.2813822906072251,0.3096680563225658,0.5835503396391869,3.864874288629729,4.624060876786521,1.448036721709419,2.567352063547686,2.708872369094082,0.5320055771023035,0.4308148131801461,2.023331196074034,0.1512175048726044,0.0215280175368612,0.0231630885510583,0.4852443312432755,0.4308148131801461,2.023331196074034,0.1512175048726044,0.0215280175368612,0.0231630885510583,0.4852443312432755,0.4308148131801461,2.023331196074034,0.1512175048726044,0.0215280175368612,0.0231630885510583,0.4852443312432755,0.4308148131801461,2.023331196074034,0.1512175048726044,0.0215280175368612,0.0231630885510583,0.4852443312432755,1.998127703773364,3.3012853643443663,1.101227424070257,1.5735222160523783,1.6449114728899694,0.5120277152841829,2.8081650520647368,5.5836312752642145,6.501583604993177,4.206994549527542,4.512365149250633,0.5298982553035021,1.4038432938609309,2.4884344633513176,1.4751634938100298,0.8742299437387782,1.0600629870636833,0.4197459498494864,2.3375772397461256,3.339293286068496,1.0137212343306958,1.5814583844970087,1.5903384046591815,0.4869961341614779,0.950902745891338,2.5102398659453193,2.870091446825767,0.8270928935658232,1.170507674721423,0.5279000070542097 +1k,34.9595922976334,30.016010590575693,3.6955114039946846,11.425554236738714,10.234295696241674,0.7152103856056928,51.49403723919621,45.43089824989166,13.736301489950655,24.093670097241016,24.023475896842484,0.8820261645913124,23.543217403368963,22.582984314652695,8.67274438400228,14.710985334183531,14.895692831927844,0.6994686460494995,52.48340779674099,48.03652080074238,17.20047536975647,29.440567616607588,28.722471108002285,0.8617566767791719,56.255478335036344,49.83306218734586,13.02195300864623,30.373405621589413,30.195715254109636,0.8780179094821214,53.89895728855406,48.89007726591565,14.649069178790135,28.643382408052908,28.11198441735683,0.8140736782785367,53.89895728855406,48.89007726591565,14.649069178790135,28.643382408052908,28.11198441735683,0.8140736782785367,53.89895728855406,48.89007726591565,14.649069178790135,28.643382408052908,28.11198441735683,0.8140736782785367,53.89895728855406,48.89007726591565,14.649069178790135,28.643382408052908,28.11198441735683,0.8140736782785367,61.18633229487469,55.994558658194705,21.79529355470753,39.77194115398926,39.28750945693206,0.852673879768475,52.00734110365251,48.036037740621786,21.279910114693163,28.573803977208566,28.00619078972203,0.8331779134869576,60.68331352514558,54.275432087342246,16.359838256117694,35.505708648653545,34.3043864590842,0.8473187069296837,53.865656846497664,47.80073113672015,12.633814796264634,28.59163422478019,28.399945483860275,0.8676608577665132,58.10779627770626,53.1788446442861,21.707498859768968,35.19240880440027,34.731288781745924,0.8380884910523891 +5k,39.104270196186626,33.16444374880192,2.8756653381343487,13.64369627504619,12.35112913731052,0.7828839690685272,57.48870752318605,51.33326923292132,17.451089018577015,32.797904822354056,32.88863987169818,0.892828518897295,48.270452924617146,44.54937233725686,15.715368181882,27.295537890775297,27.037730056270444,0.8486921544075012,56.013767992105265,51.10331545240099,17.49474037237022,30.74849893729091,29.937989572443307,0.9035753657594062,54.76977526202764,48.428196118589256,11.978872103962386,29.1431958843411,28.692988400969934,0.874132488772273,52.73975783803249,47.36067825923689,12.8060934521314,26.16715572899859,25.169424200290603,0.8350815981867965,52.73975783803249,47.36067825923689,12.8060934521314,26.16715572899859,25.169424200290603,0.8350815981867965,52.73975783803249,47.36067825923689,12.8060934521314,26.16715572899859,25.169424200290603,0.8350815981867965,52.73975783803249,47.36067825923689,12.8060934521314,26.16715572899859,25.169424200290603,0.8350815981867965,58.92559041177843,53.330231600017186,17.836532112029595,34.421406579635025,33.976456445227754,0.8648977399351128,50.6870615642952,46.644072448648,20.39240768504329,26.561548371984117,25.92481296568236,0.8300952911973,58.66871165732608,52.11839595334895,13.722466517727376,31.967876341824944,30.735447764073637,0.8376800656765699,52.03516812879777,45.77780503134915,10.47506217472233,25.702371506369836,25.19030750123965,0.8678937781705216,56.912620918833454,51.79601398850566,19.88472608153632,32.20857214956527,31.37671701700715,0.8543761656731367 +10k,39.04016452320649,33.15214888174151,2.8184624261362243,13.751787694216905,12.653757012332289,0.7956526825428009,59.06764975646131,52.913556093013966,18.8233242367348,34.91316090158198,34.99057592315347,0.8964461959302426,44.84399057895997,41.46688671504414,14.827865536541932,25.05746381065306,25.058395301870657,0.826336010709405,55.701610270706645,50.770739016004974,16.868370570975276,29.73773902223746,29.018614315651423,0.902689752738534,53.76769626572674,47.42992356678965,11.011035596320127,27.81871954823268,27.688381316325337,0.878192524343729,52.80011373193968,47.35622145271364,12.741313369670015,26.446102922675102,25.894374314433485,0.8346316877890516,52.80011373193968,47.35622145271364,12.741313369670015,26.446102922675102,25.894374314433485,0.8346316877890516,52.80011373193968,47.35622145271364,12.741313369670015,26.446102922675102,25.894374314433485,0.8346316877890516,52.80011373193968,47.35622145271364,12.741313369670015,26.446102922675102,25.894374314433485,0.8346316877890516,58.12530632693921,52.442792795931496,16.683589966304222,33.32834972613715,33.03293022848524,0.8653240330876356,50.45288363552336,46.509719373423394,20.235962579515416,26.069489072072415,25.576828778799833,0.8329620827734471,58.33440706989346,51.67512736768906,13.453431672446712,31.75238413859764,30.682964623161613,0.8420447525680065,51.33106002932208,45.11555555810329,10.204760623961306,24.73892903928464,24.451190328689275,0.8639418881519039,56.21916360382115,51.056636792979745,18.72396709640088,30.6603151104876,29.946766608740884,0.8520219075381756 +15k,39.35146130939026,33.3893609747557,2.6919493432304913,14.209230774547496,12.92935046957307,0.798079612314701,60.32710475528184,54.11043139111106,20.73495912890604,36.29463437793894,36.41417193218862,0.9002067874670029,51.053005804755614,47.20893438933282,17.80236231837347,29.0229461422151,28.7448018857421,0.8637542776614427,55.95602263572816,50.96604446578231,16.97995308261609,29.940204717375405,29.06211337216374,0.9060601958381016,54.147304763835265,47.77457877302795,11.134996124969732,27.911489519257888,27.664231157728317,0.8771339770406484,53.330662605200885,47.977488579298054,13.7430360924881,27.0984453930422,26.13129626962484,0.8352127524989622,53.330662605200885,47.977488579298054,13.7430360924881,27.0984453930422,26.13129626962484,0.8352127524989622,53.330662605200885,47.977488579298054,13.7430360924881,27.0984453930422,26.13129626962484,0.8352127524989622,53.330662605200885,47.977488579298054,13.7430360924881,27.0984453930422,26.13129626962484,0.8352127524989622,58.18240480822938,52.583530136995606,16.883673473235667,33.380973106643324,33.202182667126806,0.8685271259711449,50.41074236607987,46.39616223416936,19.92487952308717,26.499396868002357,25.886182013385874,0.8344578335285187,58.100641671237,51.51679561098018,13.729242014213668,30.92716962659006,29.867780694387307,0.8373897092342377,51.603214168549805,45.46102073550337,10.678829148020034,24.980858657267618,24.55441496505266,0.8630498663886733,57.19502017847633,52.106862659615615,20.146883816765214,32.03345035713335,31.151251845438047,0.8521424018740654 +20k,38.28284821810499,32.467935193315014,2.620634215953889,12.909023254023582,11.71730023876448,0.7924355041086674,60.832506487062155,54.728394037524666,21.977958296988536,37.40296049724705,37.44162196006756,0.901136607259512,48.443547247449175,44.6555521776317,15.59176727548575,27.03275688100976,26.801265810086395,0.85416314214468,56.04076681501125,51.08520502581324,16.989963735299348,30.094701617954517,29.162302264301612,0.905110799828828,53.91530090818076,47.48323717505256,10.894641087837911,28.106499127369172,27.98331556305807,0.8761209879219533,52.47163916860495,47.0332261198968,12.643277181350124,25.68446427296771,24.678714971778387,0.8305922095077457,52.47163916860495,47.0332261198968,12.643277181350124,25.68446427296771,24.678714971778387,0.8305922095077457,52.47163916860495,47.0332261198968,12.643277181350124,25.68446427296771,24.678714971778387,0.8305922095077457,52.47163916860495,47.0332261198968,12.643277181350124,25.68446427296771,24.678714971778387,0.8305922095077457,57.74674327800203,52.02965375140137,15.87429821987729,32.689456054430885,32.341695161283745,0.8664364232475977,50.17759637654757,46.10077286825023,19.552887677839095,25.944749033962783,25.191118400261335,0.8320967829525471,57.864730475266526,51.2403734299827,13.537007166607967,30.4877518725062,29.57607711885231,0.8412932092249393,50.54670262782469,44.38126135159692,9.807197413498848,23.720731513652108,23.272251376965823,0.8616439145161465,56.40288106711744,51.35929482121719,19.36945799493403,31.485539332141823,30.495628128187413,0.8543994006514549 +25k,37.910253679604686,32.01692922260542,2.4064603779889935,12.96195055037878,11.953366145494822,0.7995613397806883,61.33114874800577,55.22221826862441,22.221961258105324,38.15551868828684,38.25538259594189,0.9011772488355636,49.96828470037169,46.0653176059842,16.388551708689157,27.76078544768291,27.583351063247836,0.859308678522706,55.89077974807722,50.92405687648808,16.77937992038363,29.683968294312574,28.96596565900932,0.9056717679016124,54.66213735077503,48.341725778921536,11.891592515162682,28.933939483948446,28.79674392890101,0.8784477740079164,52.94294062702861,47.48633204257806,12.65209411669922,25.921457143150043,25.42314085711707,0.8307426627380429,52.94294062702861,47.48633204257806,12.65209411669922,25.921457143150043,25.42314085711707,0.8307426627380429,52.94294062702861,47.48633204257806,12.65209411669922,25.921457143150043,25.42314085711707,0.8307426627380429,52.94294062702861,47.48633204257806,12.65209411669922,25.921457143150043,25.42314085711707,0.8307426627380429,57.96189383266107,52.355247567462946,16.7353650809541,33.13264694861089,32.75953716452201,0.865638548770104,50.43671068611862,46.27008156145329,19.451039210384923,25.95066408493507,25.35578743333428,0.8351800081133842,57.770063459033786,51.13982650874356,13.649049880812218,30.540986543530195,29.445217779048033,0.8380835425704718,51.168533619903165,44.988690975617224,10.36678989899248,24.3163674774914,23.88797990096679,0.8646409833148534,56.14701377675067,51.156393091774014,18.472593105129725,31.029411447858458,29.72512132689668,0.8519543393552304 diff --git a/Government/combined results/government_crossdomain_si-en_summary.csv b/Government/combined results/government_crossdomain_si-en_summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..8009f0b7275af61faf414047504bc80f68ea6d81 --- /dev/null +++ b/Government/combined results/government_crossdomain_si-en_summary.csv @@ -0,0 +1,8 @@ +data_size,Bible_chrf_sacrebleuapi,Bible_chrfpp_sacrebleuapi,Bible_bleu_sacrebleuapi,Bible_spBLEU-flores101,Bible_spBLEU-flores200,Bible_comet,Government_chrf_sacrebleuapi,Government_chrfpp_sacrebleuapi,Government_bleu_sacrebleuapi,Government_spBLEU-flores101,Government_spBLEU-flores200,Government_comet,OpenSubtitles_chrf_sacrebleuapi,OpenSubtitles_chrfpp_sacrebleuapi,OpenSubtitles_bleu_sacrebleuapi,OpenSubtitles_spBLEU-flores101,OpenSubtitles_spBLEU-flores200,OpenSubtitles_comet,NLLB_chrf_sacrebleuapi,NLLB_chrfpp_sacrebleuapi,NLLB_bleu_sacrebleuapi,NLLB_spBLEU-flores101,NLLB_spBLEU-flores200,NLLB_comet,Wikipedia_chrf_sacrebleuapi,Wikipedia_chrfpp_sacrebleuapi,Wikipedia_bleu_sacrebleuapi,Wikipedia_spBLEU-flores101,Wikipedia_spBLEU-flores200,Wikipedia_comet,Literature_chrf_sacrebleuapi,Literature_chrfpp_sacrebleuapi,Literature_bleu_sacrebleuapi,Literature_spBLEU-flores101,Literature_spBLEU-flores200,Literature_comet,Literature-Children_chrf_sacrebleuapi,Literature-Children_chrfpp_sacrebleuapi,Literature-Children_bleu_sacrebleuapi,Literature-Children_spBLEU-flores101,Literature-Children_spBLEU-flores200,Literature-Children_comet,Literature-Kathlyn_chrf_sacrebleuapi,Literature-Kathlyn_chrfpp_sacrebleuapi,Literature-Kathlyn_bleu_sacrebleuapi,Literature-Kathlyn_spBLEU-flores101,Literature-Kathlyn_spBLEU-flores200,Literature-Kathlyn_comet,Literature-Old_chrf_sacrebleuapi,Literature-Old_chrfpp_sacrebleuapi,Literature-Old_bleu_sacrebleuapi,Literature-Old_spBLEU-flores101,Literature-Old_spBLEU-flores200,Literature-Old_comet,News_chrf_sacrebleuapi,News_chrfpp_sacrebleuapi,News_bleu_sacrebleuapi,News_spBLEU-flores101,News_spBLEU-flores200,News_comet,Maths_chrf_sacrebleuapi,Maths_chrfpp_sacrebleuapi,Maths_bleu_sacrebleuapi,Maths_spBLEU-flores101,Maths_spBLEU-flores200,Maths_comet,Health_chrf_sacrebleuapi,Health_chrfpp_sacrebleuapi,Health_bleu_sacrebleuapi,Health_spBLEU-flores101,Health_spBLEU-flores200,Health_comet,FloresPlus_chrf_sacrebleuapi,FloresPlus_chrfpp_sacrebleuapi,FloresPlus_bleu_sacrebleuapi,FloresPlus_spBLEU-flores101,FloresPlus_spBLEU-flores200,FloresPlus_comet,Jailbreak_chrf_sacrebleuapi,Jailbreak_chrfpp_sacrebleuapi,Jailbreak_bleu_sacrebleuapi,Jailbreak_spBLEU-flores101,Jailbreak_spBLEU-flores200,Jailbreak_comet +baseline,45.7982751634026,44.93582368126017,22.655268358949783,24.23433589434345,23.81285028233166,0.7768540764153004,32.43705692240101,30.84695416771855,14.55282284869292,14.241602139556662,13.844908765044012,0.6612490080296993,0.2093695928524757,0.4610373422708847,0.2172444536953513,0.0348241300282007,0.0354425516877704,0.5909831962287426,4.342796789294033,4.749983204532853,2.807257100878493,2.5871081712100445,2.45253724891628,0.6258792190551757,29.78788601544511,28.904571784948065,14.614483505858551,15.328936597680402,15.164980042762892,0.65542942070961,12.68270519652462,12.506382621891632,4.647287447640901,4.748886789179166,4.84982681362838,0.5844122405648231,12.68270519652462,12.506382621891632,4.647287447640901,4.748886789179166,4.84982681362838,0.5844122405648231,12.68270519652462,12.506382621891632,4.647287447640901,4.748886789179166,4.84982681362838,0.5844122405648231,12.68270519652462,12.506382621891632,4.647287447640901,4.748886789179166,4.84982681362838,0.5844122405648231,28.476586976813845,27.171542359315197,12.62598973960596,12.65714651911421,12.389014001011189,0.6620795749127865,11.317448157669336,12.51008597255433,10.716065134149002,9.704364195477536,9.059869661837732,0.5851124048382044,44.92138874126114,42.792813526751225,20.948990193440864,24.18940878364083,23.40765789077685,0.7220131568694382,22.55840081391738,22.048393660536043,11.962472512396154,11.919058023999186,11.377242895806102,0.6375348256687402,13.70091286495427,13.301510063094474,7.9429792134790675,7.060747562369329,7.125536393775505,0.5843882857114077 +1k,45.519679436405895,44.56427531232765,21.64082810028898,23.12556477445832,22.756168684278343,0.7835294347405434,57.7801523647287,54.89723004749681,28.536492739972218,28.88016835917535,30.227912618064472,0.8545789907276631,64.47308072896288,63.74329046003354,43.67043920882924,43.98564205774316,44.25832162381529,0.8969421654343604,63.41284019658625,62.61060550849287,42.91050967755352,42.32787525245691,42.58545376539262,0.9012708021769568,55.132224891054925,53.208937399375245,27.568990711729228,29.601467860214452,30.229356099978883,0.8444438032507896,44.48527430359607,43.45372079058745,20.5791049450655,19.968255129646845,20.22726305440647,0.8111989430785179,44.48527430359607,43.45372079058745,20.5791049450655,19.968255129646845,20.22726305440647,0.8111989430785179,44.48527430359607,43.45372079058745,20.5791049450655,19.968255129646845,20.22726305440647,0.8111989430785179,44.48527430359607,43.45372079058745,20.5791049450655,19.968255129646845,20.22726305440647,0.8111989430785179,54.5944648728102,51.89513492854887,23.597437634350108,25.583373745829206,26.626821242478208,0.8546181761026382,58.57429503758378,57.988276525550745,39.17008812369143,39.31751521238576,39.553216712025744,0.8474531008601188,58.823952962234195,55.76889243030728,24.96873919761974,31.08439877453402,31.533984745001874,0.85347719641214,58.104840438076465,56.060526082433626,30.543987665322973,32.04481816309639,32.80023559608612,0.8685261862786863,54.9625739402548,53.21327625787566,30.63372234457116,30.852209509922588,31.487794552918498,0.8601958205401897 +5k,45.33522734519968,44.4217335563957,21.733600421347013,22.91738522167107,22.648938874611055,0.7763690453767776,60.99733266989447,58.223940529836106,32.76016680774736,34.19087842338811,35.339936404370995,0.8665921680033207,63.40878654318058,62.604126471230536,41.30336470327033,42.09400841117228,42.33976466887083,0.8930004394203425,62.78110346546808,62.05912069488389,42.21173315676192,41.70838449001414,41.91348074095811,0.9005061542755713,54.013321807431225,51.992516773370546,25.638065555853593,28.404409789625227,28.94878056957761,0.8408416794836521,43.69492188032749,42.61634074514172,19.82485043746595,19.36560532468245,19.55606985774085,0.807296268671751,43.69492188032749,42.61634074514172,19.82485043746595,19.36560532468245,19.55606985774085,0.807296268671751,43.69492188032749,42.61634074514172,19.82485043746595,19.36560532468245,19.55606985774085,0.807296268671751,43.69492188032749,42.61634074514172,19.82485043746595,19.36560532468245,19.55606985774085,0.807296268671751,55.00139455274541,52.31023841866155,23.658658442202828,25.794596124102387,26.69403926211541,0.8537135955691337,57.49368058033715,56.93484539279011,38.38131159782538,37.96777084280895,38.54294766248619,0.8448873843252659,57.73196066859465,54.65553290904695,23.68447635480341,29.445175969422657,29.92273280986862,0.8502491591686613,56.64196178285397,54.44740562310189,28.154022271469724,29.694605434536683,30.57014291561554,0.865584895898231,54.680471231588754,52.920082556983104,30.35033519201316,30.535097787680854,31.254736434782973,0.8590239198207855 +10k,45.07330718302318,44.13769071062262,21.65309928276407,22.884942868586275,22.55774563190717,0.7763391099870205,62.73844567044389,59.981579805973475,34.628357862719746,36.654224341305095,37.66515942097653,0.8708139849305153,62.57068956655811,61.76668331962446,40.21677890085725,41.20475903534689,41.44644800048119,0.8913131302297115,62.2963419600587,61.51956040246825,41.345476841818595,41.07059044087541,41.2644213431573,0.8979451639677248,54.56450179765024,52.47018091510572,25.835169532912634,28.99903907364859,29.498197853102567,0.8424103186130524,43.69342098316437,42.546686819581005,19.19400950178123,18.807084984445588,18.98680461063105,0.8059120564162732,43.69342098316437,42.546686819581005,19.19400950178123,18.807084984445588,18.98680461063105,0.8059120564162732,43.69342098316437,42.546686819581005,19.19400950178123,18.807084984445588,18.98680461063105,0.8059120564162732,43.69342098316437,42.546686819581005,19.19400950178123,18.807084984445588,18.98680461063105,0.8059120564162732,55.16912437882716,52.41452548941682,23.225707785715844,25.69047415193137,26.629105316212765,0.8533352153301239,57.016858143481265,56.42857018784528,37.39970757061199,37.218307322272175,37.74691390025642,0.8421808927953244,57.48438223269277,54.35155986906376,23.155730792464308,29.116973964187466,29.53171280944268,0.8489664266953307,56.99668379741255,54.809779080789056,28.151436901598544,29.99779167953794,30.877600166083894,0.8655534790085239,53.9237987618724,52.13672088503674,28.302042608214855,29.449375428703405,30.324157627931157,0.8554237298518419 +15k,45.43490212145097,44.461716905870546,21.836615165017935,23.113368684205422,22.79028610179947,0.7760936993658543,63.66743657611551,60.907282231187885,35.963847478597884,38.062048194354155,39.14898865638359,0.8755780427753925,62.309569067666494,61.51447385810135,39.85246405730881,40.53007397285569,40.78251697321025,0.8903046472668648,62.13359523376011,61.36838945744591,41.138863850499035,40.83376787968107,41.02992900037648,0.8981701793280233,54.596528187164026,52.52762694790201,25.839339896954755,29.11411895865148,29.585157945833636,0.8438672468662262,43.36910379797703,42.289734486998555,19.171156780144635,18.83634896985832,19.03263358047044,0.8026190398931503,43.36910379797703,42.289734486998555,19.171156780144635,18.83634896985832,19.03263358047044,0.8026190398931503,43.36910379797703,42.289734486998555,19.171156780144635,18.83634896985832,19.03263358047044,0.8026190398931503,43.36910379797703,42.289734486998555,19.171156780144635,18.83634896985832,19.03263358047044,0.8026190398931503,54.79661168252055,52.118872979463774,23.385029031710246,25.404756406612243,26.521867470756305,0.8521652072072029,57.310069958060105,56.76185730321125,38.43488474377299,38.07576167493571,38.69220960171209,0.8414851614534855,57.449860202434664,54.331743280233425,23.193473569032133,28.956658292795417,29.306935414676797,0.8493524404054278,56.66471171297911,54.55280182410841,28.147612954372494,29.996577761750903,30.809015410443397,0.8639266135782121,54.27109503581603,52.4599949261298,29.295788397067184,29.68282584631609,30.324293926971475,0.8559750660508871 +20k,44.91420789245757,43.873439137375634,21.301441819365863,22.351071106075548,22.206533266775025,0.7740990107506514,64.29823445582777,61.601305651445806,36.75204629114596,38.70142850164052,39.84540391536018,0.8764319905340672,61.747554780003625,60.91758251594824,38.6864159002422,39.59130143461301,39.86300529902638,0.8888738882243633,62.05599317037146,61.280281004093354,40.91603177497263,40.8178257183554,41.027606741137376,0.8979786402037664,54.349890181366554,52.24562779681402,25.14899465512415,28.50412595867878,28.93093362503844,0.8428374643325806,43.40740277127494,42.345012525977396,19.0703681106256,18.888681230286217,19.042522371038515,0.8063090334832669,43.40740277127494,42.345012525977396,19.0703681106256,18.888681230286217,19.042522371038515,0.8063090334832669,43.40740277127494,42.345012525977396,19.0703681106256,18.888681230286217,19.042522371038515,0.8063090334832669,43.40740277127494,42.345012525977396,19.0703681106256,18.888681230286217,19.042522371038515,0.8063090334832669,55.56985871516454,52.88673440348383,23.521531094749506,26.347645725152898,27.213331193706768,0.8544390945136547,57.20566080895111,56.58499485696334,37.49051482258888,37.57353683685339,38.15952997638359,0.844492649435997,57.10297271238651,54.03252746431237,22.972128657325474,28.48771281175676,28.890848978998367,0.8467513275280427,56.516526222256225,54.31503846883062,27.52217835877474,29.48182997817969,30.23545759180062,0.864110741985174,53.41263845863003,51.60699080241575,27.49977320692965,28.729020323413017,29.376364538435467,0.8524583500027657 +25k,45.44171048157175,44.52892482269925,21.98176525247642,23.20753380936823,22.94907765149802,0.774605580329895,64.58857041890641,61.87377282714578,37.248262998779424,39.46165914062977,40.52310463151705,0.8774390577673912,62.28766815442771,61.51414472315769,39.702805722647234,40.72100110310309,41.010220117202856,0.8896081002503633,61.81986238385338,61.0575686551013,40.87429966556002,40.70235601434973,40.87055641010563,0.8976878566082513,54.33205209749793,52.23850858529763,25.520181497053553,28.6899226315677,29.227820042803735,0.843238550812006,43.328827400718104,42.21937887996792,18.677081635943036,18.580052705829587,18.800094198809916,0.8038805679678916,43.328827400718104,42.21937887996792,18.677081635943036,18.580052705829587,18.800094198809916,0.8038805679678916,43.328827400718104,42.21937887996792,18.677081635943036,18.580052705829587,18.800094198809916,0.8038805679678916,43.328827400718104,42.21937887996792,18.677081635943036,18.580052705829587,18.800094198809916,0.8038805679678916,55.619649765821144,52.95920751394399,23.96108915448688,26.647218304162458,27.498005055406196,0.8542312571704388,57.0557577035196,56.463857046699296,37.12052800347161,37.13924306449029,37.77450764408659,0.8438830208182335,57.212120300084024,54.15630021537241,23.175948602163185,28.811125019591863,29.158821803333304,0.8477959724289648,56.731496713185145,54.60594823245899,27.95596981666909,29.922833678434063,30.64866952047091,0.8645521375854968,53.88183375650033,52.08565740531584,28.428531036040155,28.957961328026386,29.46748347959513,0.8550191217660904 diff --git a/Government/combined results/government_crossdomain_si-ta_summary.csv b/Government/combined results/government_crossdomain_si-ta_summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..7c729cf6d1d4c78819a4a68532daf26e76f49fb4 --- /dev/null +++ b/Government/combined results/government_crossdomain_si-ta_summary.csv @@ -0,0 +1,8 @@ +data_size,Bible_chrf_sacrebleuapi,Bible_chrfpp_sacrebleuapi,Bible_bleu_sacrebleuapi,Bible_spBLEU-flores101,Bible_spBLEU-flores200,Bible_comet,Government_chrf_sacrebleuapi,Government_chrfpp_sacrebleuapi,Government_bleu_sacrebleuapi,Government_spBLEU-flores101,Government_spBLEU-flores200,Government_comet,OpenSubtitles_chrf_sacrebleuapi,OpenSubtitles_chrfpp_sacrebleuapi,OpenSubtitles_bleu_sacrebleuapi,OpenSubtitles_spBLEU-flores101,OpenSubtitles_spBLEU-flores200,OpenSubtitles_comet,NLLB_chrf_sacrebleuapi,NLLB_chrfpp_sacrebleuapi,NLLB_bleu_sacrebleuapi,NLLB_spBLEU-flores101,NLLB_spBLEU-flores200,NLLB_comet,Wikipedia_chrf_sacrebleuapi,Wikipedia_chrfpp_sacrebleuapi,Wikipedia_bleu_sacrebleuapi,Wikipedia_spBLEU-flores101,Wikipedia_spBLEU-flores200,Wikipedia_comet,Literature_chrf_sacrebleuapi,Literature_chrfpp_sacrebleuapi,Literature_bleu_sacrebleuapi,Literature_spBLEU-flores101,Literature_spBLEU-flores200,Literature_comet,Literature-Children_chrf_sacrebleuapi,Literature-Children_chrfpp_sacrebleuapi,Literature-Children_bleu_sacrebleuapi,Literature-Children_spBLEU-flores101,Literature-Children_spBLEU-flores200,Literature-Children_comet,Literature-Kathlyn_chrf_sacrebleuapi,Literature-Kathlyn_chrfpp_sacrebleuapi,Literature-Kathlyn_bleu_sacrebleuapi,Literature-Kathlyn_spBLEU-flores101,Literature-Kathlyn_spBLEU-flores200,Literature-Kathlyn_comet,Literature-Old_chrf_sacrebleuapi,Literature-Old_chrfpp_sacrebleuapi,Literature-Old_bleu_sacrebleuapi,Literature-Old_spBLEU-flores101,Literature-Old_spBLEU-flores200,Literature-Old_comet,News_chrf_sacrebleuapi,News_chrfpp_sacrebleuapi,News_bleu_sacrebleuapi,News_spBLEU-flores101,News_spBLEU-flores200,News_comet,Maths_chrf_sacrebleuapi,Maths_chrfpp_sacrebleuapi,Maths_bleu_sacrebleuapi,Maths_spBLEU-flores101,Maths_spBLEU-flores200,Maths_comet,Health_chrf_sacrebleuapi,Health_chrfpp_sacrebleuapi,Health_bleu_sacrebleuapi,Health_spBLEU-flores101,Health_spBLEU-flores200,Health_comet,FloresPlus_chrf_sacrebleuapi,FloresPlus_chrfpp_sacrebleuapi,FloresPlus_bleu_sacrebleuapi,FloresPlus_spBLEU-flores101,FloresPlus_spBLEU-flores200,FloresPlus_comet,Jailbreak_chrf_sacrebleuapi,Jailbreak_chrfpp_sacrebleuapi,Jailbreak_bleu_sacrebleuapi,Jailbreak_spBLEU-flores101,Jailbreak_spBLEU-flores200,Jailbreak_comet +baseline,,,,,,,1.506499672099712,2.4557752982830663,1.4632295174138816,0.8168146703604148,0.9846654424814892,0.5261105810403823,0.1483256577480597,0.4502203040744262,0.2168377791627246,0.0158720623622216,0.0192236979810407,0.5707517316937447,0.3991315937132054,1.2016165396211462,0.1672614921158257,0.0492256066085154,0.0612625807307166,0.5897180299162865,0.6226172276234757,1.569811801568743,0.3599066800468254,0.1444285046708023,0.1507449763486991,0.5247272368520498,0.2009138710399402,0.9862013885203196,0.0439269115867961,0.0149505862470798,0.0160947403179194,0.5033425367050652,0.2009138710399402,0.9862013885203196,0.0439269115867961,0.0149505862470798,0.0160947403179194,0.5033425367050652,0.2009138710399402,0.9862013885203196,0.0439269115867961,0.0149505862470798,0.0160947403179194,0.5033425367050652,0.2009138710399402,0.9862013885203196,0.0439269115867961,0.0149505862470798,0.0160947403179194,0.5033425367050652,0.5369858883410602,1.5497125670514913,0.2322594182936931,0.0536642255607852,0.0773424208521478,0.5412610062968827,,,,,,,1.013064442321414,2.1382657200712902,0.9734489037312932,0.6359235764087909,0.7222283891295722,0.5001927641961648,0.5628607944577833,1.5334494611820455,0.601637909996467,0.2290556112313874,0.3074346832828144,0.5284592901382286,0.7098839047708805,1.6801276275033277,2.236073717290074,0.5717536472337825,0.808304947689558,0.5240307556092739 +1k,,,,,,,50.2443796188764,44.10493824421842,12.538969053246612,22.42627877247282,22.439774320757984,0.8743745173215867,36.323185021540496,33.21552307918757,11.559699054015397,20.3981987484158,20.47530392566763,0.7550140042155981,53.82644072837088,48.96399499026332,15.64400772655998,28.21661867520769,27.593187279545745,0.8876809190942,48.56815297697707,42.42238610194449,7.914428956367575,22.110250492175208,21.96760787336064,0.8537849247455597,45.13866684662315,40.07546567521024,8.198705487657577,18.005897082619562,17.594096148416693,0.8022552307798257,45.13866684662315,40.07546567521024,8.198705487657577,18.005897082619562,17.594096148416693,0.8022552307798257,45.13866684662315,40.07546567521024,8.198705487657577,18.005897082619562,17.594096148416693,0.8022552307798257,45.13866684662315,40.07546567521024,8.198705487657577,18.005897082619562,17.594096148416693,0.8022552307798257,53.2617833571283,47.44848569038954,11.42307905567309,26.463722638642164,26.46904198918585,0.859532229016505,,,,,,,53.78217793027862,47.105039169606194,9.66112278327546,25.42321036216343,24.746129997956235,0.8144897919300259,49.377805890374155,43.35548016326561,9.269762772435334,23.17183827724759,23.25917858100641,0.855572857316068,53.71652084735038,48.50413378360178,16.66258277987578,27.250011859180137,26.361991957852965,0.8376016582548619 +5k,,,,,,,56.12097833718678,49.733543358075515,17.030684140516204,30.7426792607728,30.89678369082139,0.8861886198818684,54.432059902914645,50.075506543199296,17.528247788305706,30.09849940414254,29.195537045213,0.8908426854386926,54.10965653462958,48.94280558343474,14.383224952538336,27.135943356000563,26.12773441010943,0.8999825488074803,47.54393797687138,41.262209610796106,6.82105375829823,21.135531898017184,20.775951029697076,0.8490756225883961,44.14071777997267,38.70461289488636,6.745391082329602,16.030624195019897,15.50282372158476,0.8027440217356102,44.14071777997267,38.70461289488636,6.745391082329602,16.030624195019897,15.50282372158476,0.8027440217356102,44.14071777997267,38.70461289488636,6.745391082329602,16.030624195019897,15.50282372158476,0.8027440217356102,44.14071777997267,38.70461289488636,6.745391082329602,16.030624195019897,15.50282372158476,0.8027440217356102,50.95998031662033,44.90239311813765,8.83243198122357,22.702569083306024,22.694441463061317,0.8536119359966007,,,,,,,51.37994416681284,44.600702669565,7.689568469864875,22.39699110248858,21.570591767918582,0.8083506729263736,47.23390610823547,41.11550442607534,7.782337803679392,19.932887956146672,19.798590547136538,0.8459384798149582,51.81668421812679,46.51265806217224,14.118832177970004,23.904679503172375,23.24996088975882,0.8370880727469922 +10k,,,,,,,58.14078422125692,51.75968329106598,18.738581491878676,33.452466220048635,33.60204293320304,0.8897950460314751,53.154454788185866,48.712022861518555,15.884186660493718,28.40768736031095,27.62017506581964,0.8876451987326145,53.303848750785654,48.15741744708899,13.72785644479278,26.326082651119886,25.30305922974426,0.89893731930169,47.526336143455275,41.23071711558105,6.9203346734527695,20.920052776866168,20.688363725891698,0.850786651223898,44.03344124736107,38.56380996498546,6.175457135375574,15.423270454345603,14.782094050787254,0.8010131987495399,44.03344124736107,38.56380996498546,6.175457135375574,15.423270454345603,14.782094050787254,0.8010131987495399,44.03344124736107,38.56380996498546,6.175457135375574,15.423270454345603,14.782094050787254,0.8010131987495399,44.03344124736107,38.56380996498546,6.175457135375574,15.423270454345603,14.782094050787254,0.8010131987495399,50.5543703039938,44.53653984259175,8.793252120251214,21.94804983747638,22.03497418894363,0.8499003647255802,,,,,,,51.05445562150454,44.1282584335198,7.43661377510804,22.47372435594028,21.762093358602204,0.8153357454480565,46.836618790209656,40.6531192195075,7.401129420771832,19.285564207292083,19.20115461267066,0.8505266855651211,51.49816428271053,46.14675934942884,14.048096092055278,23.686153244821465,22.950627944652982,0.8389946821331978 +15k,,,,,,,58.51502500815897,52.10340591896176,18.71465430045571,33.94814988996791,34.16824189172424,0.892300804167986,54.127066085395946,49.69329220502086,17.16696700590091,29.26894214159221,28.652638825484704,0.8884524757713079,53.51653697424845,48.41602536871634,14.103769750493685,26.635338195111228,25.807689005505196,0.8983319042863336,47.63386011934315,41.36829753464779,6.992413270637968,21.42097035492968,21.228218568403623,0.8501114219129086,44.42504437023244,39.027253448053614,6.704736604280777,16.12462739360897,15.357443866101152,0.80511938022936,44.42504437023244,39.027253448053614,6.704736604280777,16.12462739360897,15.357443866101152,0.80511938022936,44.42504437023244,39.027253448053614,6.704736604280777,16.12462739360897,15.357443866101152,0.80511938022936,44.42504437023244,39.027253448053614,6.704736604280777,16.12462739360897,15.357443866101152,0.80511938022936,50.44887726763213,44.51688307506956,9.174720208715458,22.10982496645205,22.061340830915096,0.849138046314399,,,,,,,50.641585133776026,43.91872731729796,7.56702385722543,21.66294710805543,20.83484070879369,0.8097094015731495,46.62147952798226,40.48548461848681,7.200193895231319,19.35224094154367,19.205403285057585,0.8485663407463802,51.00509178274327,45.801115742436075,13.660645461567997,23.874330078442313,23.09008631877295,0.8354445858299733 +20k,,,,,,,59.14881079657924,52.82544950369273,20.215442354544713,35.25547044957205,35.461619164613,0.894321031153202,53.08459688059487,48.65732276542112,16.262591401317337,28.4537268730661,27.756742230218972,0.8849549402892589,53.24051678377969,48.073056935221835,13.568906081074187,26.30433318149175,25.29748470040835,0.896335870414392,47.74047028670271,41.385700049697526,7.188332326652838,21.410848685094138,21.39304268462389,0.8521688247025013,43.93742835498326,38.49243562653017,6.303843648287157,15.444236178288651,14.804014853927448,0.7984641714718654,43.93742835498326,38.49243562653017,6.303843648287157,15.444236178288651,14.804014853927448,0.7984641714718654,43.93742835498326,38.49243562653017,6.303843648287157,15.444236178288651,14.804014853927448,0.7984641714718654,43.93742835498326,38.49243562653017,6.303843648287157,15.444236178288651,14.804014853927448,0.7984641714718654,49.75691223157466,43.71992192067232,8.308092514389523,21.38375516081867,21.43980144304131,0.8479972150056593,,,,,,,50.260324332065466,43.4422812064727,7.505271716002482,21.512867888862676,20.674334748665892,0.8072064611110457,46.487538553989125,40.31292580462015,6.9582131221492585,19.146769036928447,18.933340558617992,0.8471615392965055,50.91020212917918,45.596668896008374,13.316260018099728,23.26058858982264,22.461570864255275,0.8351807125210762 +25k,,,,,,,59.812702487886824,53.44480979898254,20.33552064617674,35.86678178644539,36.00773367568179,0.8959461055696011,53.1108460847585,48.70099406890239,16.50568694824706,27.93487100441276,27.014044900074236,0.8846168394088745,53.72629202724089,48.57216552361423,13.806014017060688,26.541412165796235,25.46648470378095,0.8990643055392573,47.47793495852912,41.258516450053776,7.22054423551598,21.45037334491328,21.22070792686838,0.8479302596896887,44.032299218428186,38.542971823818256,6.216689408046166,15.355083659335548,14.624086944787535,0.8002112233466621,44.032299218428186,38.542971823818256,6.216689408046166,15.355083659335548,14.624086944787535,0.8002112233466621,44.032299218428186,38.542971823818256,6.216689408046166,15.355083659335548,14.624086944787535,0.8002112233466621,44.032299218428186,38.542971823818256,6.216689408046166,15.355083659335548,14.624086944787535,0.8002112233466621,49.78500522186812,43.7689835650637,8.20206190883595,21.166895541021457,21.231448498492565,0.8485575679299091,,,,,,,50.329279921202954,43.48567558700648,7.346455770868597,21.272291935038847,20.46408102851401,0.8077254467168633,46.21132146865297,40.03171949968911,6.851872356963542,18.500530348417875,18.308961611100344,0.8455016231348392,50.37771910111668,45.07576589162182,12.36394486911458,22.6437505498606,21.71292236539571,0.8344147505164147 diff --git a/Government/combined results/government_crossdomain_ta-en_summary.csv b/Government/combined results/government_crossdomain_ta-en_summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..ed226a98bf265c659565cc86f0167779d21d2aa0 --- /dev/null +++ b/Government/combined results/government_crossdomain_ta-en_summary.csv @@ -0,0 +1,8 @@ +data_size,Bible_chrf_sacrebleuapi,Bible_chrfpp_sacrebleuapi,Bible_bleu_sacrebleuapi,Bible_spBLEU-flores101,Bible_spBLEU-flores200,Bible_comet,Government_chrf_sacrebleuapi,Government_chrfpp_sacrebleuapi,Government_bleu_sacrebleuapi,Government_spBLEU-flores101,Government_spBLEU-flores200,Government_comet,OpenSubtitles_chrf_sacrebleuapi,OpenSubtitles_chrfpp_sacrebleuapi,OpenSubtitles_bleu_sacrebleuapi,OpenSubtitles_spBLEU-flores101,OpenSubtitles_spBLEU-flores200,OpenSubtitles_comet,NLLB_chrf_sacrebleuapi,NLLB_chrfpp_sacrebleuapi,NLLB_bleu_sacrebleuapi,NLLB_spBLEU-flores101,NLLB_spBLEU-flores200,NLLB_comet,Wikipedia_chrf_sacrebleuapi,Wikipedia_chrfpp_sacrebleuapi,Wikipedia_bleu_sacrebleuapi,Wikipedia_spBLEU-flores101,Wikipedia_spBLEU-flores200,Wikipedia_comet,Literature_chrf_sacrebleuapi,Literature_chrfpp_sacrebleuapi,Literature_bleu_sacrebleuapi,Literature_spBLEU-flores101,Literature_spBLEU-flores200,Literature_comet,Literature-Children_chrf_sacrebleuapi,Literature-Children_chrfpp_sacrebleuapi,Literature-Children_bleu_sacrebleuapi,Literature-Children_spBLEU-flores101,Literature-Children_spBLEU-flores200,Literature-Children_comet,Literature-Kathlyn_chrf_sacrebleuapi,Literature-Kathlyn_chrfpp_sacrebleuapi,Literature-Kathlyn_bleu_sacrebleuapi,Literature-Kathlyn_spBLEU-flores101,Literature-Kathlyn_spBLEU-flores200,Literature-Kathlyn_comet,Literature-Old_chrf_sacrebleuapi,Literature-Old_chrfpp_sacrebleuapi,Literature-Old_bleu_sacrebleuapi,Literature-Old_spBLEU-flores101,Literature-Old_spBLEU-flores200,Literature-Old_comet,News_chrf_sacrebleuapi,News_chrfpp_sacrebleuapi,News_bleu_sacrebleuapi,News_spBLEU-flores101,News_spBLEU-flores200,News_comet,Maths_chrf_sacrebleuapi,Maths_chrfpp_sacrebleuapi,Maths_bleu_sacrebleuapi,Maths_spBLEU-flores101,Maths_spBLEU-flores200,Maths_comet,Health_chrf_sacrebleuapi,Health_chrfpp_sacrebleuapi,Health_bleu_sacrebleuapi,Health_spBLEU-flores101,Health_spBLEU-flores200,Health_comet,FloresPlus_chrf_sacrebleuapi,FloresPlus_chrfpp_sacrebleuapi,FloresPlus_bleu_sacrebleuapi,FloresPlus_spBLEU-flores101,FloresPlus_spBLEU-flores200,FloresPlus_comet,Jailbreak_chrf_sacrebleuapi,Jailbreak_chrfpp_sacrebleuapi,Jailbreak_bleu_sacrebleuapi,Jailbreak_spBLEU-flores101,Jailbreak_spBLEU-flores200,Jailbreak_comet +baseline,39.47622083734604,38.535686343285626,16.80805320253603,18.917533482605823,18.198721079028463,0.7528944167643785,52.3529295476339,49.38957776608853,22.983161463191124,22.78860494341881,23.888948978622665,0.8210956664085388,23.287211611003936,22.38927432915715,11.122744297292002,9.407024275001389,10.92931256708631,0.6389645711779595,42.07568030691544,40.858521039377784,26.18871728655078,23.279777518510983,25.91722171225571,0.7494174346327782,59.68646735470296,57.866382438661454,33.204412644274505,34.89017215436297,35.637236082588075,0.8468928661346435,33.532064508931796,32.15883836468728,14.273128624048406,14.123921514861046,15.284138938637296,0.6710098042256303,33.532064508931796,32.15883836468728,14.273128624048406,14.123921514861046,15.284138938637296,0.6710098042256303,33.532064508931796,32.15883836468728,14.273128624048406,14.123921514861046,15.284138938637296,0.6710098042256303,33.532064508931796,32.15883836468728,14.273128624048406,14.123921514861046,15.284138938637296,0.6710098042256303,55.95338991980899,54.10241056867392,32.40037756016967,33.277017412245605,34.37400931232348,0.8045579167195114,48.55646207985666,47.844840747445275,30.852783835081933,30.937461958666827,31.65396680699585,0.7837131512463092,60.18050958026944,57.52052470528929,30.45956583501814,33.85661078371377,34.56109868596526,0.8350872081518174,55.356985866580544,53.27514272756617,28.884337353160277,30.380139274020152,31.23775259989815,0.843577752648135,42.33599917092071,40.69509676471764,23.927384098520275,22.840365209874985,23.52094887359705,0.7381379562616348 +1k,39.75161387490352,38.70065033403796,16.618038541811128,18.634568567182622,18.002394763247366,0.756501064568758,56.05655518747233,53.31425305044379,27.395680142968754,27.31640954628235,28.43727114616367,0.8340802385210991,60.51562192009797,59.771888856269854,38.24495304092191,38.30404615626526,38.57233271486981,0.8747753919959068,63.30388362788709,62.561316987392054,43.1055615067307,42.21331735764612,42.484750301588114,0.8913898070157318,59.45726298732943,57.526204765255976,32.25733837627613,33.865652765902965,34.71708040021201,0.8478907163143158,50.85534112089745,50.03031690407106,26.34812694098741,25.71520616359661,26.279905137766537,0.8200640871568963,50.85534112089745,50.03031690407106,26.34812694098741,25.71520616359661,26.279905137766537,0.8200640871568963,50.85534112089745,50.03031690407106,26.34812694098741,25.71520616359661,26.279905137766537,0.8200640871568963,50.85534112089745,50.03031690407106,26.34812694098741,25.71520616359661,26.279905137766537,0.8200640871568963,60.6933361148328,58.92145564017714,34.55620099741434,34.9155807710764,35.97092486853162,0.8604976444179753,54.64784196754577,53.89762744806972,34.78873357474227,34.48074447431171,35.23792224710771,0.8342977927327156,59.45104570223155,56.60695794787922,28.579661522093712,32.17800497697126,32.881421889606045,0.8324152611792087,56.0838536046274,53.98404543363633,28.32553559027332,29.500632062307492,30.417167497105098,0.8509534669546742,55.75054509800249,54.428314269123,33.533878099037544,30.933041977866377,31.898920036378996,0.8466845436096191 +5k,39.37151886439601,38.37489519796626,16.399897348626975,18.26562117923812,17.627149209826893,0.7546233857274055,60.107155518614285,57.4457710791638,32.311082846928464,33.55085234615088,34.4590611198238,0.8479743801653385,60.2375088708884,59.542621070701095,37.73203593706005,38.06462375547255,38.33088629383616,0.8731431738883257,62.8475124059938,62.144966584928895,42.70043472874664,42.00573936028934,42.24314774074809,0.8903522048958289,59.11444411843373,57.148035802254526,31.35505151525585,34.01561366893348,34.56963405352514,0.8486373663544655,50.115983991974566,49.31229876069414,26.280567187594468,25.576055242195256,26.052869267090234,0.8162053002182532,50.115983991974566,49.31229876069414,26.280567187594468,25.576055242195256,26.052869267090234,0.8162053002182532,50.115983991974566,49.31229876069414,26.280567187594468,25.576055242195256,26.052869267090234,0.8162053002182532,50.115983991974566,49.31229876069414,26.280567187594468,25.576055242195256,26.052869267090234,0.8162053002182532,60.047573839598925,58.19651120554607,33.17435801105844,33.95815999261179,35.077209258460485,0.8596431332299987,54.71448645869131,53.98212664331717,34.865633675762915,34.49562297253713,35.2138555042198,0.8373731119334697,59.18970070926514,56.47359668928438,28.929360390084657,32.009640327426226,32.956460565560505,0.8328773201704025,55.77564075635998,53.61759194041663,27.47879700783284,28.93272675238803,29.848581440335153,0.8498090575688442,55.30634053524792,54.06575575883763,32.67612800144492,30.587048687078873,31.414799560176046,0.8421250923871993 +10k,39.66184981994951,38.70144410559059,16.635038725941627,18.45114127947077,17.833644803220327,0.7544466648101806,61.75798035193796,59.12903021473232,34.39499240839612,35.92470564198692,36.87249579225605,0.8535116041898727,59.37888756243647,58.611080366930466,36.011471758769666,36.921736269966054,37.18628782193747,0.8704207756072283,62.68994539190883,62.00439961885627,42.1481485980801,41.777717693770626,41.99095634978417,0.8899459793635505,58.71705205081543,56.79024100743819,30.6912228651309,33.38481512896798,33.970360670851335,0.844701763510704,50.17644250494791,49.375202104535795,26.14306719178659,25.516428252417697,26.009296617616663,0.8162300244870561,50.17644250494791,49.375202104535795,26.14306719178659,25.516428252417697,26.009296617616663,0.8162300244870561,50.17644250494791,49.375202104535795,26.14306719178659,25.516428252417697,26.009296617616663,0.8162300244870561,50.17644250494791,49.375202104535795,26.14306719178659,25.516428252417697,26.009296617616663,0.8162300244870561,59.44241851014225,57.543245973599454,32.02068197304198,32.989606765740824,34.080175890389235,0.8565062239287369,53.84290157280306,53.064500963953186,33.947287247661755,33.50291595801113,34.28204636472693,0.8322957644760609,58.86040077530651,56.06605342114281,28.22128878884421,31.735624558849164,32.58044952164305,0.831830090880394,55.43089314826211,53.2716331352053,26.94205797497817,28.53422766568191,29.344144666730266,0.8467663534780736,54.54161727059267,53.27600517263037,30.879667285745647,29.763568275179793,30.61953628467286,0.8417990252375602 +15k,38.93958526618946,37.9423666899648,15.89971395202742,17.724165475640607,17.08788519688558,0.7514738410115241,63.04912140181778,60.405523019129504,35.798060148238854,37.48408588690264,38.46184102871189,0.8601772767901421,58.97133468965048,58.25553140519777,35.8433465718555,36.49129606679034,36.78553642188729,0.868731607362628,62.122531248298095,61.423738860976584,41.175402204866195,40.927687728202656,41.18346186472626,0.8877022689095867,58.65983465313128,56.6802895135369,30.142395620650245,33.11914588724891,33.70708416313059,0.8469915004372597,50.190185691728914,49.424318295275064,26.457919601300983,25.555387915175853,25.992960120207965,0.8157399566844106,50.190185691728914,49.424318295275064,26.457919601300983,25.555387915175853,25.992960120207965,0.8157399566844106,50.190185691728914,49.424318295275064,26.457919601300983,25.555387915175853,25.992960120207965,0.8157399566844106,50.190185691728914,49.424318295275064,26.457919601300983,25.555387915175853,25.992960120207965,0.8157399566844106,59.30171072542917,57.48852277318919,32.08427537674978,33.049149792465755,34.08395652798429,0.8557109833661333,53.84658011107133,53.045838570945094,33.67318492445719,33.17528266750687,33.94058941032087,0.8335989883840084,59.067049911385126,56.3283334193523,28.487133101658152,31.97818434390073,32.75584316497059,0.8321624001860619,55.68135527981136,53.55910697731873,27.455037344067588,29.32706115143437,30.20855458604444,0.8470760650432156,55.045920882965426,53.797912454365495,32.82592941773117,30.46379047561909,31.30141057598678,0.8432262063622474 +20k,39.352881441156754,38.407028109180736,16.471995156082222,18.271513709309875,17.61234070563459,0.7510786295533181,63.70925685695593,61.05296545304575,36.695646360709205,38.62036185696437,39.57677341749939,0.8613841069936752,59.737405956280995,59.04830807240404,37.091242067017184,37.87127352387377,38.09516634037666,0.8707491488307715,62.539739987267374,61.82136271659,41.69018052840806,41.77167609895496,41.98730022828941,0.8877965231954724,59.11743537461197,57.25481268525855,31.142725749303025,33.95221240983595,34.52268266906763,0.8468463129401207,50.505420243857714,49.75731011942967,26.53612539809372,25.79033326888846,26.262142190743685,0.815742032567936,50.505420243857714,49.75731011942967,26.53612539809372,25.79033326888846,26.262142190743685,0.815742032567936,50.505420243857714,49.75731011942967,26.53612539809372,25.79033326888846,26.262142190743685,0.815742032567936,50.505420243857714,49.75731011942967,26.53612539809372,25.79033326888846,26.262142190743685,0.815742032567936,59.56161636541984,57.769991001503406,32.340413891555414,33.55668401317413,34.61805405095608,0.8567443510973789,54.390175800976536,53.56059851807995,34.23743796064806,33.83403558143353,34.5719096302564,0.8334979208707809,58.8758507333153,56.191984827088895,28.76203338344732,31.58639392202739,32.41846059697566,0.8308889925479889,55.37387878212861,53.254707170996426,26.958480768793823,28.634835686816626,29.578988533306852,0.8464345760614034,54.974062519610186,53.75881317245834,31.844467583229925,30.594213601152543,31.40440967247881,0.8428259556293487 +25k,38.65940473878675,37.649390484982774,15.81276195468595,17.58532704888579,17.004959992318227,0.7474953307807446,64.04725087499365,61.42962540191179,37.09339503750719,39.02163223194449,39.98901600838739,0.8628479547798633,59.519555151016256,58.830867638406104,36.621497819485775,37.49370983310459,37.777534996640874,0.8696107387244701,62.606355219389755,61.935482023287214,42.00175990650263,41.943221614836965,42.1864893150037,0.8881275742368226,58.63879617725423,56.64403548938728,29.95204599338054,33.153971722664544,33.74928818585589,0.8460434170663357,50.391050771844135,49.58735372946614,26.358126611234137,25.747113279621463,26.223809152734898,0.8133344068709347,50.391050771844135,49.58735372946614,26.358126611234137,25.747113279621463,26.223809152734898,0.8133344068709347,50.391050771844135,49.58735372946614,26.358126611234137,25.747113279621463,26.223809152734898,0.8133344068709347,50.391050771844135,49.58735372946614,26.358126611234137,25.747113279621463,26.223809152734898,0.8133344068709347,59.55414861647426,57.74703304137459,32.094170005209584,33.25771474480451,34.33577540413442,0.8558003563119705,53.86665625745603,53.15634700092144,34.00862873558283,33.6738715465911,34.46886136923999,0.8329405805170536,58.890632616108704,56.16350397751192,28.45907815148269,31.653800864205223,32.43401341399923,0.8308236742317676,55.05869435825216,52.89229749013958,26.56091591789399,28.275743167021503,29.247876665620577,0.845015874228223,54.31364825927708,53.049530758905064,31.48795173143003,29.637699578882415,30.300190451925477,0.840070496827364 diff --git a/Government/combined results/government_crossdomain_ta-si_summary.csv b/Government/combined results/government_crossdomain_ta-si_summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..6e58adc3c89b50d8df5b671548396ecc2cbb2e50 --- /dev/null +++ b/Government/combined results/government_crossdomain_ta-si_summary.csv @@ -0,0 +1,8 @@ +data_size,Bible_chrf_sacrebleuapi,Bible_chrfpp_sacrebleuapi,Bible_bleu_sacrebleuapi,Bible_spBLEU-flores101,Bible_spBLEU-flores200,Bible_comet,Government_chrf_sacrebleuapi,Government_chrfpp_sacrebleuapi,Government_bleu_sacrebleuapi,Government_spBLEU-flores101,Government_spBLEU-flores200,Government_comet,OpenSubtitles_chrf_sacrebleuapi,OpenSubtitles_chrfpp_sacrebleuapi,OpenSubtitles_bleu_sacrebleuapi,OpenSubtitles_spBLEU-flores101,OpenSubtitles_spBLEU-flores200,OpenSubtitles_comet,NLLB_chrf_sacrebleuapi,NLLB_chrfpp_sacrebleuapi,NLLB_bleu_sacrebleuapi,NLLB_spBLEU-flores101,NLLB_spBLEU-flores200,NLLB_comet,Wikipedia_chrf_sacrebleuapi,Wikipedia_chrfpp_sacrebleuapi,Wikipedia_bleu_sacrebleuapi,Wikipedia_spBLEU-flores101,Wikipedia_spBLEU-flores200,Wikipedia_comet,Literature_chrf_sacrebleuapi,Literature_chrfpp_sacrebleuapi,Literature_bleu_sacrebleuapi,Literature_spBLEU-flores101,Literature_spBLEU-flores200,Literature_comet,Literature-Children_chrf_sacrebleuapi,Literature-Children_chrfpp_sacrebleuapi,Literature-Children_bleu_sacrebleuapi,Literature-Children_spBLEU-flores101,Literature-Children_spBLEU-flores200,Literature-Children_comet,Literature-Kathlyn_chrf_sacrebleuapi,Literature-Kathlyn_chrfpp_sacrebleuapi,Literature-Kathlyn_bleu_sacrebleuapi,Literature-Kathlyn_spBLEU-flores101,Literature-Kathlyn_spBLEU-flores200,Literature-Kathlyn_comet,Literature-Old_chrf_sacrebleuapi,Literature-Old_chrfpp_sacrebleuapi,Literature-Old_bleu_sacrebleuapi,Literature-Old_spBLEU-flores101,Literature-Old_spBLEU-flores200,Literature-Old_comet,News_chrf_sacrebleuapi,News_chrfpp_sacrebleuapi,News_bleu_sacrebleuapi,News_spBLEU-flores101,News_spBLEU-flores200,News_comet,Maths_chrf_sacrebleuapi,Maths_chrfpp_sacrebleuapi,Maths_bleu_sacrebleuapi,Maths_spBLEU-flores101,Maths_spBLEU-flores200,Maths_comet,Health_chrf_sacrebleuapi,Health_chrfpp_sacrebleuapi,Health_bleu_sacrebleuapi,Health_spBLEU-flores101,Health_spBLEU-flores200,Health_comet,FloresPlus_chrf_sacrebleuapi,FloresPlus_chrfpp_sacrebleuapi,FloresPlus_bleu_sacrebleuapi,FloresPlus_spBLEU-flores101,FloresPlus_spBLEU-flores200,FloresPlus_comet,Jailbreak_chrf_sacrebleuapi,Jailbreak_chrfpp_sacrebleuapi,Jailbreak_bleu_sacrebleuapi,Jailbreak_spBLEU-flores101,Jailbreak_spBLEU-flores200,Jailbreak_comet +baseline,,,,,,,2.1264790250606924,2.8059466038175693,1.6139690857599398,1.1365380107647691,0.963688045159638,0.5284281494617462,0.7973949256386279,2.406827458170219,1.0616449149333549,0.2178005764513419,0.2316938084927182,0.5833025423586369,0.77098643919795,1.998452512735229,0.3758215093238002,0.1157819799315809,0.1505558957515414,0.5912962484657764,0.8516461597485614,1.8100795334610311,0.3236137104946885,0.1423561347192973,0.1365804452582944,0.5140674581825733,0.4109079281980246,1.4658801399815615,0.1070041023726659,0.0221457000919698,0.0201868617670006,0.5111609247796659,0.4109079281980246,1.4658801399815615,0.1070041023726659,0.0221457000919698,0.0201868617670006,0.5111609247796659,0.4109079281980246,1.4658801399815615,0.1070041023726659,0.0221457000919698,0.0201868617670006,0.5111609247796659,0.4109079281980246,1.4658801399815615,0.1070041023726659,0.0221457000919698,0.0201868617670006,0.5111609247796659,2.028414304903063,2.5912661213586685,0.9266677410931268,0.4042689190115334,0.3726255123294969,0.5168055228949073,,,,,,,2.623521733310948,3.1080422505838414,1.0873166695415593,1.1456848084903108,0.9758022354772898,0.4921825256730829,0.8482613919797445,1.87764022307322,0.6389513759492589,0.2639885277142922,0.2856255702972314,0.5167262381303451,2.155647341058514,3.0479886610603177,2.7193614809686384,0.836017800841948,0.8711490692234946,0.5428231197297573 +1k,,,,,,,46.95468711618753,42.71863536497049,13.946403489944109,28.67080157940711,35.66790765231362,0.8795496428310872,41.19187592870378,38.419747113569834,13.543599122251214,23.98665182406473,26.149743096018017,0.8803802770078182,43.44096000550498,40.04578032668024,12.345930720666471,23.55773478102019,27.371812657261128,0.8940775383131777,39.239179268723184,35.08154507375792,7.089033852800834,20.430535571924903,23.64374864390406,0.8513488554060459,32.21343052357557,29.07701645656392,5.4597256404869,12.505948053229249,14.27471265084141,0.8002993582459851,32.21343052357557,29.07701645656392,5.4597256404869,12.505948053229249,14.27471265084141,0.8002993582459851,32.21343052357557,29.07701645656392,5.4597256404869,12.505948053229249,14.27471265084141,0.8002993582459851,32.21343052357557,29.07701645656392,5.4597256404869,12.505948053229249,14.27471265084141,0.8002993582459851,38.72222656917083,34.64803824339464,7.588713776478428,21.290457189445423,24.660552995976907,0.8313557970968466,,,,,,,38.80244151460313,35.118936902723284,9.915471801860118,22.41068104302556,25.31076371035167,0.7955503560108493,40.185809049862115,36.195736062588395,8.655722843861811,22.222520270610033,25.833272205353627,0.8500500880329034,40.2176191045428,37.05086886311183,13.14964171911592,22.21999833121419,25.24383735605519,0.820767950668931 +5k,,,,,,,51.41926015585591,47.17738904375178,17.444572340694904,35.27541311412441,41.78334972760448,0.8926729651689529,34.33365229924564,31.362802703379582,7.442684778858739,18.426800660151997,20.06852154162444,0.8642446266561746,42.03424671487618,38.71582745228877,11.287307485401328,22.42073353637314,26.325389675886218,0.8902426442085833,38.86199731494143,34.62355952934438,6.433818298061117,19.883944774417436,23.10354666443403,0.8495119138360023,31.32002834430054,28.134170682083813,4.480587464149585,11.491425934951907,13.227056488225278,0.799190194723695,31.32002834430054,28.134170682083813,4.480587464149585,11.491425934951907,13.227056488225278,0.799190194723695,31.32002834430054,28.134170682083813,4.480587464149585,11.491425934951907,13.227056488225278,0.799190194723695,31.32002834430054,28.134170682083813,4.480587464149585,11.491425934951907,13.227056488225278,0.799190194723695,38.06765279142419,33.94830309744808,7.201166739204406,20.49284853547799,23.95498462560696,0.8293779508601273,,,,,,,37.65115327528594,33.743185841828364,8.574490363016755,20.69555348481924,23.218023579085937,0.7880995444269204,39.12278945002164,35.135536187359804,7.654368709400647,21.329424819398096,24.89436969453728,0.8447011141437787,38.58916766639129,35.548020932245,12.016463336897296,20.18377192507685,22.706979516016947,0.8178093786835671 +10k,,,,,,,53.003758933588315,48.821356943977854,19.31417763140684,37.860609529253814,44.2256042868842,0.8984571205079556,32.779991853044535,29.769163253223105,6.336988867612174,17.075376725173953,18.95436559347672,0.8551582052856683,41.14962374342426,37.83151456057326,10.654342943989796,21.77550293245442,25.623770716708265,0.8870724192334603,38.4952395569435,34.30218847145008,6.345584298933479,19.66453846144426,22.87273195578639,0.8488303147852421,30.75703384311096,27.58530232637968,4.349458365486436,10.994093247594384,12.700897148810842,0.7969094352745121,30.75703384311096,27.58530232637968,4.349458365486436,10.994093247594384,12.700897148810842,0.7969094352745121,30.75703384311096,27.58530232637968,4.349458365486436,10.994093247594384,12.700897148810842,0.7969094352745121,30.75703384311096,27.58530232637968,4.349458365486436,10.994093247594384,12.700897148810842,0.7969094352745121,37.45298774264474,33.35210096895649,6.719988257674699,19.863863864849968,23.370670517369696,0.825703306333913,,,,,,,37.03803416828515,33.12868511463382,8.343706010206716,20.31863699567055,22.899490207344897,0.7829769134749563,38.55509522752336,34.58953090015751,7.345789897949188,20.66906789337079,24.236483807724387,0.8423951678567018,38.24081710579901,35.136434247784166,11.725754619794358,20.08262685955402,22.570575210159387,0.8129719280898571 +15k,,,,,,,53.96876394544066,49.68875362882517,19.57129556993449,39.16798179399488,45.52770375688494,0.9014797831177712,32.52286357531791,29.487865443464617,6.248544947911525,17.09394300575521,18.70591304753473,0.8551399844735861,40.78065473084989,37.457620865710304,10.342019107119588,21.709779587946738,25.558254074279883,0.8867994582004328,38.44667851910203,34.258587708481436,6.186473063416697,19.75736103493816,23.017708074232512,0.8492549396157265,30.847155564452667,27.666299764563313,4.727389743288037,11.18376805290996,12.876095051785802,0.799267442432982,30.847155564452667,27.666299764563313,4.727389743288037,11.18376805290996,12.876095051785802,0.799267442432982,30.847155564452667,27.666299764563313,4.727389743288037,11.18376805290996,12.876095051785802,0.799267442432982,30.847155564452667,27.666299764563313,4.727389743288037,11.18376805290996,12.876095051785802,0.799267442432982,37.39422991381425,33.25335478871849,6.7076017167597035,19.666608785867624,23.247194308801653,0.8248296706306357,,,,,,,36.94074923703173,33.023939801820724,7.766407385787166,19.567224746683117,22.092532643274968,0.7912866747652998,38.45302325056041,34.450685923333786,7.005155614085869,20.396857388683156,23.95739109565358,0.8433839754595351,37.70430986761613,34.67567508096997,11.514139456675752,19.386135763634236,21.834840330552364,0.8098987563550473 +20k,,,,,,,55.04726933015037,50.79367153843363,21.29440586973716,40.66185853738951,46.80437621858133,0.904481227517128,32.37243219819934,29.3066901510309,5.980697619581673,16.681144463590545,18.476464714958414,0.8546403640657664,40.70668509577528,37.42139152547206,10.40911683528101,21.36231324881069,25.230424046965325,0.8851708894780559,38.315255110238624,34.157239328495564,6.343790929618099,19.59889799872615,22.762480237872943,0.8501061606109143,30.15441975702176,27.009932319452407,4.1769704779573775,10.643300021881029,12.345638717868718,0.7941161549406935,30.15441975702176,27.009932319452407,4.1769704779573775,10.643300021881029,12.345638717868718,0.7941161549406935,30.15441975702176,27.009932319452407,4.1769704779573775,10.643300021881029,12.345638717868718,0.7941161549406935,30.15441975702176,27.009932319452407,4.1769704779573775,10.643300021881029,12.345638717868718,0.7941161549406935,37.28571821610571,33.188914491541766,6.5162986350236345,19.537762120428734,22.92513616951124,0.8252781428276531,,,,,,,37.08153859266698,33.15436499964011,7.982183313326316,19.493380535411852,21.932104589943624,0.7907066747964341,38.394709144731394,34.40134327692656,7.262813656482166,20.372393786070468,23.936931551113577,0.8433336493643847,37.12786168243185,34.14130791151598,10.769647874978968,19.216753233688237,21.49082430021167,0.8091989916861058 +25k,,,,,,,55.164204075164214,50.910692720283855,21.097118075257388,41.11269138796458,47.31720115936583,0.9045515403747558,32.115498882583545,29.0352322424891,5.715035714226997,16.597641369098486,18.362433692058858,0.8535309534668922,40.78127075286322,37.45895860642715,10.199632950644968,21.514486477221592,25.243830966859765,0.8861108106996676,38.24462187210634,34.101542942234126,6.004275881201997,19.309101197122633,22.61815711724994,0.8478409900069237,30.04457413782446,26.872293760071443,3.998389396606069,10.3177642268497,12.016513765533656,0.7933493303191074,30.04457413782446,26.872293760071443,3.998389396606069,10.3177642268497,12.016513765533656,0.7933493303191074,30.04457413782446,26.872293760071443,3.998389396606069,10.3177642268497,12.016513765533656,0.7933493303191074,30.04457413782446,26.872293760071443,3.998389396606069,10.3177642268497,12.016513765533656,0.7933493303191074,37.26994610605991,33.210986519051716,6.6850183927221325,19.603112341323836,23.067051210855503,0.822933440081655,,,,,,,37.12234504381412,33.219539031380336,7.841096632808772,19.806591952411605,22.352163119496407,0.7895335042659117,38.21677557606723,34.181725386295795,6.884206151628509,20.18145543822757,23.79556439497183,0.8404766332372846,37.32610532361566,34.27432287596386,10.97646499413233,19.120775618412832,21.580127054022135,0.8094877264499665 diff --git a/Government/nllb_bible_eval_government_en-si_baseline.csv b/Government/nllb_bible_eval_government_en-si_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..e2ab8a644c5fff59a3552ad1a5e7387c45997422 --- /dev/null +++ b/Government/nllb_bible_eval_government_en-si_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.2074466786941936,0.9924046443660439,0.01141848384807256,0.002187182226141608,0.0023078106301359044,0.351035983428359 diff --git a/Government/nllb_bible_eval_government_en-ta_baseline.csv b/Government/nllb_bible_eval_government_en-ta_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..a2e9a2ffa2017708acfd7ddba29c49aacf32f3fc --- /dev/null +++ b/Government/nllb_bible_eval_government_en-ta_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.18204285178401997,1.1067003517608263,0.011082371190767125,0.002537100657462906,0.0027115722959845288,0.35557668295502665 diff --git a/Government/nllb_bible_eval_government_si-en_baseline.csv b/Government/nllb_bible_eval_government_si-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..f0ecc72c5e75778f859d96119e50a1a525c20c50 --- /dev/null +++ b/Government/nllb_bible_eval_government_si-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,45.7982751634026,44.93582368126017,22.655268358949783,24.23433589434345,23.81285028233166,0.7768540764153004 diff --git a/Government/nllb_bible_eval_government_ta-en_baseline.csv b/Government/nllb_bible_eval_government_ta-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..f5d0991756d445d1b7d742044905f5f50503b2fb --- /dev/null +++ b/Government/nllb_bible_eval_government_ta-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,39.47622083734604,38.535686343285626,16.80805320253603,18.917533482605826,18.198721079028466,0.7528944167643785 diff --git a/Government/nllb_floresplus_eval_government_en-si_baseline.csv b/Government/nllb_floresplus_eval_government_en-si_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..37a49a7937a9b8890001fd758c671a2d36c8c130 --- /dev/null +++ b/Government/nllb_floresplus_eval_government_en-si_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.783503884977641,1.7245734048828818,0.5241960228541523,0.2973223112210829,0.3139251616408782,0.46672248145337164 diff --git a/Government/nllb_floresplus_eval_government_en-ta_baseline.csv b/Government/nllb_floresplus_eval_government_en-ta_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..ced0d878d15ecf8f2cbd3ee6e9e55e78ed854719 --- /dev/null +++ b/Government/nllb_floresplus_eval_government_en-ta_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,2.3375772397461256,3.339293286068496,1.0137212343306958,1.5814583844970087,1.5903384046591815,0.48699613416147797 diff --git a/Government/nllb_floresplus_eval_government_si-en_baseline.csv b/Government/nllb_floresplus_eval_government_si-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..fe344967d4f45206c9edc7a0e867f9597859b633 --- /dev/null +++ b/Government/nllb_floresplus_eval_government_si-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,22.55840081391738,22.048393660536043,11.962472512396154,11.919058023999186,11.377242895806102,0.6375348256687402 diff --git a/Government/nllb_floresplus_eval_government_si-ta_baseline.csv b/Government/nllb_floresplus_eval_government_si-ta_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..96e2e354616351091c3b8bd59377d617258659cd --- /dev/null +++ b/Government/nllb_floresplus_eval_government_si-ta_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.5628607944577833,1.5334494611820457,0.601637909996467,0.2290556112313874,0.3074346832828144,0.5284592901382286 diff --git a/Government/nllb_floresplus_eval_government_ta-en_baseline.csv b/Government/nllb_floresplus_eval_government_ta-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..0f0c0ff7507b8145c634d74379a211f327845974 --- /dev/null +++ b/Government/nllb_floresplus_eval_government_ta-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,55.356985866580544,53.27514272756617,28.884337353160277,30.380139274020152,31.23775259989815,0.843577752648135 diff --git a/Government/nllb_floresplus_eval_government_ta-si_baseline.csv b/Government/nllb_floresplus_eval_government_ta-si_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..2de644f903c059b3a4bba45cf43861dbb5aa46a9 --- /dev/null +++ b/Government/nllb_floresplus_eval_government_ta-si_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.8482613919797445,1.87764022307322,0.6389513759492589,0.2639885277142922,0.2856255702972314,0.5167262381303451 diff --git a/Government/nllb_government_eval_bible_en-si_combined.csv b/Government/nllb_government_eval_bible_en-si_combined.csv new file mode 100644 index 0000000000000000000000000000000000000000..dc62cfe262667732c50121f88ec1a39907a13ebe --- /dev/null +++ b/Government/nllb_government_eval_bible_en-si_combined.csv @@ -0,0 +1,7 @@ +model,data_size,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-si_final_model,1k,23.51630496074869,20.36270633286557,1.6177223328136954,7.698804887228647,7.142460377516814,0.6451798568218946 +en-si_final_model,5k,17.38218305920879,14.720861483249708,0.6110980468231604,4.57594442109759,4.215972757003348,0.5760666883587837 +en-si_final_model,10k,9.883167308842296,8.08164852593724,0.1224908082681523,1.6782909910263053,1.51688499860891,0.4502922486960888 +en-si_final_model,15k,10.333521780820629,8.453661202177699,0.0665625178426419,1.5401320162594443,1.3016910398397168,0.4509024264812469 +en-si_final_model,20k,31.761604881464237,27.54796001571456,2.7451206165921933,13.882638289577836,13.98031568061058,0.7690197389423847 +en-si_final_model,25k,34.456340875004464,29.91541262040702,2.927627634611363,16.043010632978344,16.549307363154742,0.7995621707439423 diff --git a/Government/nllb_government_eval_bible_en-ta_combined.csv b/Government/nllb_government_eval_bible_en-ta_combined.csv new file mode 100644 index 0000000000000000000000000000000000000000..5ea4dc1ad2d2312e43732d63098e41f40a71ba11 --- /dev/null +++ b/Government/nllb_government_eval_bible_en-ta_combined.csv @@ -0,0 +1,7 @@ +model,data_size,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +en-ta_final_model,1k,34.9595922976334,30.016010590575693,3.6955114039946846,11.425554236738714,10.234295696241674,0.7152103856056928 +en-ta_final_model,5k,39.10427019618663,33.16444374880192,2.8756653381343487,13.64369627504619,12.35112913731052,0.7828839690685272 +en-ta_final_model,10k,39.04016452320649,33.15214888174151,2.8184624261362243,13.751787694216905,12.653757012332289,0.7956526825428009 +en-ta_final_model,15k,39.35146130939026,33.3893609747557,2.6919493432304917,14.209230774547496,12.92935046957307,0.798079612314701 +en-ta_final_model,20k,38.28284821810499,32.467935193315014,2.620634215953889,12.909023254023582,11.71730023876448,0.7924355041086674 +en-ta_final_model,25k,37.910253679604686,32.01692922260542,2.406460377988993,12.96195055037878,11.953366145494822,0.7995613397806883 diff --git a/Government/nllb_government_eval_bible_si-en_combined.csv b/Government/nllb_government_eval_bible_si-en_combined.csv new file mode 100644 index 0000000000000000000000000000000000000000..dc10c14e66121a2fa52529e613194dc718587e3b --- /dev/null +++ b/Government/nllb_government_eval_bible_si-en_combined.csv @@ -0,0 +1,7 @@ +model,data_size,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +si-en_final_model,1k,45.519679436405895,44.56427531232765,21.64082810028898,23.12556477445832,22.756168684278343,0.7835294347405434 +si-en_final_model,5k,45.33522734519968,44.4217335563957,21.733600421347013,22.91738522167107,22.64893887461105,0.7763690453767776 +si-en_final_model,10k,45.07330718302318,44.13769071062262,21.65309928276407,22.884942868586275,22.55774563190717,0.7763391099870205 +si-en_final_model,15k,45.43490212145097,44.46171690587055,21.836615165017932,23.113368684205422,22.79028610179947,0.7760936993658543 +si-en_final_model,20k,44.91420789245757,43.873439137375634,21.301441819365863,22.351071106075548,22.206533266775025,0.7740990107506514 +si-en_final_model,25k,45.44171048157175,44.52892482269925,21.98176525247642,23.20753380936823,22.94907765149802,0.774605580329895 diff --git a/Government/nllb_government_eval_bible_ta-en_combined.csv b/Government/nllb_government_eval_bible_ta-en_combined.csv new file mode 100644 index 0000000000000000000000000000000000000000..2c582d0d05ae8dcce9e6c361a118bb5873d8e559 --- /dev/null +++ b/Government/nllb_government_eval_bible_ta-en_combined.csv @@ -0,0 +1,7 @@ +model,data_size,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +ta-en_final_model,1k,39.75161387490352,38.70065033403796,16.618038541811128,18.634568567182622,18.002394763247366,0.756501064568758 +ta-en_final_model,5k,39.37151886439601,38.37489519796626,16.399897348626975,18.26562117923812,17.627149209826893,0.7546233857274055 +ta-en_final_model,10k,39.66184981994951,38.70144410559059,16.635038725941627,18.45114127947077,17.833644803220327,0.7544466648101806 +ta-en_final_model,15k,38.93958526618946,37.9423666899648,15.89971395202742,17.724165475640607,17.08788519688558,0.7514738410115241 +ta-en_final_model,20k,39.352881441156754,38.407028109180736,16.471995156082222,18.271513709309875,17.61234070563459,0.7510786295533181 +ta-en_final_model,25k,38.65940473878675,37.64939048498277,15.81276195468595,17.58532704888579,17.004959992318227,0.7474953307807446 diff --git a/Government/nllb_government_eval_en-si_baseline.csv b/Government/nllb_government_eval_en-si_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..152e46f66ae3586e03b468c4144e431f5ca62a80 --- /dev/null +++ b/Government/nllb_government_eval_en-si_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,1.9596249942394388,2.525279654343046,1.5781507896905118,1.169071966253855,1.023921429080648,0.5461680391209032 diff --git a/Government/nllb_government_eval_en-si_combined.csv b/Government/nllb_government_eval_en-si_combined.csv new file mode 100644 index 0000000000000000000000000000000000000000..bfb3e138c33e538f71b583a461a8267388f54e77 --- /dev/null +++ b/Government/nllb_government_eval_en-si_combined.csv @@ -0,0 +1,9 @@ +model,data_size,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,baseline,1.9596249942394388,2.525279654343046,1.5781507896905118,1.169071966253855,1.023921429080648,0.5461680391209032 +en-si_final_model,1k,7.749347640855546,6.205670520341053,0.0431638824976943,0.2019887583922939,0.3459742982443122,0.379277535039626 +en-si_final_model,5k,8.479920459754377,6.767447703245162,0.1766950042221418,0.8276832754112516,1.3128873218794352,0.4958247425564809 +en-si_final_model,10k,7.902449368038944,6.167021841193213,0.1291368591937965,0.6381179265527395,1.0437367875692796,0.4717434812504041 +en-si_final_model,15k,9.654245870815185,7.730708090493862,0.1164836727252356,0.3898976160878543,0.5182228231390497,0.3989743923192021 +en-si_final_model,20k,56.878869875934015,52.578890021001925,22.17879298077314,43.632198515862186,49.66696619897928,0.921906360505286 +en-si_final_model,25k,57.28053792873068,53.00412598137066,22.52790319592445,44.41770266772205,50.36404431511058,0.9223912889015912 +en-si_final_model,59k,58.63215636177139,54.39797812614528,23.21054740802092,46.32217428726776,52.096225229635294,0.9270260595853944 diff --git a/Government/nllb_government_eval_government_en-si_baseline.csv b/Government/nllb_government_eval_government_en-si_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..d6f5855d5da0a969b8c0f1e032874a108915f0b2 --- /dev/null +++ b/Government/nllb_government_eval_government_en-si_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,1.9957824955315875,2.499977992441443,1.6800513225163616,1.212937758336134,1.055154151403294,0.5387619452029466 diff --git a/Government/nllb_government_eval_government_en-ta_baseline.csv b/Government/nllb_government_eval_government_en-ta_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..fcf1111a422a803f5527c9e9aa422c3437ab7b4c --- /dev/null +++ b/Government/nllb_government_eval_government_en-ta_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,2.7326091482853245,3.4402962062567553,1.8190494396216714,1.5257398406012643,1.6493600666293629,0.533683074772358 diff --git a/Government/nllb_government_eval_government_si-en_baseline.csv b/Government/nllb_government_eval_government_si-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..3fac512cb60bdcb198a62a864901f92c19cb8b07 --- /dev/null +++ b/Government/nllb_government_eval_government_si-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,32.43705692240101,30.84695416771855,14.55282284869292,14.241602139556662,13.844908765044012,0.6612490080296993 diff --git a/Government/nllb_government_eval_government_si-ta_baseline.csv b/Government/nllb_government_eval_government_si-ta_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..02031283a6997aee8cd90f6b9e4d9a321c2f0cd8 --- /dev/null +++ b/Government/nllb_government_eval_government_si-ta_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,1.5064996720997121,2.4557752982830663,1.4632295174138816,0.8168146703604148,0.9846654424814891,0.5261105810403823 diff --git a/Government/nllb_government_eval_government_ta-en_baseline.csv b/Government/nllb_government_eval_government_ta-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..5dde6cf5a33d1ae72a80239c6aeb535c1c0df270 --- /dev/null +++ b/Government/nllb_government_eval_government_ta-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,52.3529295476339,49.38957776608853,22.983161463191124,22.78860494341881,23.888948978622665,0.8210956664085388 diff --git a/Government/nllb_government_eval_government_ta-si_baseline.csv b/Government/nllb_government_eval_government_ta-si_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..6f3c4e75c98af6c75b37e41fca7c521ee1d668a3 --- /dev/null +++ b/Government/nllb_government_eval_government_ta-si_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,2.1264790250606924,2.8059466038175693,1.6139690857599398,1.1365380107647693,0.9636880451596379,0.5284281494617462 diff --git a/Government/nllb_health_eval_government_en-si_baseline.csv b/Government/nllb_health_eval_government_en-si_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..3e8905ca03305260421117fc66d32a9773db8989 --- /dev/null +++ b/Government/nllb_health_eval_government_en-si_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,2.1293772294034072,2.6482080421096397,1.1779582804713167,1.235578054683879,1.073537231998086,0.4134376797113526 diff --git a/Government/nllb_health_eval_government_en-ta_baseline.csv b/Government/nllb_health_eval_government_en-ta_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..a592a1db54c7244e79ded47e437887bb73f57ffc --- /dev/null +++ b/Government/nllb_health_eval_government_en-ta_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,1.4038432938609309,2.4884344633513176,1.4751634938100298,0.8742299437387782,1.0600629870636833,0.4197459498494864 diff --git a/Government/nllb_health_eval_government_si-en_baseline.csv b/Government/nllb_health_eval_government_si-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..6cdbc0d20492992503d535827fe48ce063c13e79 --- /dev/null +++ b/Government/nllb_health_eval_government_si-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,44.92138874126114,42.792813526751225,20.948990193440864,24.18940878364083,23.40765789077685,0.7220131568694382 diff --git a/Government/nllb_health_eval_government_si-ta_baseline.csv b/Government/nllb_health_eval_government_si-ta_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..c4c28a9ffd6dc5991d5b0bcac339ebfc1adb976a --- /dev/null +++ b/Government/nllb_health_eval_government_si-ta_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,1.013064442321414,2.1382657200712902,0.9734489037312932,0.6359235764087909,0.7222283891295722,0.5001927641961648 diff --git a/Government/nllb_health_eval_government_ta-en_baseline.csv b/Government/nllb_health_eval_government_ta-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..fca637a481c3b3203fd5ef210600960233be1611 --- /dev/null +++ b/Government/nllb_health_eval_government_ta-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,60.18050958026944,57.52052470528929,30.45956583501814,33.85661078371377,34.56109868596526,0.8350872081518174 diff --git a/Government/nllb_health_eval_government_ta-si_baseline.csv b/Government/nllb_health_eval_government_ta-si_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..898a6cc3bc8ac377f9ff8ef6275bc091f8dc1ed0 --- /dev/null +++ b/Government/nllb_health_eval_government_ta-si_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,2.623521733310948,3.1080422505838414,1.0873166695415593,1.1456848084903108,0.9758022354772898,0.4921825256730829 diff --git a/Government/nllb_jailbreak_eval_government_en-si_baseline.csv b/Government/nllb_jailbreak_eval_government_en-si_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..991d3f36dcff2f76ea7b73201dd5d128ab6c29e1 --- /dev/null +++ b/Government/nllb_jailbreak_eval_government_en-si_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,2.7299557155007927,3.4342239238448125,3.070198561059441,1.377379113964754,1.4588223666474471,0.519491742938757 diff --git a/Government/nllb_jailbreak_eval_government_en-ta_baseline.csv b/Government/nllb_jailbreak_eval_government_en-ta_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..8c2670bf95e0057bf9bf5e093547b40926a9a1fe --- /dev/null +++ b/Government/nllb_jailbreak_eval_government_en-ta_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.950902745891338,2.5102398659453193,2.8700914468257674,0.8270928935658232,1.170507674721423,0.5279000070542097 diff --git a/Government/nllb_jailbreak_eval_government_si-en_baseline.csv b/Government/nllb_jailbreak_eval_government_si-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..5fe57b6cbff1421e866301fc2e432a6805a0f43d --- /dev/null +++ b/Government/nllb_jailbreak_eval_government_si-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,13.70091286495427,13.301510063094474,7.9429792134790675,7.060747562369329,7.125536393775505,0.5843882857114077 diff --git a/Government/nllb_jailbreak_eval_government_si-ta_baseline.csv b/Government/nllb_jailbreak_eval_government_si-ta_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..82810afa169816583e08cf6b2a963d6256f226a4 --- /dev/null +++ b/Government/nllb_jailbreak_eval_government_si-ta_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.7098839047708805,1.6801276275033277,2.2360737172900746,0.5717536472337825,0.808304947689558,0.5240307556092739 diff --git a/Government/nllb_jailbreak_eval_government_ta-en_baseline.csv b/Government/nllb_jailbreak_eval_government_ta-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..ff19f8cde1527a8d28ab47413a8c42b8ebd59ac6 --- /dev/null +++ b/Government/nllb_jailbreak_eval_government_ta-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,42.33599917092071,40.69509676471764,23.927384098520275,22.840365209874985,23.52094887359705,0.7381379562616348 diff --git a/Government/nllb_jailbreak_eval_government_ta-si_baseline.csv b/Government/nllb_jailbreak_eval_government_ta-si_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..73a3da09f80d7fe25ef523a1d20d953a325124eb --- /dev/null +++ b/Government/nllb_jailbreak_eval_government_ta-si_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,2.155647341058514,3.0479886610603177,2.7193614809686384,0.836017800841948,0.8711490692234946,0.5428231197297573 diff --git a/Government/nllb_literature-children_eval_government_en-si_baseline.csv b/Government/nllb_literature-children_eval_government_en-si_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..7c3f21a2dac26651601f5b0286a5a00e4becc143 --- /dev/null +++ b/Government/nllb_literature-children_eval_government_en-si_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.5405674952727352,1.72281406584829,0.21065968916749836,0.021384744828425506,0.020206560241073472,0.5059475341588259 diff --git a/Government/nllb_literature-children_eval_government_en-ta_baseline.csv b/Government/nllb_literature-children_eval_government_en-ta_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..f4798a8a0fa479f1e8883f42ffb4e3af611d9474 --- /dev/null +++ b/Government/nllb_literature-children_eval_government_en-ta_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.4308148131801461,2.023331196074034,0.15121750487260444,0.02152801753686127,0.023163088551058326,0.4852443312432755 diff --git a/Government/nllb_literature-children_eval_government_si-en_baseline.csv b/Government/nllb_literature-children_eval_government_si-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..81fa91a39c27073184d64fd7fb35ff776a1d83b0 --- /dev/null +++ b/Government/nllb_literature-children_eval_government_si-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,12.68270519652462,12.506382621891632,4.647287447640901,4.748886789179166,4.84982681362838,0.5844122405648231 diff --git a/Government/nllb_literature-children_eval_government_si-ta_baseline.csv b/Government/nllb_literature-children_eval_government_si-ta_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..3e918518b533b9b7a9d313b4c47e5253d1569022 --- /dev/null +++ b/Government/nllb_literature-children_eval_government_si-ta_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.2009138710399402,0.9862013885203197,0.043926911586796125,0.014950586247079848,0.016094740317919472,0.5033425367050652 diff --git a/Government/nllb_literature-children_eval_government_ta-en_baseline.csv b/Government/nllb_literature-children_eval_government_ta-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..d05ffc7aa190aa4db1e32c3763d1ed7d9df231c1 --- /dev/null +++ b/Government/nllb_literature-children_eval_government_ta-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,33.532064508931796,32.15883836468728,14.273128624048406,14.123921514861046,15.284138938637296,0.6710098042256303 diff --git a/Government/nllb_literature-children_eval_government_ta-si_baseline.csv b/Government/nllb_literature-children_eval_government_ta-si_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..c2589a1161431c3c42f3ac941c14907fdb4be8e8 --- /dev/null +++ b/Government/nllb_literature-children_eval_government_ta-si_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.41090792819802463,1.4658801399815615,0.10700410237266599,0.022145700091969856,0.020186861767000696,0.5111609247796659 diff --git a/Government/nllb_literature-kathlyn_eval_government_en-si_baseline.csv b/Government/nllb_literature-kathlyn_eval_government_en-si_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..7c3f21a2dac26651601f5b0286a5a00e4becc143 --- /dev/null +++ b/Government/nllb_literature-kathlyn_eval_government_en-si_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.5405674952727352,1.72281406584829,0.21065968916749836,0.021384744828425506,0.020206560241073472,0.5059475341588259 diff --git a/Government/nllb_literature-kathlyn_eval_government_en-ta_baseline.csv b/Government/nllb_literature-kathlyn_eval_government_en-ta_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..f4798a8a0fa479f1e8883f42ffb4e3af611d9474 --- /dev/null +++ b/Government/nllb_literature-kathlyn_eval_government_en-ta_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.4308148131801461,2.023331196074034,0.15121750487260444,0.02152801753686127,0.023163088551058326,0.4852443312432755 diff --git a/Government/nllb_literature-kathlyn_eval_government_si-en_baseline.csv b/Government/nllb_literature-kathlyn_eval_government_si-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..81fa91a39c27073184d64fd7fb35ff776a1d83b0 --- /dev/null +++ b/Government/nllb_literature-kathlyn_eval_government_si-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,12.68270519652462,12.506382621891632,4.647287447640901,4.748886789179166,4.84982681362838,0.5844122405648231 diff --git a/Government/nllb_literature-kathlyn_eval_government_si-ta_baseline.csv b/Government/nllb_literature-kathlyn_eval_government_si-ta_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..3e918518b533b9b7a9d313b4c47e5253d1569022 --- /dev/null +++ b/Government/nllb_literature-kathlyn_eval_government_si-ta_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.2009138710399402,0.9862013885203197,0.043926911586796125,0.014950586247079848,0.016094740317919472,0.5033425367050652 diff --git a/Government/nllb_literature-kathlyn_eval_government_ta-en_baseline.csv b/Government/nllb_literature-kathlyn_eval_government_ta-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..d05ffc7aa190aa4db1e32c3763d1ed7d9df231c1 --- /dev/null +++ b/Government/nllb_literature-kathlyn_eval_government_ta-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,33.532064508931796,32.15883836468728,14.273128624048406,14.123921514861046,15.284138938637296,0.6710098042256303 diff --git a/Government/nllb_literature-kathlyn_eval_government_ta-si_baseline.csv b/Government/nllb_literature-kathlyn_eval_government_ta-si_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..c2589a1161431c3c42f3ac941c14907fdb4be8e8 --- /dev/null +++ b/Government/nllb_literature-kathlyn_eval_government_ta-si_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.41090792819802463,1.4658801399815615,0.10700410237266599,0.022145700091969856,0.020186861767000696,0.5111609247796659 diff --git a/Government/nllb_literature-old_eval_government_en-si_baseline.csv b/Government/nllb_literature-old_eval_government_en-si_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..7c3f21a2dac26651601f5b0286a5a00e4becc143 --- /dev/null +++ b/Government/nllb_literature-old_eval_government_en-si_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.5405674952727352,1.72281406584829,0.21065968916749836,0.021384744828425506,0.020206560241073472,0.5059475341588259 diff --git a/Government/nllb_literature-old_eval_government_en-ta_baseline.csv b/Government/nllb_literature-old_eval_government_en-ta_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..f4798a8a0fa479f1e8883f42ffb4e3af611d9474 --- /dev/null +++ b/Government/nllb_literature-old_eval_government_en-ta_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.4308148131801461,2.023331196074034,0.15121750487260444,0.02152801753686127,0.023163088551058326,0.4852443312432755 diff --git a/Government/nllb_literature-old_eval_government_si-en_baseline.csv b/Government/nllb_literature-old_eval_government_si-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..81fa91a39c27073184d64fd7fb35ff776a1d83b0 --- /dev/null +++ b/Government/nllb_literature-old_eval_government_si-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,12.68270519652462,12.506382621891632,4.647287447640901,4.748886789179166,4.84982681362838,0.5844122405648231 diff --git a/Government/nllb_literature-old_eval_government_si-ta_baseline.csv b/Government/nllb_literature-old_eval_government_si-ta_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..3e918518b533b9b7a9d313b4c47e5253d1569022 --- /dev/null +++ b/Government/nllb_literature-old_eval_government_si-ta_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.2009138710399402,0.9862013885203197,0.043926911586796125,0.014950586247079848,0.016094740317919472,0.5033425367050652 diff --git a/Government/nllb_literature-old_eval_government_ta-en_baseline.csv b/Government/nllb_literature-old_eval_government_ta-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..d05ffc7aa190aa4db1e32c3763d1ed7d9df231c1 --- /dev/null +++ b/Government/nllb_literature-old_eval_government_ta-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,33.532064508931796,32.15883836468728,14.273128624048406,14.123921514861046,15.284138938637296,0.6710098042256303 diff --git a/Government/nllb_literature-old_eval_government_ta-si_baseline.csv b/Government/nllb_literature-old_eval_government_ta-si_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..c2589a1161431c3c42f3ac941c14907fdb4be8e8 --- /dev/null +++ b/Government/nllb_literature-old_eval_government_ta-si_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.41090792819802463,1.4658801399815615,0.10700410237266599,0.022145700091969856,0.020186861767000696,0.5111609247796659 diff --git a/Government/nllb_literature_eval_government_en-si_baseline.csv b/Government/nllb_literature_eval_government_en-si_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..7c3f21a2dac26651601f5b0286a5a00e4becc143 --- /dev/null +++ b/Government/nllb_literature_eval_government_en-si_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.5405674952727352,1.72281406584829,0.21065968916749836,0.021384744828425506,0.020206560241073472,0.5059475341588259 diff --git a/Government/nllb_literature_eval_government_en-ta_baseline.csv b/Government/nllb_literature_eval_government_en-ta_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..f4798a8a0fa479f1e8883f42ffb4e3af611d9474 --- /dev/null +++ b/Government/nllb_literature_eval_government_en-ta_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.4308148131801461,2.023331196074034,0.15121750487260444,0.02152801753686127,0.023163088551058326,0.4852443312432755 diff --git a/Government/nllb_literature_eval_government_si-en_baseline.csv b/Government/nllb_literature_eval_government_si-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..81fa91a39c27073184d64fd7fb35ff776a1d83b0 --- /dev/null +++ b/Government/nllb_literature_eval_government_si-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,12.68270519652462,12.506382621891632,4.647287447640901,4.748886789179166,4.84982681362838,0.5844122405648231 diff --git a/Government/nllb_literature_eval_government_si-ta_baseline.csv b/Government/nllb_literature_eval_government_si-ta_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..3e918518b533b9b7a9d313b4c47e5253d1569022 --- /dev/null +++ b/Government/nllb_literature_eval_government_si-ta_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.2009138710399402,0.9862013885203197,0.043926911586796125,0.014950586247079848,0.016094740317919472,0.5033425367050652 diff --git a/Government/nllb_literature_eval_government_ta-en_baseline.csv b/Government/nllb_literature_eval_government_ta-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..d05ffc7aa190aa4db1e32c3763d1ed7d9df231c1 --- /dev/null +++ b/Government/nllb_literature_eval_government_ta-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,33.532064508931796,32.15883836468728,14.273128624048406,14.123921514861046,15.284138938637296,0.6710098042256303 diff --git a/Government/nllb_literature_eval_government_ta-si_baseline.csv b/Government/nllb_literature_eval_government_ta-si_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..c2589a1161431c3c42f3ac941c14907fdb4be8e8 --- /dev/null +++ b/Government/nllb_literature_eval_government_ta-si_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.41090792819802463,1.4658801399815615,0.10700410237266599,0.022145700091969856,0.020186861767000696,0.5111609247796659 diff --git a/Government/nllb_maths_eval_government_en-si_baseline.csv b/Government/nllb_maths_eval_government_en-si_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..e02db3a775274041982dcf94481a0ca3e2634009 --- /dev/null +++ b/Government/nllb_maths_eval_government_en-si_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,3.386902647626311,5.396413898661497,7.043350945997881,4.407736007891258,4.186116810472018,0.5152128710597753 diff --git a/Government/nllb_maths_eval_government_en-ta_baseline.csv b/Government/nllb_maths_eval_government_en-ta_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..94cdd1dd0add315c730d8ec161cb2b450e072e73 --- /dev/null +++ b/Government/nllb_maths_eval_government_en-ta_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,2.8081650520647368,5.5836312752642145,6.501583604993177,4.206994549527542,4.512365149250633,0.5298982553035021 diff --git a/Government/nllb_maths_eval_government_si-en_baseline.csv b/Government/nllb_maths_eval_government_si-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..eedeb8efa3d33b52312126a6caee9cd81e701c00 --- /dev/null +++ b/Government/nllb_maths_eval_government_si-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,11.317448157669336,12.51008597255433,10.716065134149002,9.704364195477536,9.059869661837732,0.5851124048382044 diff --git a/Government/nllb_maths_eval_government_ta-en_baseline.csv b/Government/nllb_maths_eval_government_ta-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..5e07141099d678c2a9e9872de338eb7c8fe83504 --- /dev/null +++ b/Government/nllb_maths_eval_government_ta-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,48.55646207985666,47.844840747445275,30.852783835081933,30.937461958666827,31.65396680699585,0.7837131512463092 diff --git a/Government/nllb_news_eval_government_en-si_baseline.csv b/Government/nllb_news_eval_government_en-si_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..c8b81b6b129b12053673fcdde7678deff5e7cc58 --- /dev/null +++ b/Government/nllb_news_eval_government_en-si_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,1.1743217071010075,1.713707163740193,0.7760985154863841,0.4451472768674177,0.3925480474445332,0.5711525277197361 diff --git a/Government/nllb_news_eval_government_en-ta_baseline.csv b/Government/nllb_news_eval_government_en-ta_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..2d785703add94d09053ed6b0df4cb902c0fb5929 --- /dev/null +++ b/Government/nllb_news_eval_government_en-ta_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,1.9981277037733642,3.3012853643443663,1.101227424070257,1.5735222160523783,1.6449114728899694,0.5120277152841829 diff --git a/Government/nllb_news_eval_government_si-en_baseline.csv b/Government/nllb_news_eval_government_si-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..fc8cdb2e532b4719744044a363d1ae8d43d87009 --- /dev/null +++ b/Government/nllb_news_eval_government_si-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,28.476586976813845,27.171542359315197,12.625989739605961,12.65714651911421,12.389014001011187,0.6620795749127865 diff --git a/Government/nllb_news_eval_government_si-ta_baseline.csv b/Government/nllb_news_eval_government_si-ta_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..31bc5a6de51edb1329593f74582fa003bd517e1c --- /dev/null +++ b/Government/nllb_news_eval_government_si-ta_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.5369858883410602,1.5497125670514913,0.23225941829369318,0.053664225560785234,0.07734242085214785,0.5412610062968827 diff --git a/Government/nllb_news_eval_government_ta-en_baseline.csv b/Government/nllb_news_eval_government_ta-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..74489922346a208d158484699ed4a2cf7fe46f09 --- /dev/null +++ b/Government/nllb_news_eval_government_ta-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,55.95338991980899,54.10241056867392,32.40037756016967,33.277017412245605,34.37400931232348,0.8045579167195114 diff --git a/Government/nllb_news_eval_government_ta-si_baseline.csv b/Government/nllb_news_eval_government_ta-si_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..3e6d8be8a536c7317a2b35ba6ca6cee061904710 --- /dev/null +++ b/Government/nllb_news_eval_government_ta-si_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,2.028414304903063,2.5912661213586685,0.9266677410931267,0.4042689190115334,0.3726255123294969,0.5168055228949073 diff --git a/Government/nllb_nllb_eval_government_en-si_baseline.csv b/Government/nllb_nllb_eval_government_en-si_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..abc4e82cac29116e5f5f71fd4e2ad4a8ed7a518c --- /dev/null +++ b/Government/nllb_nllb_eval_government_en-si_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.7585231071916326,2.0418635894125736,0.311198372233723,0.09001862218812294,0.10510945108344491,0.5752390958219766 diff --git a/Government/nllb_nllb_eval_government_en-ta_baseline.csv b/Government/nllb_nllb_eval_government_en-ta_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..ad3e2820fb5467a944b6eda7bda836b05de94dd7 --- /dev/null +++ b/Government/nllb_nllb_eval_government_en-ta_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.8396751780130931,2.4111867421786233,0.3066246178883174,0.28138229060722514,0.30966805632256583,0.5835503396391869 diff --git a/Government/nllb_nllb_eval_government_si-en_baseline.csv b/Government/nllb_nllb_eval_government_si-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..4e48558e729a24624fe2983be210ad1210212ece --- /dev/null +++ b/Government/nllb_nllb_eval_government_si-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,4.342796789294033,4.749983204532853,2.8072571008784926,2.5871081712100445,2.45253724891628,0.6258792190551757 diff --git a/Government/nllb_nllb_eval_government_si-ta_baseline.csv b/Government/nllb_nllb_eval_government_si-ta_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..d23240721d1802752d489fabb4148a6283df1ef8 --- /dev/null +++ b/Government/nllb_nllb_eval_government_si-ta_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.3991315937132054,1.2016165396211462,0.16726149211582575,0.04922560660851545,0.06126258073071669,0.5897180299162865 diff --git a/Government/nllb_nllb_eval_government_ta-en_baseline.csv b/Government/nllb_nllb_eval_government_ta-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..e161faaba2e55c924f2b642f413d46a9a90bc44e --- /dev/null +++ b/Government/nllb_nllb_eval_government_ta-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,42.075680306915444,40.858521039377784,26.18871728655078,23.279777518510983,25.91722171225571,0.7494174346327782 diff --git a/Government/nllb_nllb_eval_government_ta-si_baseline.csv b/Government/nllb_nllb_eval_government_ta-si_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..5faf26ff85d9f89e84f762c495dcdcc74855be8c --- /dev/null +++ b/Government/nllb_nllb_eval_government_ta-si_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.77098643919795,1.998452512735229,0.37582150932380026,0.11578197993158097,0.15055589575154146,0.5912962484657764 diff --git a/Government/nllb_opensubtitles_eval_government_en-si_baseline.csv b/Government/nllb_opensubtitles_eval_government_en-si_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..e4ef173aa02c1c90467ab5012208ca9a8d8dc426 --- /dev/null +++ b/Government/nllb_opensubtitles_eval_government_en-si_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.847093121279806,2.4684363097189745,1.396130818258411,0.1335446333113439,0.1565595240682873,0.6057251301407814 diff --git a/Government/nllb_opensubtitles_eval_government_en-ta_baseline.csv b/Government/nllb_opensubtitles_eval_government_en-ta_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..080bbf3a4f1b9914e909140be59a79742ac91061 --- /dev/null +++ b/Government/nllb_opensubtitles_eval_government_en-ta_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.7106221154527631,2.570921700813326,1.4788269145982211,0.18926472288885485,0.2536608589994819,0.6126294483244419 diff --git a/Government/nllb_opensubtitles_eval_government_si-en_baseline.csv b/Government/nllb_opensubtitles_eval_government_si-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..976529b3be3f2eb15a73978a7772ee63081f2113 --- /dev/null +++ b/Government/nllb_opensubtitles_eval_government_si-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.20936959285247572,0.4610373422708847,0.21724445369535136,0.0348241300282007,0.03544255168777041,0.5909831962287426 diff --git a/Government/nllb_opensubtitles_eval_government_si-ta_baseline.csv b/Government/nllb_opensubtitles_eval_government_si-ta_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..1560d0d7d76ccf8a07c1340ecb8f2fe766562f58 --- /dev/null +++ b/Government/nllb_opensubtitles_eval_government_si-ta_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.14832565774805978,0.4502203040744262,0.2168377791627246,0.015872062362221606,0.019223697981040722,0.5707517316937447 diff --git a/Government/nllb_opensubtitles_eval_government_ta-en_baseline.csv b/Government/nllb_opensubtitles_eval_government_ta-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..ed922b0b01e275922867dce02bb56914d28eb1de --- /dev/null +++ b/Government/nllb_opensubtitles_eval_government_ta-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,23.287211611003936,22.38927432915715,11.122744297292002,9.407024275001389,10.92931256708631,0.6389645711779595 diff --git a/Government/nllb_opensubtitles_eval_government_ta-si_baseline.csv b/Government/nllb_opensubtitles_eval_government_ta-si_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..bfd2be94a9604d781b78703a3da6f8149ab792bd --- /dev/null +++ b/Government/nllb_opensubtitles_eval_government_ta-si_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.7973949256386279,2.406827458170219,1.0616449149333547,0.21780057645134193,0.23169380849271828,0.5833025423586369 diff --git a/Government/nllb_wikipedia_eval_government_en-si_baseline.csv b/Government/nllb_wikipedia_eval_government_en-si_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..939d3dd2347dde3415c9e45a31b40153c839fdd2 --- /dev/null +++ b/Government/nllb_wikipedia_eval_government_en-si_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.7443504702963069,1.597579383453706,0.2952394162547146,0.1644145649375478,0.15698657911106723,0.4998222410529852 diff --git a/Government/nllb_wikipedia_eval_government_en-ta_baseline.csv b/Government/nllb_wikipedia_eval_government_en-ta_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..307a53dade725e81c482650ea9c0c8127300cf11 --- /dev/null +++ b/Government/nllb_wikipedia_eval_government_en-ta_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,3.8648742886297285,4.624060876786521,1.448036721709419,2.567352063547686,2.708872369094082,0.5320055771023035 diff --git a/Government/nllb_wikipedia_eval_government_si-en_baseline.csv b/Government/nllb_wikipedia_eval_government_si-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..35c95bde68e39f3099ee2cebaf03b16f6959d19e --- /dev/null +++ b/Government/nllb_wikipedia_eval_government_si-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,29.78788601544511,28.904571784948065,14.614483505858551,15.328936597680402,15.164980042762892,0.65542942070961 diff --git a/Government/nllb_wikipedia_eval_government_si-ta_baseline.csv b/Government/nllb_wikipedia_eval_government_si-ta_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..6035fc4d8a6e5a1607a8053e87c6553fc46f1170 --- /dev/null +++ b/Government/nllb_wikipedia_eval_government_si-ta_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.6226172276234757,1.569811801568743,0.35990668004682547,0.1444285046708023,0.15074497634869918,0.5247272368520498 diff --git a/Government/nllb_wikipedia_eval_government_ta-en_baseline.csv b/Government/nllb_wikipedia_eval_government_ta-en_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..3ea7d5b2019b39fb82fce6c7e18179b5200b3b3b --- /dev/null +++ b/Government/nllb_wikipedia_eval_government_ta-en_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,59.68646735470296,57.866382438661454,33.204412644274505,34.89017215436297,35.637236082588075,0.8468928661346435 diff --git a/Government/nllb_wikipedia_eval_government_ta-si_baseline.csv b/Government/nllb_wikipedia_eval_government_ta-si_baseline.csv new file mode 100644 index 0000000000000000000000000000000000000000..8ebf3d8c36bc264bcc64ff01b73ded526dadf1e0 --- /dev/null +++ b/Government/nllb_wikipedia_eval_government_ta-si_baseline.csv @@ -0,0 +1,2 @@ +model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet +facebook/nllb-200-distilled-600M,0.8516461597485614,1.8100795334610313,0.3236137104946885,0.1423561347192973,0.13658044525829446,0.5140674581825733