Upload folder using huggingface_hub

.gitattributes

@@ -92,3 +92,19 @@ MultiDomain/1k/si-en/preprocess_si-en/train.si-en.en.idx filter=lfs diff=lfs mer
 MultiDomain/1k/si-en/preprocess_si-en/train.si-en.si.idx filter=lfs diff=lfs merge=lfs -text
 MultiDomain/1k/ta-en/preprocess_ta-en/train.ta-en.en.idx filter=lfs diff=lfs merge=lfs -text
 MultiDomain/1k/ta-en/preprocess_ta-en/train.ta-en.ta.idx filter=lfs diff=lfs merge=lfs -text
+News/10k/en-si/preprocess_en-si/train.en-si.en.idx filter=lfs diff=lfs merge=lfs -text
+News/10k/en-si/preprocess_en-si/train.en-si.si.idx filter=lfs diff=lfs merge=lfs -text
+News/10k/en-ta/preprocess_en-ta/train.en-ta.en.idx filter=lfs diff=lfs merge=lfs -text
+News/10k/en-ta/preprocess_en-ta/train.en-ta.ta.idx filter=lfs diff=lfs merge=lfs -text
+News/10k/si-en/preprocess_si-en/train.si-en.en.idx filter=lfs diff=lfs merge=lfs -text
+News/10k/si-en/preprocess_si-en/train.si-en.si.idx filter=lfs diff=lfs merge=lfs -text
+News/10k/si-ta/preprocess_si-ta/train.si-ta.si.idx filter=lfs diff=lfs merge=lfs -text
+News/10k/si-ta/preprocess_si-ta/train.si-ta.ta.idx filter=lfs diff=lfs merge=lfs -text
+News/10k/ta-en/preprocess_ta-en/train.ta-en.en.idx filter=lfs diff=lfs merge=lfs -text
+News/10k/ta-en/preprocess_ta-en/train.ta-en.ta.idx filter=lfs diff=lfs merge=lfs -text
+News/10k/ta-si/preprocess_ta-si/train.ta-si.si.idx filter=lfs diff=lfs merge=lfs -text
+News/10k/ta-si/preprocess_ta-si/train.ta-si.ta.idx filter=lfs diff=lfs merge=lfs -text
+News/15k/en-si/preprocess_en-si/train.en-si.en.idx filter=lfs diff=lfs merge=lfs -text
+News/15k/en-si/preprocess_en-si/train.en-si.si.idx filter=lfs diff=lfs merge=lfs -text
+News/15k/si-en/preprocess_si-en/train.si-en.en.idx filter=lfs diff=lfs merge=lfs -text
+News/15k/si-en/preprocess_si-en/train.si-en.si.idx filter=lfs diff=lfs merge=lfs -text

News/10k/en-si/cross_domain_outputs/Bible/en-si.csv

@@ -0,0 +1,2 @@
+model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet
+_,12.700361405767083,10.786058423682865,0.044820874657850326,0.13029832771516386,0.1885894252177564,0.4069176128655672

News/10k/en-si/cross_domain_outputs/Bible/preprocess_en-si/preprocess.log

@@ -0,0 +1,6 @@
+Namespace(aim_repo=None, aim_run_hash=None, align_suffix=None, alignfile=None, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, azureml_logging=False, bf16=False, bpe=None, cpu=False, criterion='cross_entropy', dataset_impl='mmap', destdir='../trained_models/Fairseq/News/10k/en-si/cross_domain_outputs/Bible/preprocess_en-si/', dict_only=False, empty_cache_freq=0, fp16=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, joined_dictionary=False, log_file=None, log_format=None, log_interval=100, lr_scheduler='fixed', memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_progress_bar=False, nwordssrc=-1, nwordstgt=-1, on_cpu_convert_precision=False, only_source=False, optimizer=None, padding_factor=8, plasma_path='/tmp/plasma', profile=False, quantization_config_path=None, reset_logging=False, scoring='bleu', seed=1, source_lang='en', srcdict='../trained_models/Fairseq/News/10k/en-si/preprocess_en-si/dict.en.txt', suppress_crashes=False, target_lang='si', task='translation', tensorboard_logdir=None, testpref='../trained_models/Fairseq/News/10k/en-si/cross_domain_outputs/Bible/encoded/test.bpe', tgtdict='../trained_models/Fairseq/News/10k/en-si/preprocess_en-si/dict.si.txt', threshold_loss_scale=None, thresholdsrc=0, thresholdtgt=0, tokenizer=None, tpu=False, trainpref=None, use_plasma_view=False, user_dir=None, validpref=None, wandb_project=None, workers=16)
+[en] Dictionary: 23192 types
+[en] ../trained_models/Fairseq/News/10k/en-si/cross_domain_outputs/Bible/encoded/test.bpe.en: 1000 sents, 38875 tokens, 0.514% replaced (by <unk>)
+[si] Dictionary: 23192 types
+[si] ../trained_models/Fairseq/News/10k/en-si/cross_domain_outputs/Bible/encoded/test.bpe.si: 1000 sents, 38886 tokens, 0.895% replaced (by <unk>)
+Wrote preprocessed data to ../trained_models/Fairseq/News/10k/en-si/cross_domain_outputs/Bible/preprocess_en-si/

News/10k/en-si/cross_domain_outputs/Bible/preprocess_en-si/test.en-si.en.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00d53cdc697a1cc0bf832a0d1ed407e0729d682f350f8841adf72c9667a07ef7
+size 77750

News/10k/en-si/cross_domain_outputs/Bible/preprocess_en-si/test.en-si.si.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d3c0d8ace698af8734bad43e46c6a1042b08c68cc96b280b378114c51e0ccc8
+size 77772

News/10k/en-si/cross_domain_outputs/FloresPlus/en-si.csv

@@ -0,0 +1,2 @@
+model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet
+_,18.842213560911976,16.683506913244965,0.6524360770489132,2.2865591778004988,3.331469592575144,0.45170238600830315

News/10k/en-si/cross_domain_outputs/FloresPlus/preprocess_en-si/preprocess.log

@@ -0,0 +1,6 @@
+Namespace(aim_repo=None, aim_run_hash=None, align_suffix=None, alignfile=None, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, azureml_logging=False, bf16=False, bpe=None, cpu=False, criterion='cross_entropy', dataset_impl='mmap', destdir='../trained_models/Fairseq/News/10k/en-si/cross_domain_outputs/FloresPlus/preprocess_en-si/', dict_only=False, empty_cache_freq=0, fp16=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, joined_dictionary=False, log_file=None, log_format=None, log_interval=100, lr_scheduler='fixed', memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_progress_bar=False, nwordssrc=-1, nwordstgt=-1, on_cpu_convert_precision=False, only_source=False, optimizer=None, padding_factor=8, plasma_path='/tmp/plasma', profile=False, quantization_config_path=None, reset_logging=False, scoring='bleu', seed=1, source_lang='en', srcdict='../trained_models/Fairseq/News/10k/en-si/preprocess_en-si/dict.en.txt', suppress_crashes=False, target_lang='si', task='translation', tensorboard_logdir=None, testpref='../trained_models/Fairseq/News/10k/en-si/cross_domain_outputs/FloresPlus/encoded/test.bpe', tgtdict='../trained_models/Fairseq/News/10k/en-si/preprocess_en-si/dict.si.txt', threshold_loss_scale=None, thresholdsrc=0, thresholdtgt=0, tokenizer=None, tpu=False, trainpref=None, use_plasma_view=False, user_dir=None, validpref=None, wandb_project=None, workers=16)
+[en] Dictionary: 23192 types
+[en] ../trained_models/Fairseq/News/10k/en-si/cross_domain_outputs/FloresPlus/encoded/test.bpe.en: 1012 sents, 33439 tokens, 0.721% replaced (by <unk>)
+[si] Dictionary: 23192 types
+[si] ../trained_models/Fairseq/News/10k/en-si/cross_domain_outputs/FloresPlus/encoded/test.bpe.si: 1012 sents, 32770 tokens, 0.384% replaced (by <unk>)
+Wrote preprocessed data to ../trained_models/Fairseq/News/10k/en-si/cross_domain_outputs/FloresPlus/preprocess_en-si/

News/10k/en-si/cross_domain_outputs/FloresPlus/preprocess_en-si/test.en-si.en.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef117664a286d4c744bf1da6cfa25dc8c75c4c0bd5ab0760b834e3ec29412c57
+size 66878

News/10k/en-si/cross_domain_outputs/FloresPlus/preprocess_en-si/test.en-si.si.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f8990051992af8b71b6e56084e77a5c4c7706be98ce94de8986f22c32f26fa1
+size 65540

News/10k/en-si/cross_domain_outputs/Government/en-si.csv

@@ -0,0 +1,2 @@
+model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet
+_,20.961822675101338,18.369415228442705,0.7314532367696964,2.5331620992816553,4.757277700737659,0.5172673012912273

News/10k/en-si/cross_domain_outputs/Government/preprocess_en-si/preprocess.log

@@ -0,0 +1,6 @@
+Namespace(aim_repo=None, aim_run_hash=None, align_suffix=None, alignfile=None, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, azureml_logging=False, bf16=False, bpe=None, cpu=False, criterion='cross_entropy', dataset_impl='mmap', destdir='../trained_models/Fairseq/News/10k/en-si/cross_domain_outputs/Government/preprocess_en-si/', dict_only=False, empty_cache_freq=0, fp16=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, joined_dictionary=False, log_file=None, log_format=None, log_interval=100, lr_scheduler='fixed', memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_progress_bar=False, nwordssrc=-1, nwordstgt=-1, on_cpu_convert_precision=False, only_source=False, optimizer=None, padding_factor=8, plasma_path='/tmp/plasma', profile=False, quantization_config_path=None, reset_logging=False, scoring='bleu', seed=1, source_lang='en', srcdict='../trained_models/Fairseq/News/10k/en-si/preprocess_en-si/dict.en.txt', suppress_crashes=False, target_lang='si', task='translation', tensorboard_logdir=None, testpref='../trained_models/Fairseq/News/10k/en-si/cross_domain_outputs/Government/encoded/test.bpe', tgtdict='../trained_models/Fairseq/News/10k/en-si/preprocess_en-si/dict.si.txt', threshold_loss_scale=None, thresholdsrc=0, thresholdtgt=0, tokenizer=None, tpu=False, trainpref=None, use_plasma_view=False, user_dir=None, validpref=None, wandb_project=None, workers=16)
+[en] Dictionary: 23192 types
+[en] ../trained_models/Fairseq/News/10k/en-si/cross_domain_outputs/Government/encoded/test.bpe.en: 1000 sents, 29766 tokens, 0.87% replaced (by <unk>)
+[si] Dictionary: 23192 types
+[si] ../trained_models/Fairseq/News/10k/en-si/cross_domain_outputs/Government/encoded/test.bpe.si: 1000 sents, 26906 tokens, 0.431% replaced (by <unk>)
+Wrote preprocessed data to ../trained_models/Fairseq/News/10k/en-si/cross_domain_outputs/Government/preprocess_en-si/

News/10k/en-si/cross_domain_outputs/Government/preprocess_en-si/test.en-si.en.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa80cfe29cb17f5482b070810b188e6fe7cf270b7611d25b9ec78c30e6d571ad
+size 59532

News/10k/en-si/cross_domain_outputs/Government/preprocess_en-si/test.en-si.si.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fc08faac50281d0926f5d0bfb0aa48c93ff46de14f78b22ae71e2db7032a7be
+size 53812

News/10k/en-si/cross_domain_outputs/Health/en-si.csv

@@ -0,0 +1,2 @@
+model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet
+_,17.494907482156616,15.440556938321294,0.6197304472706424,1.860316931132211,2.5341179662407027,0.4516362350308493

News/10k/en-si/cross_domain_outputs/Health/preprocess_en-si/preprocess.log

@@ -0,0 +1,6 @@
+Namespace(aim_repo=None, aim_run_hash=None, align_suffix=None, alignfile=None, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, azureml_logging=False, bf16=False, bpe=None, cpu=False, criterion='cross_entropy', dataset_impl='mmap', destdir='../trained_models/Fairseq/News/10k/en-si/cross_domain_outputs/Health/preprocess_en-si/', dict_only=False, empty_cache_freq=0, fp16=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, joined_dictionary=False, log_file=None, log_format=None, log_interval=100, lr_scheduler='fixed', memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_progress_bar=False, nwordssrc=-1, nwordstgt=-1, on_cpu_convert_precision=False, only_source=False, optimizer=None, padding_factor=8, plasma_path='/tmp/plasma', profile=False, quantization_config_path=None, reset_logging=False, scoring='bleu', seed=1, source_lang='en', srcdict='../trained_models/Fairseq/News/10k/en-si/preprocess_en-si/dict.en.txt', suppress_crashes=False, target_lang='si', task='translation', tensorboard_logdir=None, testpref='../trained_models/Fairseq/News/10k/en-si/cross_domain_outputs/Health/encoded/test.bpe', tgtdict='../trained_models/Fairseq/News/10k/en-si/preprocess_en-si/dict.si.txt', threshold_loss_scale=None, thresholdsrc=0, thresholdtgt=0, tokenizer=None, tpu=False, trainpref=None, use_plasma_view=False, user_dir=None, validpref=None, wandb_project=None, workers=16)
+[en] Dictionary: 23192 types
+[en] ../trained_models/Fairseq/News/10k/en-si/cross_domain_outputs/Health/encoded/test.bpe.en: 890 sents, 36079 tokens, 1.63% replaced (by <unk>)
+[si] Dictionary: 23192 types
+[si] ../trained_models/Fairseq/News/10k/en-si/cross_domain_outputs/Health/encoded/test.bpe.si: 890 sents, 38843 tokens, 0.824% replaced (by <unk>)
+Wrote preprocessed data to ../trained_models/Fairseq/News/10k/en-si/cross_domain_outputs/Health/preprocess_en-si/

News/10k/en-si/cross_domain_outputs/Health/preprocess_en-si/test.en-si.en.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7227109f7e3e553c20214f4dc05720c5c2a544235bbfcad1f35f879210056bd1
+size 72158

News/10k/en-si/cross_domain_outputs/Health/preprocess_en-si/test.en-si.si.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed3641f2e0b3d5dffbe585703cf0e02879ad91d4b802df5e402c51b2795ef9d3
+size 77686

News/10k/en-si/cross_domain_outputs/Jailbreak/en-si.csv

@@ -0,0 +1,2 @@
+model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet
+_,15.14353816217845,13.74271441269579,0.5659447806020211,1.368785115405643,1.9510623706477506,0.47273843444883823