Upload folder using huggingface_hub

.gitattributes

@@ -118,3 +118,7 @@ MultilingualMultiDomain/MultiDomain/preprocess_multilingual/train.si-en.en.idx f
 MultilingualMultiDomain/MultiDomain/preprocess_multilingual/train.si-en.si.idx filter=lfs diff=lfs merge=lfs -text
 MultilingualMultiDomain/MultiDomain/preprocess_multilingual/train.ta-en.en.idx filter=lfs diff=lfs merge=lfs -text
 MultilingualMultiDomain/MultiDomain/preprocess_multilingual/train.ta-en.ta.idx filter=lfs diff=lfs merge=lfs -text
+Wikipedia-PE-Original-20k/20k/en-si/preprocess_en-si/train.en-si.en.idx filter=lfs diff=lfs merge=lfs -text
+Wikipedia-PE-Original-20k/20k/en-si/preprocess_en-si/train.en-si.si.idx filter=lfs diff=lfs merge=lfs -text
+Wikipedia-PE-Original-20k/20k/si-en/preprocess_si-en/train.si-en.en.idx filter=lfs diff=lfs merge=lfs -text
+Wikipedia-PE-Original-20k/20k/si-en/preprocess_si-en/train.si-en.si.idx filter=lfs diff=lfs merge=lfs -text

Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Bible/en-si.csv

@@ -0,0 +1,2 @@
+model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet
+_,21.691831689932283,18.33750417704841,0.2265759128802811,3.1247599352077424,3.5712934734899666,0.5796927464306354

Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Bible/preprocess_en-si/preprocess.log

@@ -0,0 +1,6 @@
+Namespace(aim_repo=None, aim_run_hash=None, align_suffix=None, alignfile=None, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, azureml_logging=False, bf16=False, bpe=None, cpu=False, criterion='cross_entropy', dataset_impl='mmap', destdir='../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Bible/preprocess_en-si/', dict_only=False, empty_cache_freq=0, fp16=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, joined_dictionary=False, log_file=None, log_format=None, log_interval=100, lr_scheduler='fixed', memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_progress_bar=False, nwordssrc=-1, nwordstgt=-1, on_cpu_convert_precision=False, only_source=False, optimizer=None, padding_factor=8, plasma_path='/tmp/plasma', profile=False, quantization_config_path=None, reset_logging=False, scoring='bleu', seed=1, source_lang='en', srcdict='../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/preprocess_en-si/dict.en.txt', suppress_crashes=False, target_lang='si', task='translation', tensorboard_logdir=None, testpref='../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Bible/encoded/test.bpe', tgtdict='../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/preprocess_en-si/dict.si.txt', threshold_loss_scale=None, thresholdsrc=0, thresholdtgt=0, tokenizer=None, tpu=False, trainpref=None, use_plasma_view=False, user_dir=None, validpref=None, wandb_project=None, workers=16)
+[en] Dictionary: 24000 types
+[en] ../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Bible/encoded/test.bpe.en: 1000 sents, 37888 tokens, 0.0633% replaced (by <unk>)
+[si] Dictionary: 24000 types
+[si] ../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Bible/encoded/test.bpe.si: 1000 sents, 39575 tokens, 0.331% replaced (by <unk>)
+Wrote preprocessed data to ../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Bible/preprocess_en-si/

Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Bible/preprocess_en-si/test.en-si.en.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fafc591d20b9e41dd4d94b383c247107fbb1a152ae059a60e2eab6cff17b261c
+size 75776

Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Bible/preprocess_en-si/test.en-si.si.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5f0049e7ac517255cfd76ae39d19d14393f07d08b2886402ea3deaf86f8f6e7
+size 79150

Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/FloresPlus/en-si.csv

@@ -0,0 +1,2 @@
+model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet
+_,33.0278803221443,29.94851973499558,5.844689929037086,13.033971363035237,15.394431618679924,0.647400431069932

Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/FloresPlus/preprocess_en-si/preprocess.log

@@ -0,0 +1,6 @@
+Namespace(aim_repo=None, aim_run_hash=None, align_suffix=None, alignfile=None, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, azureml_logging=False, bf16=False, bpe=None, cpu=False, criterion='cross_entropy', dataset_impl='mmap', destdir='../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/FloresPlus/preprocess_en-si/', dict_only=False, empty_cache_freq=0, fp16=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, joined_dictionary=False, log_file=None, log_format=None, log_interval=100, lr_scheduler='fixed', memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_progress_bar=False, nwordssrc=-1, nwordstgt=-1, on_cpu_convert_precision=False, only_source=False, optimizer=None, padding_factor=8, plasma_path='/tmp/plasma', profile=False, quantization_config_path=None, reset_logging=False, scoring='bleu', seed=1, source_lang='en', srcdict='../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/preprocess_en-si/dict.en.txt', suppress_crashes=False, target_lang='si', task='translation', tensorboard_logdir=None, testpref='../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/FloresPlus/encoded/test.bpe', tgtdict='../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/preprocess_en-si/dict.si.txt', threshold_loss_scale=None, thresholdsrc=0, thresholdtgt=0, tokenizer=None, tpu=False, trainpref=None, use_plasma_view=False, user_dir=None, validpref=None, wandb_project=None, workers=16)
+[en] Dictionary: 24000 types
+[en] ../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/FloresPlus/encoded/test.bpe.en: 1012 sents, 34038 tokens, 0.0999% replaced (by <unk>)
+[si] Dictionary: 24000 types
+[si] ../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/FloresPlus/encoded/test.bpe.si: 1012 sents, 34192 tokens, 0.158% replaced (by <unk>)
+Wrote preprocessed data to ../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/FloresPlus/preprocess_en-si/

Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/FloresPlus/preprocess_en-si/test.en-si.en.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de253987e42f409c7b4631b6683d291fd5bef5ff63385c7f83d5ced1061e56a1
+size 68076

Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/FloresPlus/preprocess_en-si/test.en-si.si.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4328b5b695d35c8095539ca5d987aa680404256ad606402c3053f17dd97e228
+size 68384

Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Government/en-si.csv

@@ -0,0 +1,2 @@
+model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet
+_,29.840968929458878,26.517817021574512,3.6575746319155846,8.848864522911699,13.035643647713256,0.6375849218219518

Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Government/preprocess_en-si/preprocess.log

@@ -0,0 +1,6 @@
+Namespace(aim_repo=None, aim_run_hash=None, align_suffix=None, alignfile=None, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, azureml_logging=False, bf16=False, bpe=None, cpu=False, criterion='cross_entropy', dataset_impl='mmap', destdir='../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Government/preprocess_en-si/', dict_only=False, empty_cache_freq=0, fp16=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, joined_dictionary=False, log_file=None, log_format=None, log_interval=100, lr_scheduler='fixed', memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_progress_bar=False, nwordssrc=-1, nwordstgt=-1, on_cpu_convert_precision=False, only_source=False, optimizer=None, padding_factor=8, plasma_path='/tmp/plasma', profile=False, quantization_config_path=None, reset_logging=False, scoring='bleu', seed=1, source_lang='en', srcdict='../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/preprocess_en-si/dict.en.txt', suppress_crashes=False, target_lang='si', task='translation', tensorboard_logdir=None, testpref='../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Government/encoded/test.bpe', tgtdict='../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/preprocess_en-si/dict.si.txt', threshold_loss_scale=None, thresholdsrc=0, thresholdtgt=0, tokenizer=None, tpu=False, trainpref=None, use_plasma_view=False, user_dir=None, validpref=None, wandb_project=None, workers=16)
+[en] Dictionary: 24000 types
+[en] ../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Government/encoded/test.bpe.en: 1000 sents, 33131 tokens, 0.175% replaced (by <unk>)
+[si] Dictionary: 24000 types
+[si] ../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Government/encoded/test.bpe.si: 1000 sents, 30120 tokens, 0.11% replaced (by <unk>)
+Wrote preprocessed data to ../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Government/preprocess_en-si/

Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Government/preprocess_en-si/test.en-si.en.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c6ee290e26632d52866753182eb02e9d47e370e9cee9b36f2aeb34ce88e3e3d
+size 66262

Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Government/preprocess_en-si/test.en-si.si.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88e85d20f6d466183aa3fcb0509c13524c457a960149c5e2a50c03e49c3b7935
+size 60240

Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Health/en-si.csv

@@ -0,0 +1,2 @@
+model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet
+_,27.18335769859328,24.928568271053457,4.742176496555776,8.391057113408785,10.36724548079996,0.573557390188903

Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Health/preprocess_en-si/preprocess.log

@@ -0,0 +1,6 @@
+Namespace(aim_repo=None, aim_run_hash=None, align_suffix=None, alignfile=None, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, azureml_logging=False, bf16=False, bpe=None, cpu=False, criterion='cross_entropy', dataset_impl='mmap', destdir='../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Health/preprocess_en-si/', dict_only=False, empty_cache_freq=0, fp16=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, joined_dictionary=False, log_file=None, log_format=None, log_interval=100, lr_scheduler='fixed', memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_progress_bar=False, nwordssrc=-1, nwordstgt=-1, on_cpu_convert_precision=False, only_source=False, optimizer=None, padding_factor=8, plasma_path='/tmp/plasma', profile=False, quantization_config_path=None, reset_logging=False, scoring='bleu', seed=1, source_lang='en', srcdict='../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/preprocess_en-si/dict.en.txt', suppress_crashes=False, target_lang='si', task='translation', tensorboard_logdir=None, testpref='../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Health/encoded/test.bpe', tgtdict='../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/preprocess_en-si/dict.si.txt', threshold_loss_scale=None, thresholdsrc=0, thresholdtgt=0, tokenizer=None, tpu=False, trainpref=None, use_plasma_view=False, user_dir=None, validpref=None, wandb_project=None, workers=16)
+[en] Dictionary: 24000 types
+[en] ../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Health/encoded/test.bpe.en: 890 sents, 38444 tokens, 0.101% replaced (by <unk>)
+[si] Dictionary: 24000 types
+[si] ../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Health/encoded/test.bpe.si: 890 sents, 42637 tokens, 0.188% replaced (by <unk>)
+Wrote preprocessed data to ../trained_models/Fairseq/Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Health/preprocess_en-si/

Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Health/preprocess_en-si/test.en-si.en.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd4f15557e7785167cdbb2e8ec1e8b5db6a289930c2e619f0980f53118aa1ab7
+size 76888

Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Health/preprocess_en-si/test.en-si.si.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45bcbece923d7ec6da69a7b2f1f2cc6336fc5d67e8b4361b0dd1cdc49079d731
+size 85274

Wikipedia-PE-Original-20k/20k/en-si/cross_domain_outputs/Jailbreak/en-si.csv

@@ -0,0 +1,2 @@
+model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet
+_,27.070626461073672,24.74653296933718,4.151959481125155,8.372885149231397,10.060726337960114,0.6038470848053694