Update eole-config.yaml

Show huggingface datasets rather than local files

eole-config.yaml

@@ -8,8 +8,8 @@ tensorboard: true
 tensorboard_log_dir: tensorboard
 
 ### Vocab
-src_vocab: faen/fa.eole.vocab
-tgt_vocab: faen/en.eole.vocab
+src_vocab: fa.eole.vocab
+tgt_vocab: en.eole.vocab
 src_vocab_size: 32000
 tgt_vocab_size: 32000
 vocab_size_multiple: 8
@@ -18,26 +18,29 @@ n_sample: 0
 
 data:
     corpus_1:
-        path_src: faen/train.cleaned.filtered.fa
-        path_tgt: faen/train.cleaned.filtered.en
+        path_src: hf://quickmt/quickmt-train.fa-en/fa
+        path_tgt: hf://quickmt/quickmt-train.fa-en/en
+        path_sco: hf://quickmt/quickmt-train.fa-en/sco
         weight: 2
     corpus_2:
-        path_src: /home/mark/mt/data/newscrawl.backtrans.fa
-        path_tgt: /home/mark/mt/data/newscrawl.2024.en
+        path_src: hf://quickmt/newscrawl2024-en-backtranslated-fa/fa
+        path_tgt: hf://quickmt/newscrawl2024-en-backtranslated-fa/en
+        path_sco: hf://quickmt/newscrawl2024-en-backtranslated-fa/sco
         weight: 1
     corpus_3:
-        path_src: /home/mark/mt/data/madlad.backtrans.fa
-        path_tgt: /home/mark/mt/data/madlad.en
+        path_src: hf://quickmt/madlad400-en-backtranslated-fa/fa
+        path_tgt: hf://quickmt/madlad400-en-backtranslated-fa/en
+        path_sco: hf://quickmt/madlad400-en-backtranslated-fa/sco
         weight: 2
     valid:
-        path_src: faen/dev.fa
-        path_tgt: faen/dev.en
+        path_src: dev.fa
+        path_tgt: dev.en
 
 transforms: [sentencepiece, filtertoolong]
 transforms_configs:
   sentencepiece:
-    src_subword_model: "faen/fa.spm.model"
-    tgt_subword_model: "faen/en.spm.model"
+    src_subword_model: "fa.spm.model"
+    tgt_subword_model: "en.spm.model"
   filtertoolong:
     src_seq_length: 256
     tgt_seq_length: 256
@@ -55,7 +58,6 @@ training:
     gpu_ranks: [0]
 
     # Batching 120,000 tokens
-    # For RTX 5090, 15000 batch size, accum_count 8
     batch_type: "tokens"
     batch_size: 6000
     valid_batch_size: 2048
@@ -66,7 +68,6 @@ training:
     # Optimizer & Compute
     compute_dtype: "fp16"
     optim: "adamw"
-    #use_amp: True
     learning_rate: 3.0
     warmup_steps: 5000
     decay_method: "noam"