test

run_mlm_flax_stream_tpunew.py

@@ -180,7 +180,7 @@ class DataTrainingArguments:
         default=10000, metadata={"help": "The number of examples to pre-load for shuffling."}
     )
     num_train_steps: int = field(default=50000, metadata={"help": "The number of training steps."})
-    num_eval_samples: int = field(default=50, metadata={"help": "The number of samples to be used for evaluation"})
+    num_eval_samples: int = field(default=10000, metadata={"help": "The number of samples to be used for evaluation"})
 
     def __post_init__(self):
         if self.dataset_name is None and self.train_file is None and self.validation_file is None:
@@ -269,11 +269,12 @@ class FlaxDataCollatorForLanguageModeling:
 def generate_batch_splits(samples_idx: np.ndarray, batch_size: int) -> np.ndarray:
     num_samples = len(samples_idx)
     samples_to_remove = num_samples % batch_size
-
+    
     if samples_to_remove != 0:
         samples_idx = samples_idx[:-samples_to_remove]
     sections_split = num_samples // batch_size
     batch_idx = np.split(samples_idx, sections_split)
+    
     return batch_idx
 
 def advance_iter_and_group_samples(train_iterator, num_samples, max_seq_length):
@@ -561,13 +562,9 @@ if __name__ == "__main__":
     train_metrics = []
     eval_metrics = []
     
-    if num_of_hosts > 1:
-        training_iter = iter(torch.utils.data.DataLoader(tokenized_datasets.with_format("torch"), batch_size=1, shuffle=False, num_workers=dataset.n_shards, collate_fn=lambda x: x))
-    else: 
-        training_iter = iter(tokenized_datasets) 
+    training_iter = iter(torch.utils.data.DataLoader(tokenized_datasets.with_format("torch"), batch_size=1, shuffle=False, num_workers=dataset.n_shards, collate_fn=lambda x: x))
     
     max_seq_length = min(data_args.max_seq_length, tokenizer.model_max_length)
-
     eval_samples = advance_iter_and_group_samples(training_iter, data_args.num_eval_samples, max_seq_length)
     
 

run_nb_roberta_base_scandi_tpunew8.sh

@@ -0,0 +1,23 @@
+python run_mlm_flax_stream_tpunew.py \
+    --output_dir="../nb-roberta-base-scandi-tpunew8" \
+    --hub_model_id="NbAiLab/nb-roberta-base-scandi-tpunew8" \
+    --hub_private_repo=True \
+    --model_name_or_path="xlm-roberta-base" \
+    --config_name="./config_base.json" \
+    --tokenizer_name="./" \
+    --dataset_name="NbAiLab/scandinavian" \
+    --max_seq_length="512" \
+    --weight_decay="0.01" \
+    --per_device_train_batch_size="62" \
+    --per_device_eval_batch_size="62" \
+    --learning_rate="4e-4" \
+    --warmup_steps="1000" \
+    --overwrite_output_dir \
+    --num_train_steps="10000" \
+    --adam_beta1="0.9" \
+    --adam_beta2="0.98" \
+    --logging_steps="50" \
+    --save_steps="50" \
+    --eval_steps="50" \
+    --dtype="bfloat16" \
+    --push_to_hub