mt5 and from scratch

base_scandvoc_511_scratch.gin

@@ -0,0 +1,13 @@
+include 't5x/examples/t5/t5_1_1/base.gin'
+include 't5x/configs/runs/pretrain.gin'
+#iinclude 't5x/configs/runs/finetune.gin'
+
+# Register necessary SeqIO Tasks/Mixtures.
+import t5.data.mixtures
+import tasks
+
+include 'base.gin'
+
+VOCABULARY = @seqio.SentencePieceVocabulary()
+seqio.SentencePieceVocabulary.sentencepiece_model_file = "gs://nb-t5/t5/vocabs/wikipedia/no-da-en-sv-nn-is_32000_unigram.sp.model"
+seqio.SentencePieceVocabulary.extra_ids = 100

msmall_scratch.gin

@@ -0,0 +1,19 @@
+include 't5x/examples/t5/mt5/small.gin'
+include 't5x/configs/runs/pretrain.gin'
+#iinclude 't5x/configs/runs/finetune.gin'
+
+# Register necessary SeqIO Tasks/Mixtures.
+import t5.data.mixtures
+import tasks
+
+MIXTURE_OR_TASK_NAME = %gin.REQUIRED
+TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 512}
+TRAIN_STEPS = 1_500_000
+DROPOUT_RATE = 0.0 # Changed from the default since T5-1.1 recomments this.
+#INITIAL_CHECKPOINT_PATH = "gs://nb-t5x-us-central2/norwegian_NCC_plus_English_t5x_base/checkpoint_1500000"
+#INITIAL_CHECKPOINT_PATH = "gs://t5-data/pretrained_models/t5x/t5_1_1_base/checkpoint_1000000"
+INITIAL_CHECKPOINT_PATH = "gs://t5-data/pretrained_models/t5x/mt5_base/checkpoint_1000000"
+
+PjitPartitioner.num_partitions = 1
+
+

train_exp15_base_ul2_511_scandvoc_full_scratch.sh

@@ -0,0 +1,12 @@
+PROJECT_DIR=${HOME}"/models/ul2-t5x"
+T5X_DIR="../../t5x"  # directory where the t5x is cloned.
+MODEL_DIR="gs://nb-t5x-us-central2/exp15-t5-base-ul2-511-scandvoc-full-scratch"
+export PYTHONPATH=${PROJECT_DIR}
+MIXTURE_OR_TASK_NAME="scandinavian_ul2_scandvoc"
+
+python3 ${T5X_DIR}/t5x/train.py \
+  --gin_search_paths=${PROJECT_DIR} \
+  --gin_file="base_scandvoc_511_full_scratch.gin" \
+  --gin.MODEL_DIR="'${MODEL_DIR}'" \
+  --gin.MIXTURE_OR_TASK_NAME="'${MIXTURE_OR_TASK_NAME}'" \
+

train_exp16_base_span_511_scandvoc_full_scratch.sh

@@ -0,0 +1,12 @@
+PROJECT_DIR=${HOME}"/models/ul2-t5x"
+T5X_DIR="../../t5x"  # directory where the t5x is cloned.
+MODEL_DIR="gs://nb-t5x-us-central2/exp16-t5-base-span-511-scandvoc-full-scratch"
+export PYTHONPATH=${PROJECT_DIR}
+MIXTURE_OR_TASK_NAME="scandinavian_span_scandvoc"
+
+python3 ${T5X_DIR}/t5x/train.py \
+  --gin_search_paths=${PROJECT_DIR} \
+  --gin_file="base_scandvoc_511_full_scratch.gin" \
+  --gin.MODEL_DIR="'${MODEL_DIR}'" \
+  --gin.MIXTURE_OR_TASK_NAME="'${MIXTURE_OR_TASK_NAME}'" \
+

train_exp17_small_ul2_mt5voc_scratch.sh

@@ -0,0 +1,12 @@
+PROJECT_DIR=${HOME}"/models/ul2-t5x"
+T5X_DIR="../../t5x"  # directory where the t5x is cloned.
+MODEL_DIR="gs://nb-t5x-us-central2/exp17-t5-small-ul2-mt5voc-scratch"
+export PYTHONPATH=${PROJECT_DIR}
+MIXTURE_OR_TASK_NAME="scandinavian_ul2_mt5voc"
+
+python3 ${T5X_DIR}/t5x/train.py \
+  --gin_search_paths=${PROJECT_DIR} \
+  --gin_file="msmall_scratch.gin" \
+  --gin.MODEL_DIR="'${MODEL_DIR}'" \
+  --gin.MIXTURE_OR_TASK_NAME="'${MIXTURE_OR_TASK_NAME}'" \
+

train_exp18_small_span_mt5voc_scratch.sh

@@ -0,0 +1,12 @@
+PROJECT_DIR=${HOME}"/models/ul2-t5x"
+T5X_DIR="../../t5x"  # directory where the t5x is cloned.
+MODEL_DIR="gs://nb-t5x-us-central2/exp18-t5-small-span-mt5voc-scratch"
+export PYTHONPATH=${PROJECT_DIR}
+MIXTURE_OR_TASK_NAME="scandinavian_span_mt5voc"
+
+python3 ${T5X_DIR}/t5x/train.py \
+  --gin_search_paths=${PROJECT_DIR} \
+  --gin_file="msmall_scratch.gin" \
+  --gin.MODEL_DIR="'${MODEL_DIR}'" \
+  --gin.MIXTURE_OR_TASK_NAME="'${MIXTURE_OR_TASK_NAME}'" \
+