test

base.gin

@@ -8,7 +8,7 @@ include 'pretrain_cont.gin'
 import t5.data.mixtures
 import tasks
 
-MIXTURE_OR_TASK_NAME = "ncc_scandinavian_span_corruption_stream"
+MIXTURE_OR_TASK_NAME = %gin.REQUIRED
 TASK_FEATURE_LENGTHS = {"inputs": 512, "targets": 512}
 TRAIN_STEPS = 1_700_000
 DROPOUT_RATE = 0.0 # Changed from the default since T5-1.1 recomments this.

batch_nynorsk_NCC_base.sh

@@ -0,0 +1,11 @@
+PROJECT_DIR=${HOME}"/models/t5-nynorsk-oversetter"
+export PYTHONPATH=${PROJECT_DIR}
+INITIAL_CHECKPOINT_PATH=\"gs://nb-t5x-us-central2/norwegian_NCC_plus_English_t5x_base/checkpoint_1500000\"
+TRAIN_STEPS=1505000
+
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v1\" &&
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v2\" &&
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v3\" &&
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v4\" &&
+python3 ../../t5x/t5x/train.py --gin_search_paths="./" --gin.TRAIN_STEPS=${TRAIN_STEPS} --gin_file="finetune_translate_base.gin" --gin.INITIAL_CHECKPOINT_PATH=${INITIAL_CHECKPOINT_PATH}  --gin.MIXTURE_OR_TASK_NAME=\"translate\" --gin.MODEL_DIR=\"gs://nb-t5x-us-central2/finetuned/nynorsk_NCC_base_v5\"
+

tasks.py

@@ -9,16 +9,24 @@ from t5.evaluation import metrics
 from seqio import FunctionDataSource, utils
 
 TaskRegistry = seqio.TaskRegistry
-vocabulary=seqio.SentencePieceVocabulary('gs://t5-data/vocabs/cc_all.32000.100extra/sentencepiece.model', extra_ids=0)
+scand_vocabulary=seqio.SentencePieceVocabulary('gs://nb-t5/t5/vocabs/wikipedia/no-da-en-sv-nn-is_32000_unigram.sp.model', extra_ids=100)
+eng_vocabulary=seqio.SentencePieceVocabulary('gs://t5-data/vocabs/cc_all.32000.100extra/sentencepiece.model', extra_ids=0)
 
-DEFAULT_OUTPUT_FEATURES = {
+SCAND_OUTPUT_FEATURES = {
     "inputs": seqio.Feature(
-        vocabulary=vocabulary, add_eos=True,
+        vocabulary=scand_vocabulary, add_eos=True,
         required=False),
     "targets": seqio.Feature(
-        vocabulary=vocabulary, add_eos=True)
+        vocabulary=scand_vocabulary, add_eos=True)
 }
 
+ENG_OUTPUT_FEATURES = {
+    "inputs": seqio.Feature(
+        vocabulary=eng_vocabulary, add_eos=True,
+        required=False),
+    "targets": seqio.Feature(
+        vocabulary=eng_vocabulary, add_eos=True)
+}
 
 def gen_dataset(split, shuffle=False, seed=None, column="text", dataset_params=None):
     dataset = load_dataset(**dataset_params)
@@ -52,7 +60,35 @@ dataset_name = 'NbAiLab/scandinavian'
 dataset_params = {"path": dataset_name, "use_auth_token": True, "streaming": True}
 dataset_shapes = None
 TaskRegistry.add(
-    "ncc_scandinavian_span_corruption_stream",
+    "ncc_scandinavian_span_corruption_stream_engvoc",
+    source=seqio.FunctionDataSource(
+        dataset_fn=functools.partial(dataset_fn, dataset_params=dataset_params),
+        splits=("train", "validation"),
+        caching_permitted=False,
+        num_input_examples=dataset_shapes,
+        ),
+    preprocessors=[
+        functools.partial(
+            target_to_key, key_map={
+                "inputs": None,
+                "targets": None,
+            }, target_key="targets"),
+        seqio.preprocessors.tokenize,
+        # seqio.CacheDatasetPlaceholder(),
+        preprocessors.span_corruption,
+        seqio.preprocessors.append_eos_after_trim,
+    ],
+    output_features={"targets": ENG_OUTPUT_FEATURES["targets"]},
+    metric_fns=[]
+)
+
+
+# Final pretraining task used in Raffel et al., 2019 adaptated to NCC
+dataset_name = 'NbAiLab/scandinavian'
+dataset_params = {"path": dataset_name, "use_auth_token": True, "streaming": True}
+dataset_shapes = None
+TaskRegistry.add(
+    "ncc_scandinavian_span_corruption_stream_scandvoc",
     source=seqio.FunctionDataSource(
         dataset_fn=functools.partial(dataset_fn, dataset_params=dataset_params),
         splits=("train", "validation"),
@@ -70,7 +106,7 @@ TaskRegistry.add(
         preprocessors.span_corruption,
         seqio.preprocessors.append_eos_after_trim,
     ],
-    output_features={"targets": DEFAULT_OUTPUT_FEATURES["targets"]},
+    output_features={"targets": SCAND_OUTPUT_FEATURES["targets"]},
     metric_fns=[]
 )
 

train_base.sh → train_exp1_base_engvoc.sh

@@ -1,9 +1,12 @@
 PROJECT_DIR=${HOME}"/models/long-t5x"
 T5X_DIR="../../t5x"  # directory where the t5x is cloned.
-MODEL_DIR="gs://nb-t5x-us-central2/aaa-t5x-ul2-test-delete"
+MODEL_DIR="gs://nb-t5x-us-central2/exp1-t5-base-engvoc"
 export PYTHONPATH=${PROJECT_DIR}
+MIXTURE_OR_TASK_NAME="ncc_scandinavian_span_corruption_stream_engvoc"
 
 python3 ${T5X_DIR}/t5x/train.py \
   --gin_search_paths=${PROJECT_DIR} \
   --gin_file="base.gin" \
   --gin.MODEL_DIR="'${MODEL_DIR}'" \
+  --gin.MIXTURE_OR_TASK_NAME="'${MIXTURE_OR_TASK_NAME}'" \
+

train_exp1_base_scandvoc.sh

@@ -0,0 +1,12 @@
+PROJECT_DIR=${HOME}"/models/long-t5x"
+T5X_DIR="../../t5x"  # directory where the t5x is cloned.
+MODEL_DIR="gs://nb-t5x-us-central2/exp1-t5-base-scandvoc"
+export PYTHONPATH=${PROJECT_DIR}
+MIXTURE_OR_TASK_NAME="ncc_scandinavian_span_corruption_stream_scandvoc"
+
+python3 ${T5X_DIR}/t5x/train.py \
+  --gin_search_paths=${PROJECT_DIR} \
+  --gin_file="base.gin" \
+  --gin.MODEL_DIR="'${MODEL_DIR}'" \
+  --gin.MIXTURE_OR_TASK_NAME="'${MIXTURE_OR_TASK_NAME}'" \
+