Upload 12 files

aglib/meliad/transformer/configs/size/large_1200M.gin

@@ -0,0 +1,7 @@
+
+# Number of parameters = 1208M
+NUM_LAYERS = 12
+EMBED_DIM = 2048
+NUM_HEADS = 32
+HEAD_DIM = 128
+MLP_DIM = 16384

aglib/meliad/transformer/configs/size/large_600M.gin

@@ -0,0 +1,8 @@
+
+# Number of parameters = 605M
+NUM_LAYERS = 12
+EMBED_DIM = 2048
+NUM_HEADS = 16
+HEAD_DIM = 128
+MLP_DIM = 8192
+

aglib/meliad/transformer/configs/size/layer13.gin

@@ -0,0 +1,4 @@
+
+# Use 13 layers, for comparison against recurrent transformers.
+
+NUM_LAYERS = 13

aglib/meliad/transformer/configs/size/layer24.gin

@@ -0,0 +1,4 @@
+
+# Tall configuration, which doubles the number of layers and parameters.
+
+NUM_LAYERS = 24

aglib/meliad/transformer/configs/size/layer26.gin

@@ -0,0 +1,4 @@
+
+# Use 26 layers, for comparison against tall recurrent transformers.
+
+NUM_LAYERS = 26

aglib/meliad/transformer/configs/size/medium_150M.gin

@@ -0,0 +1,8 @@
+
+# Number of parameters = 151M
+NUM_LAYERS = 12
+EMBED_DIM = 1024
+NUM_HEADS = 8
+HEAD_DIM = 128
+MLP_DIM = 4096
+

aglib/meliad/transformer/configs/size/medium_300M.gin

@@ -0,0 +1,8 @@
+
+# Number of parameters = 302M
+NUM_LAYERS = 12
+EMBED_DIM = 1024
+NUM_HEADS = 16
+HEAD_DIM = 128
+MLP_DIM = 8192
+

aglib/meliad/transformer/configs/size/small.gin

@@ -0,0 +1,7 @@
+
+NUM_LAYERS = 6
+EMBED_DIM = 512
+NUM_HEADS = 8
+HEAD_DIM = 128
+MLP_DIM = 2048
+

aglib/meliad/transformer/configs/size/small_37M.gin

@@ -0,0 +1,8 @@
+
+# Number of parameters = 37M.
+NUM_LAYERS = 12
+EMBED_DIM = 512
+NUM_HEADS = 8
+HEAD_DIM = 64
+MLP_DIM = 2048
+

aglib/meliad/transformer/configs/size/small_75M.gin

@@ -0,0 +1,8 @@
+
+# Number of parameters = 75M
+NUM_LAYERS = 12
+EMBED_DIM = 512
+NUM_HEADS = 8
+HEAD_DIM = 128
+MLP_DIM = 4096
+

aglib/meliad/transformer/configs/size/small_test.gin

@@ -0,0 +1,37 @@
+# Small config for testing purposes
+
+NUM_LAYERS = 6
+EMBED_DIM = 512
+NUM_HEADS = 8
+HEAD_DIM = 128
+MLP_DIM = 2048
+DROPOUT_RATE = 0.1
+ATTN_DROPOUT_RATE = 0.1
+
+decoder_stack.TransformerTaskConfig:
+  sequence_length = 512
+  batch_size = 2
+
+transformer_layer.TransformerLayer:
+  window_length = 256
+  use_long_xl_architecture = True
+  max_unrolled_windows = -1
+  recurrent_num_states = 384  # Odd number for debugging purposes.
+  recurrent_gate_type = "bias"
+  recurrent_single_gate = False
+  recurrent_skip_ffn = True
+
+decoder_stack.DecoderStack:
+  dstack_window_length = 0
+  recurrent_layer_indices = ()  # (-1,)
+  feedback_recurrence = False
+
+training_loop.Trainer:
+  num_steps = 10_000
+  status_every_steps = 5
+  log_every_steps = 20
+  test_every_steps = 50
+  num_test_steps = 2
+  generate_every_steps = 100
+  print_input_every_steps = 100
+  checkpoint_every_steps = 200

aglib/meliad/transformer/configs/size/tiny_test.gin

@@ -0,0 +1,27 @@
+# Tiny config for testing purposes.
+
+NUM_LAYERS = 2
+EMBED_DIM = 128
+NUM_HEADS = 4
+HEAD_DIM = 32
+MLP_DIM = 256
+DROPOUT_RATE = 0.1
+ATTN_DROPOUT_RATE = 0.1
+
+decoder_stack.TransformerTaskConfig:
+  sequence_length = 256
+  batch_size = 1
+
+transformer_layer.TransformerLayer:
+  window_length = 128
+  use_long_xl_architecture = True
+
+training_loop.Trainer:
+  num_steps = 1000
+  warmup_steps = 100
+  log_every_steps = 10
+  test_every_steps = 10
+  num_test_steps = 1
+  generate_every_steps = 100
+  print_input_every_steps = 100
+  checkpoint_every_steps = 100