Add ablations

ablations/combined_remaqe/tokenized_combined_remaqe.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:188557e5ab2e719ae03024d3c7cfbe7d09a7101cd7f1aaed34303d9a0c69a033
+size 34307365

ablations/combined_remaqe/trained_arm32_L4_combined_remaqe/beam1.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b70cc65a2c60296f8c4f6cf88c20b7f5e7f7303aa7c68656291b266efc69d1e9
+size 1223170

ablations/combined_remaqe/trained_arm32_L4_combined_remaqe/beam5.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5aac3a1b6bc20dc0935c218369d456a3d19c42cb27e6b0297baf50234d1071b
+size 1218948

ablations/combined_remaqe/trained_arm32_L4_combined_remaqe/checkpoint_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a2905110e2eee764ea97e5baa00a39cbae684a9a6a2ebb17cd8b221bd515d6a
+size 96298768

ablations/combined_remaqe/trained_arm32_base_combined_remaqe/beam1.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a158061ff4accf7dddff828fcfec40d1d607a41cf85b2a04f84d4265a8155b01
+size 1221321

ablations/combined_remaqe/trained_arm32_base_combined_remaqe/beam5.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ded6f1e831902eb5a9af4971993a2c75fc934295e5f843ad1431d612f0668255
+size 1213901

ablations/combined_remaqe/trained_arm32_base_combined_remaqe/checkpoint_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8d01f00bd516f3d4a40b79725dd6a984aed68ba859083122a9151dee4916e6d
+size 143875864

ablations/noconst/tokenized_noconst.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74933be13ba3b2aa34911b97f4451c7b6c99bf35d4ba3daa0263ee9426c1aea0
+size 138509503

ablations/noconst/trained_aarch64_best_noconst/beam1.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea4f4c1ff72a8e98b31b751d135eb61fd51776bd67a51fb8597236504de30060
+size 3303135

ablations/noconst/trained_aarch64_best_noconst/checkpoint_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d5b661e4f82f5dcd60fabe7e7b3c765181028afa0469b5c909143477d52cdc4
+size 186110260

ablations/noconst/trained_aarch64_best_noconst/training.log

@@ -0,0 +1,230 @@
+{'_name': None, 'common': {'_name': None, 'no_progress_bar': False, 'log_interval': 100, 'log_format': None, 'log_file': 'aarch64/trained_aarch64_best_noconst/training.log', 'aim_repo': None, 'aim_run_hash': None, 'tensorboard_logdir': None, 'wandb_project': None, 'azureml_logging': False, 'seed': 1, 'cpu': False, 'tpu': False, 'bf16': False, 'memory_efficient_bf16': False, 'fp16': False, 'memory_efficient_fp16': False, 'fp16_no_flatten_grads': False, 'fp16_init_scale': 128, 'fp16_scale_window': None, 'fp16_scale_tolerance': 0.0, 'on_cpu_convert_precision': False, 'min_loss_scale': 0.0001, 'threshold_loss_scale': None, 'amp': False, 'amp_batch_retries': 2, 'amp_init_scale': 128, 'amp_scale_window': None, 'user_dir': None, 'empty_cache_freq': 0, 'all_gather_list_size': 16384, 'model_parallel_size': 1, 'quantization_config_path': None, 'profile': False, 'reset_logging': False, 'suppress_crashes': False, 'use_plasma_view': False, 'plasma_path': '/tmp/plasma'}, 'common_eval': {'_name': None, 'path': None, 'post_process': None, 'quiet': False, 'model_overrides': '{}', 'results_path': None}, 'distributed_training': {'_name': None, 'distributed_world_size': 1, 'distributed_num_procs': 1, 'distributed_rank': 0, 'distributed_backend': 'nccl', 'distributed_init_method': None, 'distributed_port': -1, 'device_id': 0, 'distributed_no_spawn': False, 'ddp_backend': 'pytorch_ddp', 'ddp_comm_hook': 'none', 'bucket_cap_mb': 25, 'fix_batches_to_gpus': False, 'find_unused_parameters': False, 'gradient_as_bucket_view': False, 'fast_stat_sync': False, 'heartbeat_timeout': -1, 'broadcast_buffers': False, 'slowmo_momentum': None, 'slowmo_base_algorithm': 'localsgd', 'localsgd_frequency': 3, 'nprocs_per_node': 1, 'pipeline_model_parallel': False, 'pipeline_balance': None, 'pipeline_devices': None, 'pipeline_chunks': 0, 'pipeline_encoder_balance': None, 'pipeline_encoder_devices': None, 'pipeline_decoder_balance': None, 'pipeline_decoder_devices': None, 'pipeline_checkpoint': 'never', 'zero_sharding': 'none', 'fp16': False, 'memory_efficient_fp16': False, 'tpu': False, 'no_reshard_after_forward': False, 'fp32_reduce_scatter': False, 'cpu_offload': False, 'use_sharded_state': False, 'not_fsdp_flatten_parameters': False}, 'dataset': {'_name': None, 'num_workers': 1, 'skip_invalid_size_inputs_valid_test': False, 'max_tokens': 20000, 'batch_size': None, 'required_batch_size_multiple': 8, 'required_seq_len_multiple': 1, 'dataset_impl': None, 'data_buffer_size': 10, 'train_subset': 'train', 'valid_subset': 'valid', 'combine_valid_subsets': None, 'ignore_unused_valid_subsets': False, 'validate_interval': 1, 'validate_interval_updates': 0, 'validate_after_updates': 0, 'fixed_validation_seed': None, 'disable_validation': False, 'max_tokens_valid': 20000, 'batch_size_valid': None, 'max_valid_steps': None, 'curriculum': 0, 'gen_subset': 'test', 'num_shards': 1, 'shard_id': 0, 'grouped_shuffling': False, 'update_epoch_batch_itr': False, 'update_ordered_indices_seed': False}, 'optimization': {'_name': None, 'max_epoch': 0, 'max_update': 100000, 'stop_time_hours': 0.0, 'clip_norm': 0.0, 'sentence_avg': False, 'update_freq': [1], 'lr': [0.0005], 'stop_min_lr': -1.0, 'use_bmuf': False, 'skip_remainder_batch': False}, 'checkpoint': {'_name': None, 'save_dir': 'aarch64/trained_aarch64_best_noconst', 'restore_file': 'checkpoint_last.pt', 'continue_once': None, 'finetune_from_model': None, 'reset_dataloader': False, 'reset_lr_scheduler': False, 'reset_meters': False, 'reset_optimizer': False, 'optimizer_overrides': '{}', 'save_interval': 1, 'save_interval_updates': 0, 'keep_interval_updates': -1, 'keep_interval_updates_pattern': -1, 'keep_last_epochs': -1, 'keep_best_checkpoints': 3, 'no_save': False, 'no_epoch_checkpoints': True, 'no_last_checkpoints': False, 'no_save_optimizer_state': False, 'best_checkpoint_metric': 'loss', 'maximize_best_checkpoint_metric': False, 'patience': 4, 'checkpoint_suffix': '', 'checkpoint_shard_count': 1, 'load_checkpoint_on_all_dp_ranks': False, 'write_checkpoints_asynchronously': False, 'model_parallel_size': 1}, 'bmuf': {'_name': None, 'block_lr': 1.0, 'block_momentum': 0.875, 'global_sync_iter': 50, 'warmup_iterations': 500, 'use_nbm': False, 'average_sync': False, 'distributed_world_size': 1}, 'generation': {'_name': None, 'beam': 5, 'nbest': 1, 'max_len_a': 0.0, 'max_len_b': 200, 'min_len': 1, 'match_source_len': False, 'unnormalized': False, 'no_early_stop': False, 'no_beamable_mm': False, 'lenpen': 1.0, 'unkpen': 0.0, 'replace_unk': None, 'sacrebleu': False, 'score_reference': False, 'prefix_size': 0, 'no_repeat_ngram_size': 0, 'sampling': False, 'sampling_topk': -1, 'sampling_topp': -1.0, 'constraints': None, 'temperature': 1.0, 'diverse_beam_groups': -1, 'diverse_beam_strength': 0.5, 'diversity_rate': -1.0, 'print_alignment': None, 'print_step': False, 'lm_path': None, 'lm_weight': 0.0, 'iter_decode_eos_penalty': 0.0, 'iter_decode_max_iter': 10, 'iter_decode_force_max_iter': False, 'iter_decode_with_beam': 1, 'iter_decode_with_external_reranker': False, 'retain_iter_history': False, 'retain_dropout': False, 'retain_dropout_modules': None, 'decoding_format': None, 'no_seed_provided': False, 'eos_token': None}, 'eval_lm': {'_name': None, 'output_word_probs': False, 'output_word_stats': False, 'context_window': 0, 'softmax_batch': 9223372036854775807}, 'interactive': {'_name': None, 'buffer_size': 0, 'input': '-'}, 'model': Namespace(_name='transformer', activation_dropout=0.0, activation_fn='relu', adam_betas=(0.9, 0.999), adam_eps=1e-08, adaptive_input=False, adaptive_softmax_cutoff=None, adaptive_softmax_dropout=0, aim_repo=None, aim_run_hash=None, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, arch='transformer', attention_dropout=0.0, azureml_logging=False, batch_size=None, batch_size_valid=None, best_checkpoint_metric='loss', bf16=False, bpe=None, broadcast_buffers=False, bucket_cap_mb=25, checkpoint_activations=False, checkpoint_shard_count=1, checkpoint_suffix='', clip_norm=0.0, combine_valid_subsets=None, continue_once=None, cpu=False, cpu_offload=False, criterion='cross_entropy', cross_self_attention=False, curriculum=0, data='aarch64/tokenized_dlsm_aarch64_noconst', data_buffer_size=10, dataset_impl=None, ddp_backend='pytorch_ddp', ddp_comm_hook='none', decoder_attention_heads=8, decoder_embed_dim=96, decoder_embed_path=None, decoder_ffn_embed_dim=384, decoder_input_dim=96, decoder_layerdrop=0, decoder_layers=6, decoder_layers_to_keep=None, decoder_learned_pos=False, decoder_normalize_before=False, decoder_output_dim='96', device_id=0, disable_validation=False, distributed_backend='nccl', distributed_init_method=None, distributed_no_spawn=False, distributed_num_procs=1, distributed_port=-1, distributed_rank=0, distributed_world_size=1, dropout=0.1, ema_decay=0.9999, ema_fp32=False, ema_seed_model=None, ema_start_update=0, ema_update_freq=1, empty_cache_freq=0, encoder_attention_heads=8, encoder_embed_dim=384, encoder_embed_path=None, encoder_ffn_embed_dim=1536, encoder_layerdrop=0, encoder_layers=6, encoder_layers_to_keep=None, encoder_learned_pos=False, encoder_normalize_before=False, eos=2, eval_bleu=False, eval_bleu_args='{}', eval_bleu_detok='space', eval_bleu_detok_args='{}', eval_bleu_print_samples=False, eval_bleu_remove_bpe=None, eval_tokenized_bleu=False, fast_stat_sync=False, find_unused_parameters=False, finetune_from_model=None, fix_batches_to_gpus=False, fixed_validation_seed=None, fp16=False, fp16_adam_stats=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, fp32_reduce_scatter=False, gen_subset='test', gradient_as_bucket_view=False, grouped_shuffling=False, heartbeat_timeout=-1, ignore_unused_valid_subsets=False, keep_best_checkpoints=3, keep_interval_updates=-1, keep_interval_updates_pattern=-1, keep_last_epochs=-1, layernorm_embedding=False, left_pad_source=True, left_pad_target=False, load_alignments=False, load_checkpoint_on_all_dp_ranks=False, localsgd_frequency=3, log_file='aarch64/trained_aarch64_best_noconst/training.log', log_format=None, log_interval=100, lr=[0.0005], lr_scheduler='inverse_sqrt', max_epoch=0, max_source_positions=1024, max_target_positions=1024, max_tokens=20000, max_tokens_valid=20000, max_update=100000, max_valid_steps=None, maximize_best_checkpoint_metric=False, memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_cross_attention=False, no_epoch_checkpoints=True, no_last_checkpoints=False, no_progress_bar=False, no_reshard_after_forward=False, no_save=False, no_save_optimizer_state=False, no_scale_embedding=False, no_seed_provided=False, no_token_positional_embeddings=False, not_fsdp_flatten_parameters=False, nprocs_per_node=1, num_batch_buckets=0, num_shards=1, num_workers=1, offload_activations=False, on_cpu_convert_precision=False, optimizer='adam', optimizer_overrides='{}', pad=1, patience=4, pipeline_balance=None, pipeline_checkpoint='never', pipeline_chunks=0, pipeline_decoder_balance=None, pipeline_decoder_devices=None, pipeline_devices=None, pipeline_encoder_balance=None, pipeline_encoder_devices=None, pipeline_model_parallel=False, plasma_path='/tmp/plasma', profile=False, quant_noise_pq=0, quant_noise_pq_block_size=8, quant_noise_scalar=0, quantization_config_path=None, required_batch_size_multiple=8, required_seq_len_multiple=1, reset_dataloader=False, reset_logging=False, reset_lr_scheduler=False, reset_meters=False, reset_optimizer=False, restore_file='checkpoint_last.pt', save_dir='aarch64/trained_aarch64_best_noconst', save_interval=1, save_interval_updates=0, scoring='bleu', seed=1, sentence_avg=False, shard_id=0, share_all_embeddings=False, share_decoder_input_output_embed=False, skip_invalid_size_inputs_valid_test=False, skip_remainder_batch=False, slowmo_base_algorithm='localsgd', slowmo_momentum=None, source_lang=None, stop_min_lr=-1.0, stop_time_hours=0, store_ema=False, suppress_crashes=False, target_lang=None, task='translation', tensorboard_logdir=None, threshold_loss_scale=None, tie_adaptive_weights=False, tokenizer=None, tpu=False, train_subset='train', truncate_source=False, unk=3, update_epoch_batch_itr=False, update_freq=[1], update_ordered_indices_seed=False, upsample_primary=-1, use_bmuf=False, use_old_adam=False, use_plasma_view=False, use_sharded_state=False, user_dir=None, valid_subset='valid', validate_after_updates=0, validate_interval=1, validate_interval_updates=0, wandb_project=None, warmup_init_lr=-1, warmup_updates=4000, weight_decay=0.001, write_checkpoints_asynchronously=False, zero_sharding='none'), 'task': {'_name': 'translation', 'data': 'aarch64/tokenized_dlsm_aarch64_noconst', 'source_lang': None, 'target_lang': None, 'load_alignments': False, 'left_pad_source': True, 'left_pad_target': False, 'max_source_positions': 1024, 'max_target_positions': 1024, 'upsample_primary': -1, 'truncate_source': False, 'num_batch_buckets': 0, 'train_subset': 'train', 'dataset_impl': None, 'required_seq_len_multiple': 1, 'eval_bleu': False, 'eval_bleu_args': '{}', 'eval_bleu_detok': 'space', 'eval_bleu_detok_args': '{}', 'eval_tokenized_bleu': False, 'eval_bleu_remove_bpe': None, 'eval_bleu_print_samples': False}, 'criterion': {'_name': 'cross_entropy', 'sentence_avg': False}, 'optimizer': {'_name': 'adam', 'adam_betas': [0.9, 0.999], 'adam_eps': 1e-08, 'weight_decay': 0.001, 'use_old_adam': False, 'fp16_adam_stats': False, 'tpu': False, 'lr': [0.0005]}, 'lr_scheduler': {'_name': 'inverse_sqrt', 'warmup_updates': 4000, 'warmup_init_lr': -1.0, 'lr': [0.0005]}, 'scoring': {'_name': 'bleu', 'pad': 1, 'eos': 2, 'unk': 3}, 'bpe': None, 'tokenizer': None, 'ema': {'_name': None, 'store_ema': False, 'ema_decay': 0.9999, 'ema_start_update': 0, 'ema_seed_model': None, 'ema_update_freq': 1, 'ema_fp32': False}}
+TransformerModel(
+  (encoder): TransformerEncoderBase(
+    (dropout_module): FairseqDropout()
+    (embed_tokens): Embedding(152, 384, padding_idx=1)
+    (embed_positions): SinusoidalPositionalEmbedding()
+    (layers): ModuleList(
+      (0-5): 6 x TransformerEncoderLayerBase(
+        (self_attn): MultiheadAttention(
+          (dropout_module): FairseqDropout()
+          (k_proj): Linear(in_features=384, out_features=384, bias=True)
+          (v_proj): Linear(in_features=384, out_features=384, bias=True)
+          (q_proj): Linear(in_features=384, out_features=384, bias=True)
+          (out_proj): Linear(in_features=384, out_features=384, bias=True)
+        )
+        (self_attn_layer_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+        (dropout_module): FairseqDropout()
+        (activation_dropout_module): FairseqDropout()
+        (fc1): Linear(in_features=384, out_features=1536, bias=True)
+        (fc2): Linear(in_features=1536, out_features=384, bias=True)
+        (final_layer_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+      )
+    )
+  )
+  (decoder): TransformerDecoderBase(
+    (dropout_module): FairseqDropout()
+    (embed_tokens): Embedding(40, 96, padding_idx=1)
+    (embed_positions): SinusoidalPositionalEmbedding()
+    (layers): ModuleList(
+      (0-5): 6 x TransformerDecoderLayerBase(
+        (dropout_module): FairseqDropout()
+        (self_attn): MultiheadAttention(
+          (dropout_module): FairseqDropout()
+          (k_proj): Linear(in_features=96, out_features=96, bias=True)
+          (v_proj): Linear(in_features=96, out_features=96, bias=True)
+          (q_proj): Linear(in_features=96, out_features=96, bias=True)
+          (out_proj): Linear(in_features=96, out_features=96, bias=True)
+        )
+        (activation_dropout_module): FairseqDropout()
+        (self_attn_layer_norm): LayerNorm((96,), eps=1e-05, elementwise_affine=True)
+        (encoder_attn): MultiheadAttention(
+          (dropout_module): FairseqDropout()
+          (k_proj): Linear(in_features=384, out_features=96, bias=True)
+          (v_proj): Linear(in_features=384, out_features=96, bias=True)
+          (q_proj): Linear(in_features=96, out_features=96, bias=True)
+          (out_proj): Linear(in_features=96, out_features=96, bias=True)
+        )
+        (encoder_attn_layer_norm): LayerNorm((96,), eps=1e-05, elementwise_affine=True)
+        (fc1): Linear(in_features=96, out_features=384, bias=True)
+        (fc2): Linear(in_features=384, out_features=96, bias=True)
+        (final_layer_norm): LayerNorm((96,), eps=1e-05, elementwise_affine=True)
+      )
+    )
+    (output_projection): Linear(in_features=96, out_features=40, bias=False)
+  )
+)
+task: TranslationTask
+model: TransformerModel
+criterion: CrossEntropyCriterion
+num. shared model params: 22,577,856 (num. trained: 22,577,856)
+num. expert model params: 0 (num. trained: 0)
+training on 1 devices (GPUs/TPUs)
+max tokens per device = 20000 and max sentences per device = None
+Start iterating over samples
+begin validation on "valid" subset
+epoch 001 | valid on 'valid' subset | loss 1.653 | ppl 3.14 | wps 26700.3 | wpb 2722.6 | bsz 101 | num_updates 4658
+end of epoch 1 (average epoch stats below)
+epoch 001 | loss 2.308 | ppl 4.95 | wps 8767 | ups 3.16 | wpb 2778.2 | bsz 101.6 | num_updates 4658 | lr 0.00046334 | gnorm 2.703 | train_wall 1452 | gb_free 12.9 | wall 1477
+Start iterating over samples
+begin validation on "valid" subset
+epoch 002 | valid on 'valid' subset | loss 0.938 | ppl 1.92 | wps 27666.1 | wpb 2722.6 | bsz 101 | num_updates 9316 | best_loss 0.938
+epoch 002 | valid on 'valid' subset | loss 0.938 | ppl 1.92 | wps 27666.1 | wpb 2722.6 | bsz 101 | num_updates 9316 | best_loss 0.938
+end of epoch 2 (average epoch stats below)
+epoch 002 | loss 1.284 | ppl 2.44 | wps 8920 | ups 3.21 | wpb 2778.2 | bsz 101.6 | num_updates 9316 | lr 0.000327631 | gnorm 1.316 | train_wall 1427 | gb_free 12.7 | wall 2928
+epoch 002 | loss 1.284 | ppl 2.44 | wps 8920 | ups 3.21 | wpb 2778.2 | bsz 101.6 | num_updates 9316 | lr 0.000327631 | gnorm 1.316 | train_wall 1427 | gb_free 12.7 | wall 2928
+Start iterating over samples
+begin validation on "valid" subset
+epoch 003 | valid on 'valid' subset | loss 0.715 | ppl 1.64 | wps 26749.8 | wpb 2722.6 | bsz 101 | num_updates 13974 | best_loss 0.715
+epoch 003 | valid on 'valid' subset | loss 0.715 | ppl 1.64 | wps 26749.8 | wpb 2722.6 | bsz 101 | num_updates 13974 | best_loss 0.715
+epoch 003 | valid on 'valid' subset | loss 0.715 | ppl 1.64 | wps 26749.8 | wpb 2722.6 | bsz 101 | num_updates 13974 | best_loss 0.715
+end of epoch 3 (average epoch stats below)
+epoch 003 | loss 0.803 | ppl 1.74 | wps 8866.1 | ups 3.19 | wpb 2778.2 | bsz 101.6 | num_updates 13974 | lr 0.00026751 | gnorm 1.088 | train_wall 1435 | gb_free 12.6 | wall 4387
+epoch 003 | loss 0.803 | ppl 1.74 | wps 8866.1 | ups 3.19 | wpb 2778.2 | bsz 101.6 | num_updates 13974 | lr 0.00026751 | gnorm 1.088 | train_wall 1435 | gb_free 12.6 | wall 4387
+epoch 003 | loss 0.803 | ppl 1.74 | wps 8866.1 | ups 3.19 | wpb 2778.2 | bsz 101.6 | num_updates 13974 | lr 0.00026751 | gnorm 1.088 | train_wall 1435 | gb_free 12.6 | wall 4387
+Start iterating over samples
+begin validation on "valid" subset
+epoch 004 | valid on 'valid' subset | loss 0.621 | ppl 1.54 | wps 26630.6 | wpb 2722.6 | bsz 101 | num_updates 18632 | best_loss 0.621
+epoch 004 | valid on 'valid' subset | loss 0.621 | ppl 1.54 | wps 26630.6 | wpb 2722.6 | bsz 101 | num_updates 18632 | best_loss 0.621
+epoch 004 | valid on 'valid' subset | loss 0.621 | ppl 1.54 | wps 26630.6 | wpb 2722.6 | bsz 101 | num_updates 18632 | best_loss 0.621
+epoch 004 | valid on 'valid' subset | loss 0.621 | ppl 1.54 | wps 26630.6 | wpb 2722.6 | bsz 101 | num_updates 18632 | best_loss 0.621
+end of epoch 4 (average epoch stats below)
+epoch 004 | loss 0.586 | ppl 1.5 | wps 8770.3 | ups 3.16 | wpb 2778.2 | bsz 101.6 | num_updates 18632 | lr 0.00023167 | gnorm 0.927 | train_wall 1450 | gb_free 12.9 | wall 5863
+epoch 004 | loss 0.586 | ppl 1.5 | wps 8770.3 | ups 3.16 | wpb 2778.2 | bsz 101.6 | num_updates 18632 | lr 0.00023167 | gnorm 0.927 | train_wall 1450 | gb_free 12.9 | wall 5863
+epoch 004 | loss 0.586 | ppl 1.5 | wps 8770.3 | ups 3.16 | wpb 2778.2 | bsz 101.6 | num_updates 18632 | lr 0.00023167 | gnorm 0.927 | train_wall 1450 | gb_free 12.9 | wall 5863
+epoch 004 | loss 0.586 | ppl 1.5 | wps 8770.3 | ups 3.16 | wpb 2778.2 | bsz 101.6 | num_updates 18632 | lr 0.00023167 | gnorm 0.927 | train_wall 1450 | gb_free 12.9 | wall 5863
+Start iterating over samples
+begin validation on "valid" subset
+epoch 005 | valid on 'valid' subset | loss 0.541 | ppl 1.45 | wps 26626.3 | wpb 2722.6 | bsz 101 | num_updates 23290 | best_loss 0.541
+epoch 005 | valid on 'valid' subset | loss 0.541 | ppl 1.45 | wps 26626.3 | wpb 2722.6 | bsz 101 | num_updates 23290 | best_loss 0.541
+epoch 005 | valid on 'valid' subset | loss 0.541 | ppl 1.45 | wps 26626.3 | wpb 2722.6 | bsz 101 | num_updates 23290 | best_loss 0.541
+epoch 005 | valid on 'valid' subset | loss 0.541 | ppl 1.45 | wps 26626.3 | wpb 2722.6 | bsz 101 | num_updates 23290 | best_loss 0.541
+epoch 005 | valid on 'valid' subset | loss 0.541 | ppl 1.45 | wps 26626.3 | wpb 2722.6 | bsz 101 | num_updates 23290 | best_loss 0.541
+end of epoch 5 (average epoch stats below)
+epoch 005 | loss 0.454 | ppl 1.37 | wps 8758.4 | ups 3.15 | wpb 2778.2 | bsz 101.6 | num_updates 23290 | lr 0.000207212 | gnorm 0.835 | train_wall 1452 | gb_free 12.9 | wall 7340
+epoch 005 | loss 0.454 | ppl 1.37 | wps 8758.4 | ups 3.15 | wpb 2778.2 | bsz 101.6 | num_updates 23290 | lr 0.000207212 | gnorm 0.835 | train_wall 1452 | gb_free 12.9 | wall 7340
+epoch 005 | loss 0.454 | ppl 1.37 | wps 8758.4 | ups 3.15 | wpb 2778.2 | bsz 101.6 | num_updates 23290 | lr 0.000207212 | gnorm 0.835 | train_wall 1452 | gb_free 12.9 | wall 7340
+epoch 005 | loss 0.454 | ppl 1.37 | wps 8758.4 | ups 3.15 | wpb 2778.2 | bsz 101.6 | num_updates 23290 | lr 0.000207212 | gnorm 0.835 | train_wall 1452 | gb_free 12.9 | wall 7340
+epoch 005 | loss 0.454 | ppl 1.37 | wps 8758.4 | ups 3.15 | wpb 2778.2 | bsz 101.6 | num_updates 23290 | lr 0.000207212 | gnorm 0.835 | train_wall 1452 | gb_free 12.9 | wall 7340
+Start iterating over samples
+begin validation on "valid" subset
+epoch 006 | valid on 'valid' subset | loss 0.508 | ppl 1.42 | wps 26570.2 | wpb 2722.6 | bsz 101 | num_updates 27948 | best_loss 0.508
+epoch 006 | valid on 'valid' subset | loss 0.508 | ppl 1.42 | wps 26570.2 | wpb 2722.6 | bsz 101 | num_updates 27948 | best_loss 0.508
+epoch 006 | valid on 'valid' subset | loss 0.508 | ppl 1.42 | wps 26570.2 | wpb 2722.6 | bsz 101 | num_updates 27948 | best_loss 0.508
+epoch 006 | valid on 'valid' subset | loss 0.508 | ppl 1.42 | wps 26570.2 | wpb 2722.6 | bsz 101 | num_updates 27948 | best_loss 0.508
+epoch 006 | valid on 'valid' subset | loss 0.508 | ppl 1.42 | wps 26570.2 | wpb 2722.6 | bsz 101 | num_updates 27948 | best_loss 0.508
+epoch 006 | valid on 'valid' subset | loss 0.508 | ppl 1.42 | wps 26570.2 | wpb 2722.6 | bsz 101 | num_updates 27948 | best_loss 0.508
+end of epoch 6 (average epoch stats below)
+epoch 006 | loss 0.357 | ppl 1.28 | wps 8760.3 | ups 3.15 | wpb 2778.2 | bsz 101.6 | num_updates 27948 | lr 0.000189158 | gnorm 0.782 | train_wall 1452 | gb_free 12.7 | wall 8818
+epoch 006 | loss 0.357 | ppl 1.28 | wps 8760.3 | ups 3.15 | wpb 2778.2 | bsz 101.6 | num_updates 27948 | lr 0.000189158 | gnorm 0.782 | train_wall 1452 | gb_free 12.7 | wall 8818
+epoch 006 | loss 0.357 | ppl 1.28 | wps 8760.3 | ups 3.15 | wpb 2778.2 | bsz 101.6 | num_updates 27948 | lr 0.000189158 | gnorm 0.782 | train_wall 1452 | gb_free 12.7 | wall 8818
+epoch 006 | loss 0.357 | ppl 1.28 | wps 8760.3 | ups 3.15 | wpb 2778.2 | bsz 101.6 | num_updates 27948 | lr 0.000189158 | gnorm 0.782 | train_wall 1452 | gb_free 12.7 | wall 8818
+epoch 006 | loss 0.357 | ppl 1.28 | wps 8760.3 | ups 3.15 | wpb 2778.2 | bsz 101.6 | num_updates 27948 | lr 0.000189158 | gnorm 0.782 | train_wall 1452 | gb_free 12.7 | wall 8818
+epoch 006 | loss 0.357 | ppl 1.28 | wps 8760.3 | ups 3.15 | wpb 2778.2 | bsz 101.6 | num_updates 27948 | lr 0.000189158 | gnorm 0.782 | train_wall 1452 | gb_free 12.7 | wall 8818
+Start iterating over samples
+begin validation on "valid" subset
+epoch 007 | valid on 'valid' subset | loss 0.492 | ppl 1.41 | wps 26701.1 | wpb 2722.6 | bsz 101 | num_updates 32606 | best_loss 0.492
+epoch 007 | valid on 'valid' subset | loss 0.492 | ppl 1.41 | wps 26701.1 | wpb 2722.6 | bsz 101 | num_updates 32606 | best_loss 0.492
+epoch 007 | valid on 'valid' subset | loss 0.492 | ppl 1.41 | wps 26701.1 | wpb 2722.6 | bsz 101 | num_updates 32606 | best_loss 0.492
+epoch 007 | valid on 'valid' subset | loss 0.492 | ppl 1.41 | wps 26701.1 | wpb 2722.6 | bsz 101 | num_updates 32606 | best_loss 0.492
+epoch 007 | valid on 'valid' subset | loss 0.492 | ppl 1.41 | wps 26701.1 | wpb 2722.6 | bsz 101 | num_updates 32606 | best_loss 0.492
+epoch 007 | valid on 'valid' subset | loss 0.492 | ppl 1.41 | wps 26701.1 | wpb 2722.6 | bsz 101 | num_updates 32606 | best_loss 0.492
+epoch 007 | valid on 'valid' subset | loss 0.492 | ppl 1.41 | wps 26701.1 | wpb 2722.6 | bsz 101 | num_updates 32606 | best_loss 0.492
+end of epoch 7 (average epoch stats below)
+epoch 007 | loss 0.28 | ppl 1.21 | wps 8757 | ups 3.15 | wpb 2778.2 | bsz 101.6 | num_updates 32606 | lr 0.000175126 | gnorm 0.739 | train_wall 1453 | gb_free 13 | wall 10295
+epoch 007 | loss 0.28 | ppl 1.21 | wps 8757 | ups 3.15 | wpb 2778.2 | bsz 101.6 | num_updates 32606 | lr 0.000175126 | gnorm 0.739 | train_wall 1453 | gb_free 13 | wall 10295
+epoch 007 | loss 0.28 | ppl 1.21 | wps 8757 | ups 3.15 | wpb 2778.2 | bsz 101.6 | num_updates 32606 | lr 0.000175126 | gnorm 0.739 | train_wall 1453 | gb_free 13 | wall 10295
+epoch 007 | loss 0.28 | ppl 1.21 | wps 8757 | ups 3.15 | wpb 2778.2 | bsz 101.6 | num_updates 32606 | lr 0.000175126 | gnorm 0.739 | train_wall 1453 | gb_free 13 | wall 10295
+epoch 007 | loss 0.28 | ppl 1.21 | wps 8757 | ups 3.15 | wpb 2778.2 | bsz 101.6 | num_updates 32606 | lr 0.000175126 | gnorm 0.739 | train_wall 1453 | gb_free 13 | wall 10295
+epoch 007 | loss 0.28 | ppl 1.21 | wps 8757 | ups 3.15 | wpb 2778.2 | bsz 101.6 | num_updates 32606 | lr 0.000175126 | gnorm 0.739 | train_wall 1453 | gb_free 13 | wall 10295
+epoch 007 | loss 0.28 | ppl 1.21 | wps 8757 | ups 3.15 | wpb 2778.2 | bsz 101.6 | num_updates 32606 | lr 0.000175126 | gnorm 0.739 | train_wall 1453 | gb_free 13 | wall 10295
+Start iterating over samples
+begin validation on "valid" subset
+epoch 008 | valid on 'valid' subset | loss 0.52 | ppl 1.43 | wps 26826.2 | wpb 2722.6 | bsz 101 | num_updates 37264 | best_loss 0.492
+epoch 008 | valid on 'valid' subset | loss 0.52 | ppl 1.43 | wps 26826.2 | wpb 2722.6 | bsz 101 | num_updates 37264 | best_loss 0.492
+epoch 008 | valid on 'valid' subset | loss 0.52 | ppl 1.43 | wps 26826.2 | wpb 2722.6 | bsz 101 | num_updates 37264 | best_loss 0.492
+epoch 008 | valid on 'valid' subset | loss 0.52 | ppl 1.43 | wps 26826.2 | wpb 2722.6 | bsz 101 | num_updates 37264 | best_loss 0.492
+epoch 008 | valid on 'valid' subset | loss 0.52 | ppl 1.43 | wps 26826.2 | wpb 2722.6 | bsz 101 | num_updates 37264 | best_loss 0.492
+epoch 008 | valid on 'valid' subset | loss 0.52 | ppl 1.43 | wps 26826.2 | wpb 2722.6 | bsz 101 | num_updates 37264 | best_loss 0.492
+epoch 008 | valid on 'valid' subset | loss 0.52 | ppl 1.43 | wps 26826.2 | wpb 2722.6 | bsz 101 | num_updates 37264 | best_loss 0.492
+epoch 008 | valid on 'valid' subset | loss 0.52 | ppl 1.43 | wps 26826.2 | wpb 2722.6 | bsz 101 | num_updates 37264 | best_loss 0.492
+end of epoch 8 (average epoch stats below)
+epoch 008 | loss 0.22 | ppl 1.16 | wps 8757.5 | ups 3.15 | wpb 2778.2 | bsz 101.6 | num_updates 37264 | lr 0.000163816 | gnorm 0.698 | train_wall 1453 | gb_free 12.7 | wall 11773
+epoch 008 | loss 0.22 | ppl 1.16 | wps 8757.5 | ups 3.15 | wpb 2778.2 | bsz 101.6 | num_updates 37264 | lr 0.000163816 | gnorm 0.698 | train_wall 1453 | gb_free 12.7 | wall 11773
+epoch 008 | loss 0.22 | ppl 1.16 | wps 8757.5 | ups 3.15 | wpb 2778.2 | bsz 101.6 | num_updates 37264 | lr 0.000163816 | gnorm 0.698 | train_wall 1453 | gb_free 12.7 | wall 11773
+epoch 008 | loss 0.22 | ppl 1.16 | wps 8757.5 | ups 3.15 | wpb 2778.2 | bsz 101.6 | num_updates 37264 | lr 0.000163816 | gnorm 0.698 | train_wall 1453 | gb_free 12.7 | wall 11773
+epoch 008 | loss 0.22 | ppl 1.16 | wps 8757.5 | ups 3.15 | wpb 2778.2 | bsz 101.6 | num_updates 37264 | lr 0.000163816 | gnorm 0.698 | train_wall 1453 | gb_free 12.7 | wall 11773
+epoch 008 | loss 0.22 | ppl 1.16 | wps 8757.5 | ups 3.15 | wpb 2778.2 | bsz 101.6 | num_updates 37264 | lr 0.000163816 | gnorm 0.698 | train_wall 1453 | gb_free 12.7 | wall 11773
+epoch 008 | loss 0.22 | ppl 1.16 | wps 8757.5 | ups 3.15 | wpb 2778.2 | bsz 101.6 | num_updates 37264 | lr 0.000163816 | gnorm 0.698 | train_wall 1453 | gb_free 12.7 | wall 11773
+epoch 008 | loss 0.22 | ppl 1.16 | wps 8757.5 | ups 3.15 | wpb 2778.2 | bsz 101.6 | num_updates 37264 | lr 0.000163816 | gnorm 0.698 | train_wall 1453 | gb_free 12.7 | wall 11773
+Start iterating over samples
+begin validation on "valid" subset
+epoch 009 | valid on 'valid' subset | loss 0.513 | ppl 1.43 | wps 27649.3 | wpb 2722.6 | bsz 101 | num_updates 41922 | best_loss 0.492
+epoch 009 | valid on 'valid' subset | loss 0.513 | ppl 1.43 | wps 27649.3 | wpb 2722.6 | bsz 101 | num_updates 41922 | best_loss 0.492
+epoch 009 | valid on 'valid' subset | loss 0.513 | ppl 1.43 | wps 27649.3 | wpb 2722.6 | bsz 101 | num_updates 41922 | best_loss 0.492
+epoch 009 | valid on 'valid' subset | loss 0.513 | ppl 1.43 | wps 27649.3 | wpb 2722.6 | bsz 101 | num_updates 41922 | best_loss 0.492
+epoch 009 | valid on 'valid' subset | loss 0.513 | ppl 1.43 | wps 27649.3 | wpb 2722.6 | bsz 101 | num_updates 41922 | best_loss 0.492
+epoch 009 | valid on 'valid' subset | loss 0.513 | ppl 1.43 | wps 27649.3 | wpb 2722.6 | bsz 101 | num_updates 41922 | best_loss 0.492
+epoch 009 | valid on 'valid' subset | loss 0.513 | ppl 1.43 | wps 27649.3 | wpb 2722.6 | bsz 101 | num_updates 41922 | best_loss 0.492
+epoch 009 | valid on 'valid' subset | loss 0.513 | ppl 1.43 | wps 27649.3 | wpb 2722.6 | bsz 101 | num_updates 41922 | best_loss 0.492
+epoch 009 | valid on 'valid' subset | loss 0.513 | ppl 1.43 | wps 27649.3 | wpb 2722.6 | bsz 101 | num_updates 41922 | best_loss 0.492
+end of epoch 9 (average epoch stats below)
+epoch 009 | loss 0.174 | ppl 1.13 | wps 8934.2 | ups 3.22 | wpb 2778.2 | bsz 101.6 | num_updates 41922 | lr 0.000154447 | gnorm 0.671 | train_wall 1425 | gb_free 12.7 | wall 13221
+epoch 009 | loss 0.174 | ppl 1.13 | wps 8934.2 | ups 3.22 | wpb 2778.2 | bsz 101.6 | num_updates 41922 | lr 0.000154447 | gnorm 0.671 | train_wall 1425 | gb_free 12.7 | wall 13221
+epoch 009 | loss 0.174 | ppl 1.13 | wps 8934.2 | ups 3.22 | wpb 2778.2 | bsz 101.6 | num_updates 41922 | lr 0.000154447 | gnorm 0.671 | train_wall 1425 | gb_free 12.7 | wall 13221
+epoch 009 | loss 0.174 | ppl 1.13 | wps 8934.2 | ups 3.22 | wpb 2778.2 | bsz 101.6 | num_updates 41922 | lr 0.000154447 | gnorm 0.671 | train_wall 1425 | gb_free 12.7 | wall 13221
+epoch 009 | loss 0.174 | ppl 1.13 | wps 8934.2 | ups 3.22 | wpb 2778.2 | bsz 101.6 | num_updates 41922 | lr 0.000154447 | gnorm 0.671 | train_wall 1425 | gb_free 12.7 | wall 13221
+epoch 009 | loss 0.174 | ppl 1.13 | wps 8934.2 | ups 3.22 | wpb 2778.2 | bsz 101.6 | num_updates 41922 | lr 0.000154447 | gnorm 0.671 | train_wall 1425 | gb_free 12.7 | wall 13221
+epoch 009 | loss 0.174 | ppl 1.13 | wps 8934.2 | ups 3.22 | wpb 2778.2 | bsz 101.6 | num_updates 41922 | lr 0.000154447 | gnorm 0.671 | train_wall 1425 | gb_free 12.7 | wall 13221
+epoch 009 | loss 0.174 | ppl 1.13 | wps 8934.2 | ups 3.22 | wpb 2778.2 | bsz 101.6 | num_updates 41922 | lr 0.000154447 | gnorm 0.671 | train_wall 1425 | gb_free 12.7 | wall 13221
+epoch 009 | loss 0.174 | ppl 1.13 | wps 8934.2 | ups 3.22 | wpb 2778.2 | bsz 101.6 | num_updates 41922 | lr 0.000154447 | gnorm 0.671 | train_wall 1425 | gb_free 12.7 | wall 13221
+Start iterating over samples
+begin validation on "valid" subset
+epoch 010 | valid on 'valid' subset | loss 0.591 | ppl 1.51 | wps 27575.7 | wpb 2722.6 | bsz 101 | num_updates 46580 | best_loss 0.492
+epoch 010 | valid on 'valid' subset | loss 0.591 | ppl 1.51 | wps 27575.7 | wpb 2722.6 | bsz 101 | num_updates 46580 | best_loss 0.492
+epoch 010 | valid on 'valid' subset | loss 0.591 | ppl 1.51 | wps 27575.7 | wpb 2722.6 | bsz 101 | num_updates 46580 | best_loss 0.492
+epoch 010 | valid on 'valid' subset | loss 0.591 | ppl 1.51 | wps 27575.7 | wpb 2722.6 | bsz 101 | num_updates 46580 | best_loss 0.492
+epoch 010 | valid on 'valid' subset | loss 0.591 | ppl 1.51 | wps 27575.7 | wpb 2722.6 | bsz 101 | num_updates 46580 | best_loss 0.492
+epoch 010 | valid on 'valid' subset | loss 0.591 | ppl 1.51 | wps 27575.7 | wpb 2722.6 | bsz 101 | num_updates 46580 | best_loss 0.492
+epoch 010 | valid on 'valid' subset | loss 0.591 | ppl 1.51 | wps 27575.7 | wpb 2722.6 | bsz 101 | num_updates 46580 | best_loss 0.492
+epoch 010 | valid on 'valid' subset | loss 0.591 | ppl 1.51 | wps 27575.7 | wpb 2722.6 | bsz 101 | num_updates 46580 | best_loss 0.492
+epoch 010 | valid on 'valid' subset | loss 0.591 | ppl 1.51 | wps 27575.7 | wpb 2722.6 | bsz 101 | num_updates 46580 | best_loss 0.492
+epoch 010 | valid on 'valid' subset | loss 0.591 | ppl 1.51 | wps 27575.7 | wpb 2722.6 | bsz 101 | num_updates 46580 | best_loss 0.492
+end of epoch 10 (average epoch stats below)
+epoch 010 | loss 0.137 | ppl 1.1 | wps 9058.9 | ups 3.26 | wpb 2778.2 | bsz 101.6 | num_updates 46580 | lr 0.000146521 | gnorm 0.643 | train_wall 1406 | gb_free 12.8 | wall 14650
+epoch 010 | loss 0.137 | ppl 1.1 | wps 9058.9 | ups 3.26 | wpb 2778.2 | bsz 101.6 | num_updates 46580 | lr 0.000146521 | gnorm 0.643 | train_wall 1406 | gb_free 12.8 | wall 14650
+epoch 010 | loss 0.137 | ppl 1.1 | wps 9058.9 | ups 3.26 | wpb 2778.2 | bsz 101.6 | num_updates 46580 | lr 0.000146521 | gnorm 0.643 | train_wall 1406 | gb_free 12.8 | wall 14650
+epoch 010 | loss 0.137 | ppl 1.1 | wps 9058.9 | ups 3.26 | wpb 2778.2 | bsz 101.6 | num_updates 46580 | lr 0.000146521 | gnorm 0.643 | train_wall 1406 | gb_free 12.8 | wall 14650
+epoch 010 | loss 0.137 | ppl 1.1 | wps 9058.9 | ups 3.26 | wpb 2778.2 | bsz 101.6 | num_updates 46580 | lr 0.000146521 | gnorm 0.643 | train_wall 1406 | gb_free 12.8 | wall 14650
+epoch 010 | loss 0.137 | ppl 1.1 | wps 9058.9 | ups 3.26 | wpb 2778.2 | bsz 101.6 | num_updates 46580 | lr 0.000146521 | gnorm 0.643 | train_wall 1406 | gb_free 12.8 | wall 14650
+epoch 010 | loss 0.137 | ppl 1.1 | wps 9058.9 | ups 3.26 | wpb 2778.2 | bsz 101.6 | num_updates 46580 | lr 0.000146521 | gnorm 0.643 | train_wall 1406 | gb_free 12.8 | wall 14650
+epoch 010 | loss 0.137 | ppl 1.1 | wps 9058.9 | ups 3.26 | wpb 2778.2 | bsz 101.6 | num_updates 46580 | lr 0.000146521 | gnorm 0.643 | train_wall 1406 | gb_free 12.8 | wall 14650
+epoch 010 | loss 0.137 | ppl 1.1 | wps 9058.9 | ups 3.26 | wpb 2778.2 | bsz 101.6 | num_updates 46580 | lr 0.000146521 | gnorm 0.643 | train_wall 1406 | gb_free 12.8 | wall 14650
+epoch 010 | loss 0.137 | ppl 1.1 | wps 9058.9 | ups 3.26 | wpb 2778.2 | bsz 101.6 | num_updates 46580 | lr 0.000146521 | gnorm 0.643 | train_wall 1406 | gb_free 12.8 | wall 14650
+Start iterating over samples
+begin validation on "valid" subset
+epoch 011 | valid on 'valid' subset | loss 0.593 | ppl 1.51 | wps 27763.5 | wpb 2722.6 | bsz 101 | num_updates 51238 | best_loss 0.492
+epoch 011 | valid on 'valid' subset | loss 0.593 | ppl 1.51 | wps 27763.5 | wpb 2722.6 | bsz 101 | num_updates 51238 | best_loss 0.492
+epoch 011 | valid on 'valid' subset | loss 0.593 | ppl 1.51 | wps 27763.5 | wpb 2722.6 | bsz 101 | num_updates 51238 | best_loss 0.492
+epoch 011 | valid on 'valid' subset | loss 0.593 | ppl 1.51 | wps 27763.5 | wpb 2722.6 | bsz 101 | num_updates 51238 | best_loss 0.492
+epoch 011 | valid on 'valid' subset | loss 0.593 | ppl 1.51 | wps 27763.5 | wpb 2722.6 | bsz 101 | num_updates 51238 | best_loss 0.492
+epoch 011 | valid on 'valid' subset | loss 0.593 | ppl 1.51 | wps 27763.5 | wpb 2722.6 | bsz 101 | num_updates 51238 | best_loss 0.492
+epoch 011 | valid on 'valid' subset | loss 0.593 | ppl 1.51 | wps 27763.5 | wpb 2722.6 | bsz 101 | num_updates 51238 | best_loss 0.492
+epoch 011 | valid on 'valid' subset | loss 0.593 | ppl 1.51 | wps 27763.5 | wpb 2722.6 | bsz 101 | num_updates 51238 | best_loss 0.492
+epoch 011 | valid on 'valid' subset | loss 0.593 | ppl 1.51 | wps 27763.5 | wpb 2722.6 | bsz 101 | num_updates 51238 | best_loss 0.492
+epoch 011 | valid on 'valid' subset | loss 0.593 | ppl 1.51 | wps 27763.5 | wpb 2722.6 | bsz 101 | num_updates 51238 | best_loss 0.492
+epoch 011 | valid on 'valid' subset | loss 0.593 | ppl 1.51 | wps 27763.5 | wpb 2722.6 | bsz 101 | num_updates 51238 | best_loss 0.492
+early stop since valid performance hasn't improved for last 4 runs
+end of epoch 11 (average epoch stats below)
+epoch 011 | loss 0.11 | ppl 1.08 | wps 9065.3 | ups 3.26 | wpb 2778.2 | bsz 101.6 | num_updates 51238 | lr 0.000139702 | gnorm 0.618 | train_wall 1405 | gb_free 12.9 | wall 16078
+epoch 011 | loss 0.11 | ppl 1.08 | wps 9065.3 | ups 3.26 | wpb 2778.2 | bsz 101.6 | num_updates 51238 | lr 0.000139702 | gnorm 0.618 | train_wall 1405 | gb_free 12.9 | wall 16078
+epoch 011 | loss 0.11 | ppl 1.08 | wps 9065.3 | ups 3.26 | wpb 2778.2 | bsz 101.6 | num_updates 51238 | lr 0.000139702 | gnorm 0.618 | train_wall 1405 | gb_free 12.9 | wall 16078
+epoch 011 | loss 0.11 | ppl 1.08 | wps 9065.3 | ups 3.26 | wpb 2778.2 | bsz 101.6 | num_updates 51238 | lr 0.000139702 | gnorm 0.618 | train_wall 1405 | gb_free 12.9 | wall 16078
+epoch 011 | loss 0.11 | ppl 1.08 | wps 9065.3 | ups 3.26 | wpb 2778.2 | bsz 101.6 | num_updates 51238 | lr 0.000139702 | gnorm 0.618 | train_wall 1405 | gb_free 12.9 | wall 16078
+epoch 011 | loss 0.11 | ppl 1.08 | wps 9065.3 | ups 3.26 | wpb 2778.2 | bsz 101.6 | num_updates 51238 | lr 0.000139702 | gnorm 0.618 | train_wall 1405 | gb_free 12.9 | wall 16078
+epoch 011 | loss 0.11 | ppl 1.08 | wps 9065.3 | ups 3.26 | wpb 2778.2 | bsz 101.6 | num_updates 51238 | lr 0.000139702 | gnorm 0.618 | train_wall 1405 | gb_free 12.9 | wall 16078
+epoch 011 | loss 0.11 | ppl 1.08 | wps 9065.3 | ups 3.26 | wpb 2778.2 | bsz 101.6 | num_updates 51238 | lr 0.000139702 | gnorm 0.618 | train_wall 1405 | gb_free 12.9 | wall 16078
+epoch 011 | loss 0.11 | ppl 1.08 | wps 9065.3 | ups 3.26 | wpb 2778.2 | bsz 101.6 | num_updates 51238 | lr 0.000139702 | gnorm 0.618 | train_wall 1405 | gb_free 12.9 | wall 16078
+epoch 011 | loss 0.11 | ppl 1.08 | wps 9065.3 | ups 3.26 | wpb 2778.2 | bsz 101.6 | num_updates 51238 | lr 0.000139702 | gnorm 0.618 | train_wall 1405 | gb_free 12.9 | wall 16078
+epoch 011 | loss 0.11 | ppl 1.08 | wps 9065.3 | ups 3.26 | wpb 2778.2 | bsz 101.6 | num_updates 51238 | lr 0.000139702 | gnorm 0.618 | train_wall 1405 | gb_free 12.9 | wall 16078
+done training in 16077.1 seconds

ablations/noconst/trained_arm32_best_noconst/beam1.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:205eb1691e75be23c8760c7f9a257b2be617ee3a8ec97eb588d8e0e39c13bccd
+size 2952656

ablations/noconst/trained_arm32_best_noconst/checkpoint_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b49e1926c18a8ff27057a4a94716be61dc5df10d16478eb64d94f76e9eeb3241
+size 124639436

ablations/noconst/trained_arm32_best_noconst/training.log

@@ -0,0 +1,230 @@
+{'_name': None, 'common': {'_name': None, 'no_progress_bar': False, 'log_interval': 100, 'log_format': None, 'log_file': 'arm32/trained_arm32_best_noconst/training.log', 'aim_repo': None, 'aim_run_hash': None, 'tensorboard_logdir': None, 'wandb_project': None, 'azureml_logging': False, 'seed': 1, 'cpu': False, 'tpu': False, 'bf16': False, 'memory_efficient_bf16': False, 'fp16': False, 'memory_efficient_fp16': False, 'fp16_no_flatten_grads': False, 'fp16_init_scale': 128, 'fp16_scale_window': None, 'fp16_scale_tolerance': 0.0, 'on_cpu_convert_precision': False, 'min_loss_scale': 0.0001, 'threshold_loss_scale': None, 'amp': False, 'amp_batch_retries': 2, 'amp_init_scale': 128, 'amp_scale_window': None, 'user_dir': None, 'empty_cache_freq': 0, 'all_gather_list_size': 16384, 'model_parallel_size': 1, 'quantization_config_path': None, 'profile': False, 'reset_logging': False, 'suppress_crashes': False, 'use_plasma_view': False, 'plasma_path': '/tmp/plasma'}, 'common_eval': {'_name': None, 'path': None, 'post_process': None, 'quiet': False, 'model_overrides': '{}', 'results_path': None}, 'distributed_training': {'_name': None, 'distributed_world_size': 1, 'distributed_num_procs': 1, 'distributed_rank': 0, 'distributed_backend': 'nccl', 'distributed_init_method': None, 'distributed_port': -1, 'device_id': 0, 'distributed_no_spawn': False, 'ddp_backend': 'pytorch_ddp', 'ddp_comm_hook': 'none', 'bucket_cap_mb': 25, 'fix_batches_to_gpus': False, 'find_unused_parameters': False, 'gradient_as_bucket_view': False, 'fast_stat_sync': False, 'heartbeat_timeout': -1, 'broadcast_buffers': False, 'slowmo_momentum': None, 'slowmo_base_algorithm': 'localsgd', 'localsgd_frequency': 3, 'nprocs_per_node': 1, 'pipeline_model_parallel': False, 'pipeline_balance': None, 'pipeline_devices': None, 'pipeline_chunks': 0, 'pipeline_encoder_balance': None, 'pipeline_encoder_devices': None, 'pipeline_decoder_balance': None, 'pipeline_decoder_devices': None, 'pipeline_checkpoint': 'never', 'zero_sharding': 'none', 'fp16': False, 'memory_efficient_fp16': False, 'tpu': False, 'no_reshard_after_forward': False, 'fp32_reduce_scatter': False, 'cpu_offload': False, 'use_sharded_state': False, 'not_fsdp_flatten_parameters': False}, 'dataset': {'_name': None, 'num_workers': 1, 'skip_invalid_size_inputs_valid_test': False, 'max_tokens': 20000, 'batch_size': None, 'required_batch_size_multiple': 8, 'required_seq_len_multiple': 1, 'dataset_impl': None, 'data_buffer_size': 10, 'train_subset': 'train', 'valid_subset': 'valid', 'combine_valid_subsets': None, 'ignore_unused_valid_subsets': False, 'validate_interval': 1, 'validate_interval_updates': 0, 'validate_after_updates': 0, 'fixed_validation_seed': None, 'disable_validation': False, 'max_tokens_valid': 20000, 'batch_size_valid': None, 'max_valid_steps': None, 'curriculum': 0, 'gen_subset': 'test', 'num_shards': 1, 'shard_id': 0, 'grouped_shuffling': False, 'update_epoch_batch_itr': False, 'update_ordered_indices_seed': False}, 'optimization': {'_name': None, 'max_epoch': 0, 'max_update': 100000, 'stop_time_hours': 0.0, 'clip_norm': 0.0, 'sentence_avg': False, 'update_freq': [1], 'lr': [0.0005], 'stop_min_lr': -1.0, 'use_bmuf': False, 'skip_remainder_batch': False}, 'checkpoint': {'_name': None, 'save_dir': 'arm32/trained_arm32_best_noconst', 'restore_file': 'checkpoint_last.pt', 'continue_once': None, 'finetune_from_model': None, 'reset_dataloader': False, 'reset_lr_scheduler': False, 'reset_meters': False, 'reset_optimizer': False, 'optimizer_overrides': '{}', 'save_interval': 1, 'save_interval_updates': 0, 'keep_interval_updates': -1, 'keep_interval_updates_pattern': -1, 'keep_last_epochs': -1, 'keep_best_checkpoints': 3, 'no_save': False, 'no_epoch_checkpoints': True, 'no_last_checkpoints': False, 'no_save_optimizer_state': False, 'best_checkpoint_metric': 'loss', 'maximize_best_checkpoint_metric': False, 'patience': 4, 'checkpoint_suffix': '', 'checkpoint_shard_count': 1, 'load_checkpoint_on_all_dp_ranks': False, 'write_checkpoints_asynchronously': False, 'model_parallel_size': 1}, 'bmuf': {'_name': None, 'block_lr': 1.0, 'block_momentum': 0.875, 'global_sync_iter': 50, 'warmup_iterations': 500, 'use_nbm': False, 'average_sync': False, 'distributed_world_size': 1}, 'generation': {'_name': None, 'beam': 5, 'nbest': 1, 'max_len_a': 0.0, 'max_len_b': 200, 'min_len': 1, 'match_source_len': False, 'unnormalized': False, 'no_early_stop': False, 'no_beamable_mm': False, 'lenpen': 1.0, 'unkpen': 0.0, 'replace_unk': None, 'sacrebleu': False, 'score_reference': False, 'prefix_size': 0, 'no_repeat_ngram_size': 0, 'sampling': False, 'sampling_topk': -1, 'sampling_topp': -1.0, 'constraints': None, 'temperature': 1.0, 'diverse_beam_groups': -1, 'diverse_beam_strength': 0.5, 'diversity_rate': -1.0, 'print_alignment': None, 'print_step': False, 'lm_path': None, 'lm_weight': 0.0, 'iter_decode_eos_penalty': 0.0, 'iter_decode_max_iter': 10, 'iter_decode_force_max_iter': False, 'iter_decode_with_beam': 1, 'iter_decode_with_external_reranker': False, 'retain_iter_history': False, 'retain_dropout': False, 'retain_dropout_modules': None, 'decoding_format': None, 'no_seed_provided': False, 'eos_token': None}, 'eval_lm': {'_name': None, 'output_word_probs': False, 'output_word_stats': False, 'context_window': 0, 'softmax_batch': 9223372036854775807}, 'interactive': {'_name': None, 'buffer_size': 0, 'input': '-'}, 'model': Namespace(_name='transformer', activation_dropout=0.0, activation_fn='relu', adam_betas=(0.9, 0.999), adam_eps=1e-08, adaptive_input=False, adaptive_softmax_cutoff=None, adaptive_softmax_dropout=0, aim_repo=None, aim_run_hash=None, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, arch='transformer', attention_dropout=0.0, azureml_logging=False, batch_size=None, batch_size_valid=None, best_checkpoint_metric='loss', bf16=False, bpe=None, broadcast_buffers=False, bucket_cap_mb=25, checkpoint_activations=False, checkpoint_shard_count=1, checkpoint_suffix='', clip_norm=0.0, combine_valid_subsets=None, continue_once=None, cpu=False, cpu_offload=False, criterion='cross_entropy', cross_self_attention=False, curriculum=0, data='arm32/tokenized_dlsm_arm32_noconst', data_buffer_size=10, dataset_impl=None, ddp_backend='pytorch_ddp', ddp_comm_hook='none', decoder_attention_heads=8, decoder_embed_dim=96, decoder_embed_path=None, decoder_ffn_embed_dim=384, decoder_input_dim=96, decoder_layerdrop=0, decoder_layers=4, decoder_layers_to_keep=None, decoder_learned_pos=False, decoder_normalize_before=False, decoder_output_dim='96', device_id=0, disable_validation=False, distributed_backend='nccl', distributed_init_method=None, distributed_no_spawn=False, distributed_num_procs=1, distributed_port=-1, distributed_rank=0, distributed_world_size=1, dropout=0.05, ema_decay=0.9999, ema_fp32=False, ema_seed_model=None, ema_start_update=0, ema_update_freq=1, empty_cache_freq=0, encoder_attention_heads=8, encoder_embed_dim=384, encoder_embed_path=None, encoder_ffn_embed_dim=1536, encoder_layerdrop=0, encoder_layers=4, encoder_layers_to_keep=None, encoder_learned_pos=False, encoder_normalize_before=False, eos=2, eval_bleu=False, eval_bleu_args='{}', eval_bleu_detok='space', eval_bleu_detok_args='{}', eval_bleu_print_samples=False, eval_bleu_remove_bpe=None, eval_tokenized_bleu=False, fast_stat_sync=False, find_unused_parameters=False, finetune_from_model=None, fix_batches_to_gpus=False, fixed_validation_seed=None, fp16=False, fp16_adam_stats=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, fp32_reduce_scatter=False, gen_subset='test', gradient_as_bucket_view=False, grouped_shuffling=False, heartbeat_timeout=-1, ignore_unused_valid_subsets=False, keep_best_checkpoints=3, keep_interval_updates=-1, keep_interval_updates_pattern=-1, keep_last_epochs=-1, layernorm_embedding=False, left_pad_source=True, left_pad_target=False, load_alignments=False, load_checkpoint_on_all_dp_ranks=False, localsgd_frequency=3, log_file='arm32/trained_arm32_best_noconst/training.log', log_format=None, log_interval=100, lr=[0.0005], lr_scheduler='inverse_sqrt', max_epoch=0, max_source_positions=1024, max_target_positions=1024, max_tokens=20000, max_tokens_valid=20000, max_update=100000, max_valid_steps=None, maximize_best_checkpoint_metric=False, memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_cross_attention=False, no_epoch_checkpoints=True, no_last_checkpoints=False, no_progress_bar=False, no_reshard_after_forward=False, no_save=False, no_save_optimizer_state=False, no_scale_embedding=False, no_seed_provided=False, no_token_positional_embeddings=False, not_fsdp_flatten_parameters=False, nprocs_per_node=1, num_batch_buckets=0, num_shards=1, num_workers=1, offload_activations=False, on_cpu_convert_precision=False, optimizer='adam', optimizer_overrides='{}', pad=1, patience=4, pipeline_balance=None, pipeline_checkpoint='never', pipeline_chunks=0, pipeline_decoder_balance=None, pipeline_decoder_devices=None, pipeline_devices=None, pipeline_encoder_balance=None, pipeline_encoder_devices=None, pipeline_model_parallel=False, plasma_path='/tmp/plasma', profile=False, quant_noise_pq=0, quant_noise_pq_block_size=8, quant_noise_scalar=0, quantization_config_path=None, required_batch_size_multiple=8, required_seq_len_multiple=1, reset_dataloader=False, reset_logging=False, reset_lr_scheduler=False, reset_meters=False, reset_optimizer=False, restore_file='checkpoint_last.pt', save_dir='arm32/trained_arm32_best_noconst', save_interval=1, save_interval_updates=0, scoring='bleu', seed=1, sentence_avg=False, shard_id=0, share_all_embeddings=False, share_decoder_input_output_embed=False, skip_invalid_size_inputs_valid_test=False, skip_remainder_batch=False, slowmo_base_algorithm='localsgd', slowmo_momentum=None, source_lang=None, stop_min_lr=-1.0, stop_time_hours=0, store_ema=False, suppress_crashes=False, target_lang=None, task='translation', tensorboard_logdir=None, threshold_loss_scale=None, tie_adaptive_weights=False, tokenizer=None, tpu=False, train_subset='train', truncate_source=False, unk=3, update_epoch_batch_itr=False, update_freq=[1], update_ordered_indices_seed=False, upsample_primary=-1, use_bmuf=False, use_old_adam=False, use_plasma_view=False, use_sharded_state=False, user_dir=None, valid_subset='valid', validate_after_updates=0, validate_interval=1, validate_interval_updates=0, wandb_project=None, warmup_init_lr=-1, warmup_updates=4000, weight_decay=0.001, write_checkpoints_asynchronously=False, zero_sharding='none'), 'task': {'_name': 'translation', 'data': 'arm32/tokenized_dlsm_arm32_noconst', 'source_lang': None, 'target_lang': None, 'load_alignments': False, 'left_pad_source': True, 'left_pad_target': False, 'max_source_positions': 1024, 'max_target_positions': 1024, 'upsample_primary': -1, 'truncate_source': False, 'num_batch_buckets': 0, 'train_subset': 'train', 'dataset_impl': None, 'required_seq_len_multiple': 1, 'eval_bleu': False, 'eval_bleu_args': '{}', 'eval_bleu_detok': 'space', 'eval_bleu_detok_args': '{}', 'eval_tokenized_bleu': False, 'eval_bleu_remove_bpe': None, 'eval_bleu_print_samples': False}, 'criterion': {'_name': 'cross_entropy', 'sentence_avg': False}, 'optimizer': {'_name': 'adam', 'adam_betas': [0.9, 0.999], 'adam_eps': 1e-08, 'weight_decay': 0.001, 'use_old_adam': False, 'fp16_adam_stats': False, 'tpu': False, 'lr': [0.0005]}, 'lr_scheduler': {'_name': 'inverse_sqrt', 'warmup_updates': 4000, 'warmup_init_lr': -1.0, 'lr': [0.0005]}, 'scoring': {'_name': 'bleu', 'pad': 1, 'eos': 2, 'unk': 3}, 'bpe': None, 'tokenizer': None, 'ema': {'_name': None, 'store_ema': False, 'ema_decay': 0.9999, 'ema_start_update': 0, 'ema_seed_model': None, 'ema_update_freq': 1, 'ema_fp32': False}}
+TransformerModel(
+  (encoder): TransformerEncoderBase(
+    (dropout_module): FairseqDropout()
+    (embed_tokens): Embedding(216, 384, padding_idx=1)
+    (embed_positions): SinusoidalPositionalEmbedding()
+    (layers): ModuleList(
+      (0-3): 4 x TransformerEncoderLayerBase(
+        (self_attn): MultiheadAttention(
+          (dropout_module): FairseqDropout()
+          (k_proj): Linear(in_features=384, out_features=384, bias=True)
+          (v_proj): Linear(in_features=384, out_features=384, bias=True)
+          (q_proj): Linear(in_features=384, out_features=384, bias=True)
+          (out_proj): Linear(in_features=384, out_features=384, bias=True)
+        )
+        (self_attn_layer_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+        (dropout_module): FairseqDropout()
+        (activation_dropout_module): FairseqDropout()
+        (fc1): Linear(in_features=384, out_features=1536, bias=True)
+        (fc2): Linear(in_features=1536, out_features=384, bias=True)
+        (final_layer_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+      )
+    )
+  )
+  (decoder): TransformerDecoderBase(
+    (dropout_module): FairseqDropout()
+    (embed_tokens): Embedding(40, 96, padding_idx=1)
+    (embed_positions): SinusoidalPositionalEmbedding()
+    (layers): ModuleList(
+      (0-3): 4 x TransformerDecoderLayerBase(
+        (dropout_module): FairseqDropout()
+        (self_attn): MultiheadAttention(
+          (dropout_module): FairseqDropout()
+          (k_proj): Linear(in_features=96, out_features=96, bias=True)
+          (v_proj): Linear(in_features=96, out_features=96, bias=True)
+          (q_proj): Linear(in_features=96, out_features=96, bias=True)
+          (out_proj): Linear(in_features=96, out_features=96, bias=True)
+        )
+        (activation_dropout_module): FairseqDropout()
+        (self_attn_layer_norm): LayerNorm((96,), eps=1e-05, elementwise_affine=True)
+        (encoder_attn): MultiheadAttention(
+          (dropout_module): FairseqDropout()
+          (k_proj): Linear(in_features=384, out_features=96, bias=True)
+          (v_proj): Linear(in_features=384, out_features=96, bias=True)
+          (q_proj): Linear(in_features=96, out_features=96, bias=True)
+          (out_proj): Linear(in_features=96, out_features=96, bias=True)
+        )
+        (encoder_attn_layer_norm): LayerNorm((96,), eps=1e-05, elementwise_affine=True)
+        (fc1): Linear(in_features=96, out_features=384, bias=True)
+        (fc2): Linear(in_features=384, out_features=96, bias=True)
+        (final_layer_norm): LayerNorm((96,), eps=1e-05, elementwise_affine=True)
+      )
+    )
+    (output_projection): Linear(in_features=96, out_features=40, bias=False)
+  )
+)
+task: TranslationTask
+model: TransformerModel
+criterion: CrossEntropyCriterion
+num. shared model params: 15,098,496 (num. trained: 15,098,496)
+num. expert model params: 0 (num. trained: 0)
+training on 1 devices (GPUs/TPUs)
+max tokens per device = 20000 and max sentences per device = None
+Start iterating over samples
+begin validation on "valid" subset
+epoch 001 | valid on 'valid' subset | loss 1.532 | ppl 2.89 | wps 44149.4 | wpb 2858.9 | bsz 108.4 | num_updates 3926
+end of epoch 1 (average epoch stats below)
+epoch 001 | loss 2.22 | ppl 4.66 | wps 14250.5 | ups 4.81 | wpb 2961.7 | bsz 110.3 | num_updates 3926 | lr 0.00049075 | gnorm 2.478 | train_wall 800 | gb_free 13.9 | wall 817
+Start iterating over samples
+begin validation on "valid" subset
+epoch 002 | valid on 'valid' subset | loss 0.887 | ppl 1.85 | wps 44076.7 | wpb 2858.9 | bsz 108.4 | num_updates 7852 | best_loss 0.887
+epoch 002 | valid on 'valid' subset | loss 0.887 | ppl 1.85 | wps 44076.7 | wpb 2858.9 | bsz 108.4 | num_updates 7852 | best_loss 0.887
+end of epoch 2 (average epoch stats below)
+epoch 002 | loss 1.238 | ppl 2.36 | wps 14211.8 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 7852 | lr 0.00035687 | gnorm 1.187 | train_wall 802 | gb_free 13.4 | wall 1635
+epoch 002 | loss 1.238 | ppl 2.36 | wps 14211.8 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 7852 | lr 0.00035687 | gnorm 1.187 | train_wall 802 | gb_free 13.4 | wall 1635
+Start iterating over samples
+begin validation on "valid" subset
+epoch 003 | valid on 'valid' subset | loss 0.664 | ppl 1.58 | wps 43946.5 | wpb 2858.9 | bsz 108.4 | num_updates 11778 | best_loss 0.664
+epoch 003 | valid on 'valid' subset | loss 0.664 | ppl 1.58 | wps 43946.5 | wpb 2858.9 | bsz 108.4 | num_updates 11778 | best_loss 0.664
+epoch 003 | valid on 'valid' subset | loss 0.664 | ppl 1.58 | wps 43946.5 | wpb 2858.9 | bsz 108.4 | num_updates 11778 | best_loss 0.664
+end of epoch 3 (average epoch stats below)
+epoch 003 | loss 0.78 | ppl 1.72 | wps 14233.1 | ups 4.81 | wpb 2961.7 | bsz 110.3 | num_updates 11778 | lr 0.000291383 | gnorm 0.943 | train_wall 801 | gb_free 13 | wall 2452
+epoch 003 | loss 0.78 | ppl 1.72 | wps 14233.1 | ups 4.81 | wpb 2961.7 | bsz 110.3 | num_updates 11778 | lr 0.000291383 | gnorm 0.943 | train_wall 801 | gb_free 13 | wall 2452
+epoch 003 | loss 0.78 | ppl 1.72 | wps 14233.1 | ups 4.81 | wpb 2961.7 | bsz 110.3 | num_updates 11778 | lr 0.000291383 | gnorm 0.943 | train_wall 801 | gb_free 13 | wall 2452
+Start iterating over samples
+begin validation on "valid" subset
+epoch 004 | valid on 'valid' subset | loss 0.554 | ppl 1.47 | wps 44110 | wpb 2858.9 | bsz 108.4 | num_updates 15704 | best_loss 0.554
+epoch 004 | valid on 'valid' subset | loss 0.554 | ppl 1.47 | wps 44110 | wpb 2858.9 | bsz 108.4 | num_updates 15704 | best_loss 0.554
+epoch 004 | valid on 'valid' subset | loss 0.554 | ppl 1.47 | wps 44110 | wpb 2858.9 | bsz 108.4 | num_updates 15704 | best_loss 0.554
+epoch 004 | valid on 'valid' subset | loss 0.554 | ppl 1.47 | wps 44110 | wpb 2858.9 | bsz 108.4 | num_updates 15704 | best_loss 0.554
+end of epoch 4 (average epoch stats below)
+epoch 004 | loss 0.571 | ppl 1.49 | wps 14206.7 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 15704 | lr 0.000252345 | gnorm 0.79 | train_wall 802 | gb_free 13.3 | wall 3270
+epoch 004 | loss 0.571 | ppl 1.49 | wps 14206.7 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 15704 | lr 0.000252345 | gnorm 0.79 | train_wall 802 | gb_free 13.3 | wall 3270
+epoch 004 | loss 0.571 | ppl 1.49 | wps 14206.7 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 15704 | lr 0.000252345 | gnorm 0.79 | train_wall 802 | gb_free 13.3 | wall 3270
+epoch 004 | loss 0.571 | ppl 1.49 | wps 14206.7 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 15704 | lr 0.000252345 | gnorm 0.79 | train_wall 802 | gb_free 13.3 | wall 3270
+Start iterating over samples
+begin validation on "valid" subset
+epoch 005 | valid on 'valid' subset | loss 0.496 | ppl 1.41 | wps 43747.4 | wpb 2858.9 | bsz 108.4 | num_updates 19630 | best_loss 0.496
+epoch 005 | valid on 'valid' subset | loss 0.496 | ppl 1.41 | wps 43747.4 | wpb 2858.9 | bsz 108.4 | num_updates 19630 | best_loss 0.496
+epoch 005 | valid on 'valid' subset | loss 0.496 | ppl 1.41 | wps 43747.4 | wpb 2858.9 | bsz 108.4 | num_updates 19630 | best_loss 0.496
+epoch 005 | valid on 'valid' subset | loss 0.496 | ppl 1.41 | wps 43747.4 | wpb 2858.9 | bsz 108.4 | num_updates 19630 | best_loss 0.496
+epoch 005 | valid on 'valid' subset | loss 0.496 | ppl 1.41 | wps 43747.4 | wpb 2858.9 | bsz 108.4 | num_updates 19630 | best_loss 0.496
+end of epoch 5 (average epoch stats below)
+epoch 005 | loss 0.454 | ppl 1.37 | wps 14205.6 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 19630 | lr 0.000225704 | gnorm 0.705 | train_wall 802 | gb_free 13.8 | wall 4089
+epoch 005 | loss 0.454 | ppl 1.37 | wps 14205.6 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 19630 | lr 0.000225704 | gnorm 0.705 | train_wall 802 | gb_free 13.8 | wall 4089
+epoch 005 | loss 0.454 | ppl 1.37 | wps 14205.6 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 19630 | lr 0.000225704 | gnorm 0.705 | train_wall 802 | gb_free 13.8 | wall 4089
+epoch 005 | loss 0.454 | ppl 1.37 | wps 14205.6 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 19630 | lr 0.000225704 | gnorm 0.705 | train_wall 802 | gb_free 13.8 | wall 4089
+epoch 005 | loss 0.454 | ppl 1.37 | wps 14205.6 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 19630 | lr 0.000225704 | gnorm 0.705 | train_wall 802 | gb_free 13.8 | wall 4089
+Start iterating over samples
+begin validation on "valid" subset
+epoch 006 | valid on 'valid' subset | loss 0.49 | ppl 1.4 | wps 43929.7 | wpb 2858.9 | bsz 108.4 | num_updates 23556 | best_loss 0.49
+epoch 006 | valid on 'valid' subset | loss 0.49 | ppl 1.4 | wps 43929.7 | wpb 2858.9 | bsz 108.4 | num_updates 23556 | best_loss 0.49
+epoch 006 | valid on 'valid' subset | loss 0.49 | ppl 1.4 | wps 43929.7 | wpb 2858.9 | bsz 108.4 | num_updates 23556 | best_loss 0.49
+epoch 006 | valid on 'valid' subset | loss 0.49 | ppl 1.4 | wps 43929.7 | wpb 2858.9 | bsz 108.4 | num_updates 23556 | best_loss 0.49
+epoch 006 | valid on 'valid' subset | loss 0.49 | ppl 1.4 | wps 43929.7 | wpb 2858.9 | bsz 108.4 | num_updates 23556 | best_loss 0.49
+epoch 006 | valid on 'valid' subset | loss 0.49 | ppl 1.4 | wps 43929.7 | wpb 2858.9 | bsz 108.4 | num_updates 23556 | best_loss 0.49
+end of epoch 6 (average epoch stats below)
+epoch 006 | loss 0.385 | ppl 1.31 | wps 14223.3 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 23556 | lr 0.000206039 | gnorm 0.657 | train_wall 801 | gb_free 13.7 | wall 4906
+epoch 006 | loss 0.385 | ppl 1.31 | wps 14223.3 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 23556 | lr 0.000206039 | gnorm 0.657 | train_wall 801 | gb_free 13.7 | wall 4906
+epoch 006 | loss 0.385 | ppl 1.31 | wps 14223.3 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 23556 | lr 0.000206039 | gnorm 0.657 | train_wall 801 | gb_free 13.7 | wall 4906
+epoch 006 | loss 0.385 | ppl 1.31 | wps 14223.3 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 23556 | lr 0.000206039 | gnorm 0.657 | train_wall 801 | gb_free 13.7 | wall 4906
+epoch 006 | loss 0.385 | ppl 1.31 | wps 14223.3 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 23556 | lr 0.000206039 | gnorm 0.657 | train_wall 801 | gb_free 13.7 | wall 4906
+epoch 006 | loss 0.385 | ppl 1.31 | wps 14223.3 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 23556 | lr 0.000206039 | gnorm 0.657 | train_wall 801 | gb_free 13.7 | wall 4906
+Start iterating over samples
+begin validation on "valid" subset
+epoch 007 | valid on 'valid' subset | loss 0.485 | ppl 1.4 | wps 43868.1 | wpb 2858.9 | bsz 108.4 | num_updates 27482 | best_loss 0.485
+epoch 007 | valid on 'valid' subset | loss 0.485 | ppl 1.4 | wps 43868.1 | wpb 2858.9 | bsz 108.4 | num_updates 27482 | best_loss 0.485
+epoch 007 | valid on 'valid' subset | loss 0.485 | ppl 1.4 | wps 43868.1 | wpb 2858.9 | bsz 108.4 | num_updates 27482 | best_loss 0.485
+epoch 007 | valid on 'valid' subset | loss 0.485 | ppl 1.4 | wps 43868.1 | wpb 2858.9 | bsz 108.4 | num_updates 27482 | best_loss 0.485
+epoch 007 | valid on 'valid' subset | loss 0.485 | ppl 1.4 | wps 43868.1 | wpb 2858.9 | bsz 108.4 | num_updates 27482 | best_loss 0.485
+epoch 007 | valid on 'valid' subset | loss 0.485 | ppl 1.4 | wps 43868.1 | wpb 2858.9 | bsz 108.4 | num_updates 27482 | best_loss 0.485
+epoch 007 | valid on 'valid' subset | loss 0.485 | ppl 1.4 | wps 43868.1 | wpb 2858.9 | bsz 108.4 | num_updates 27482 | best_loss 0.485
+end of epoch 7 (average epoch stats below)
+epoch 007 | loss 0.334 | ppl 1.26 | wps 14209.5 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 27482 | lr 0.000190755 | gnorm 0.637 | train_wall 802 | gb_free 13.6 | wall 5725
+epoch 007 | loss 0.334 | ppl 1.26 | wps 14209.5 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 27482 | lr 0.000190755 | gnorm 0.637 | train_wall 802 | gb_free 13.6 | wall 5725
+epoch 007 | loss 0.334 | ppl 1.26 | wps 14209.5 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 27482 | lr 0.000190755 | gnorm 0.637 | train_wall 802 | gb_free 13.6 | wall 5725
+epoch 007 | loss 0.334 | ppl 1.26 | wps 14209.5 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 27482 | lr 0.000190755 | gnorm 0.637 | train_wall 802 | gb_free 13.6 | wall 5725
+epoch 007 | loss 0.334 | ppl 1.26 | wps 14209.5 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 27482 | lr 0.000190755 | gnorm 0.637 | train_wall 802 | gb_free 13.6 | wall 5725
+epoch 007 | loss 0.334 | ppl 1.26 | wps 14209.5 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 27482 | lr 0.000190755 | gnorm 0.637 | train_wall 802 | gb_free 13.6 | wall 5725
+epoch 007 | loss 0.334 | ppl 1.26 | wps 14209.5 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 27482 | lr 0.000190755 | gnorm 0.637 | train_wall 802 | gb_free 13.6 | wall 5725
+Start iterating over samples
+begin validation on "valid" subset
+epoch 008 | valid on 'valid' subset | loss 0.503 | ppl 1.42 | wps 44034.7 | wpb 2858.9 | bsz 108.4 | num_updates 31408 | best_loss 0.485
+epoch 008 | valid on 'valid' subset | loss 0.503 | ppl 1.42 | wps 44034.7 | wpb 2858.9 | bsz 108.4 | num_updates 31408 | best_loss 0.485
+epoch 008 | valid on 'valid' subset | loss 0.503 | ppl 1.42 | wps 44034.7 | wpb 2858.9 | bsz 108.4 | num_updates 31408 | best_loss 0.485
+epoch 008 | valid on 'valid' subset | loss 0.503 | ppl 1.42 | wps 44034.7 | wpb 2858.9 | bsz 108.4 | num_updates 31408 | best_loss 0.485
+epoch 008 | valid on 'valid' subset | loss 0.503 | ppl 1.42 | wps 44034.7 | wpb 2858.9 | bsz 108.4 | num_updates 31408 | best_loss 0.485
+epoch 008 | valid on 'valid' subset | loss 0.503 | ppl 1.42 | wps 44034.7 | wpb 2858.9 | bsz 108.4 | num_updates 31408 | best_loss 0.485
+epoch 008 | valid on 'valid' subset | loss 0.503 | ppl 1.42 | wps 44034.7 | wpb 2858.9 | bsz 108.4 | num_updates 31408 | best_loss 0.485
+epoch 008 | valid on 'valid' subset | loss 0.503 | ppl 1.42 | wps 44034.7 | wpb 2858.9 | bsz 108.4 | num_updates 31408 | best_loss 0.485
+end of epoch 8 (average epoch stats below)
+epoch 008 | loss 0.291 | ppl 1.22 | wps 14209.8 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 31408 | lr 0.000178435 | gnorm 0.627 | train_wall 802 | gb_free 13.5 | wall 6543
+epoch 008 | loss 0.291 | ppl 1.22 | wps 14209.8 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 31408 | lr 0.000178435 | gnorm 0.627 | train_wall 802 | gb_free 13.5 | wall 6543
+epoch 008 | loss 0.291 | ppl 1.22 | wps 14209.8 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 31408 | lr 0.000178435 | gnorm 0.627 | train_wall 802 | gb_free 13.5 | wall 6543
+epoch 008 | loss 0.291 | ppl 1.22 | wps 14209.8 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 31408 | lr 0.000178435 | gnorm 0.627 | train_wall 802 | gb_free 13.5 | wall 6543
+epoch 008 | loss 0.291 | ppl 1.22 | wps 14209.8 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 31408 | lr 0.000178435 | gnorm 0.627 | train_wall 802 | gb_free 13.5 | wall 6543
+epoch 008 | loss 0.291 | ppl 1.22 | wps 14209.8 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 31408 | lr 0.000178435 | gnorm 0.627 | train_wall 802 | gb_free 13.5 | wall 6543
+epoch 008 | loss 0.291 | ppl 1.22 | wps 14209.8 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 31408 | lr 0.000178435 | gnorm 0.627 | train_wall 802 | gb_free 13.5 | wall 6543
+epoch 008 | loss 0.291 | ppl 1.22 | wps 14209.8 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 31408 | lr 0.000178435 | gnorm 0.627 | train_wall 802 | gb_free 13.5 | wall 6543
+Start iterating over samples
+begin validation on "valid" subset
+epoch 009 | valid on 'valid' subset | loss 0.541 | ppl 1.46 | wps 43993.1 | wpb 2858.9 | bsz 108.4 | num_updates 35334 | best_loss 0.485
+epoch 009 | valid on 'valid' subset | loss 0.541 | ppl 1.46 | wps 43993.1 | wpb 2858.9 | bsz 108.4 | num_updates 35334 | best_loss 0.485
+epoch 009 | valid on 'valid' subset | loss 0.541 | ppl 1.46 | wps 43993.1 | wpb 2858.9 | bsz 108.4 | num_updates 35334 | best_loss 0.485
+epoch 009 | valid on 'valid' subset | loss 0.541 | ppl 1.46 | wps 43993.1 | wpb 2858.9 | bsz 108.4 | num_updates 35334 | best_loss 0.485
+epoch 009 | valid on 'valid' subset | loss 0.541 | ppl 1.46 | wps 43993.1 | wpb 2858.9 | bsz 108.4 | num_updates 35334 | best_loss 0.485
+epoch 009 | valid on 'valid' subset | loss 0.541 | ppl 1.46 | wps 43993.1 | wpb 2858.9 | bsz 108.4 | num_updates 35334 | best_loss 0.485
+epoch 009 | valid on 'valid' subset | loss 0.541 | ppl 1.46 | wps 43993.1 | wpb 2858.9 | bsz 108.4 | num_updates 35334 | best_loss 0.485
+epoch 009 | valid on 'valid' subset | loss 0.541 | ppl 1.46 | wps 43993.1 | wpb 2858.9 | bsz 108.4 | num_updates 35334 | best_loss 0.485
+epoch 009 | valid on 'valid' subset | loss 0.541 | ppl 1.46 | wps 43993.1 | wpb 2858.9 | bsz 108.4 | num_updates 35334 | best_loss 0.485
+end of epoch 9 (average epoch stats below)
+epoch 009 | loss 0.254 | ppl 1.19 | wps 14221.9 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 35334 | lr 0.00016823 | gnorm 0.626 | train_wall 802 | gb_free 13.7 | wall 7361
+epoch 009 | loss 0.254 | ppl 1.19 | wps 14221.9 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 35334 | lr 0.00016823 | gnorm 0.626 | train_wall 802 | gb_free 13.7 | wall 7361
+epoch 009 | loss 0.254 | ppl 1.19 | wps 14221.9 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 35334 | lr 0.00016823 | gnorm 0.626 | train_wall 802 | gb_free 13.7 | wall 7361
+epoch 009 | loss 0.254 | ppl 1.19 | wps 14221.9 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 35334 | lr 0.00016823 | gnorm 0.626 | train_wall 802 | gb_free 13.7 | wall 7361
+epoch 009 | loss 0.254 | ppl 1.19 | wps 14221.9 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 35334 | lr 0.00016823 | gnorm 0.626 | train_wall 802 | gb_free 13.7 | wall 7361
+epoch 009 | loss 0.254 | ppl 1.19 | wps 14221.9 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 35334 | lr 0.00016823 | gnorm 0.626 | train_wall 802 | gb_free 13.7 | wall 7361
+epoch 009 | loss 0.254 | ppl 1.19 | wps 14221.9 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 35334 | lr 0.00016823 | gnorm 0.626 | train_wall 802 | gb_free 13.7 | wall 7361
+epoch 009 | loss 0.254 | ppl 1.19 | wps 14221.9 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 35334 | lr 0.00016823 | gnorm 0.626 | train_wall 802 | gb_free 13.7 | wall 7361
+epoch 009 | loss 0.254 | ppl 1.19 | wps 14221.9 | ups 4.8 | wpb 2961.7 | bsz 110.3 | num_updates 35334 | lr 0.00016823 | gnorm 0.626 | train_wall 802 | gb_free 13.7 | wall 7361
+Start iterating over samples
+begin validation on "valid" subset
+epoch 010 | valid on 'valid' subset | loss 0.57 | ppl 1.48 | wps 43885.5 | wpb 2858.9 | bsz 108.4 | num_updates 39260 | best_loss 0.485
+epoch 010 | valid on 'valid' subset | loss 0.57 | ppl 1.48 | wps 43885.5 | wpb 2858.9 | bsz 108.4 | num_updates 39260 | best_loss 0.485
+epoch 010 | valid on 'valid' subset | loss 0.57 | ppl 1.48 | wps 43885.5 | wpb 2858.9 | bsz 108.4 | num_updates 39260 | best_loss 0.485
+epoch 010 | valid on 'valid' subset | loss 0.57 | ppl 1.48 | wps 43885.5 | wpb 2858.9 | bsz 108.4 | num_updates 39260 | best_loss 0.485
+epoch 010 | valid on 'valid' subset | loss 0.57 | ppl 1.48 | wps 43885.5 | wpb 2858.9 | bsz 108.4 | num_updates 39260 | best_loss 0.485
+epoch 010 | valid on 'valid' subset | loss 0.57 | ppl 1.48 | wps 43885.5 | wpb 2858.9 | bsz 108.4 | num_updates 39260 | best_loss 0.485
+epoch 010 | valid on 'valid' subset | loss 0.57 | ppl 1.48 | wps 43885.5 | wpb 2858.9 | bsz 108.4 | num_updates 39260 | best_loss 0.485
+epoch 010 | valid on 'valid' subset | loss 0.57 | ppl 1.48 | wps 43885.5 | wpb 2858.9 | bsz 108.4 | num_updates 39260 | best_loss 0.485
+epoch 010 | valid on 'valid' subset | loss 0.57 | ppl 1.48 | wps 43885.5 | wpb 2858.9 | bsz 108.4 | num_updates 39260 | best_loss 0.485
+epoch 010 | valid on 'valid' subset | loss 0.57 | ppl 1.48 | wps 43885.5 | wpb 2858.9 | bsz 108.4 | num_updates 39260 | best_loss 0.485
+end of epoch 10 (average epoch stats below)
+epoch 010 | loss 0.22 | ppl 1.16 | wps 14234.6 | ups 4.81 | wpb 2961.7 | bsz 110.3 | num_updates 39260 | lr 0.000159597 | gnorm 0.62 | train_wall 801 | gb_free 13.7 | wall 8177
+epoch 010 | loss 0.22 | ppl 1.16 | wps 14234.6 | ups 4.81 | wpb 2961.7 | bsz 110.3 | num_updates 39260 | lr 0.000159597 | gnorm 0.62 | train_wall 801 | gb_free 13.7 | wall 8177
+epoch 010 | loss 0.22 | ppl 1.16 | wps 14234.6 | ups 4.81 | wpb 2961.7 | bsz 110.3 | num_updates 39260 | lr 0.000159597 | gnorm 0.62 | train_wall 801 | gb_free 13.7 | wall 8177
+epoch 010 | loss 0.22 | ppl 1.16 | wps 14234.6 | ups 4.81 | wpb 2961.7 | bsz 110.3 | num_updates 39260 | lr 0.000159597 | gnorm 0.62 | train_wall 801 | gb_free 13.7 | wall 8177
+epoch 010 | loss 0.22 | ppl 1.16 | wps 14234.6 | ups 4.81 | wpb 2961.7 | bsz 110.3 | num_updates 39260 | lr 0.000159597 | gnorm 0.62 | train_wall 801 | gb_free 13.7 | wall 8177
+epoch 010 | loss 0.22 | ppl 1.16 | wps 14234.6 | ups 4.81 | wpb 2961.7 | bsz 110.3 | num_updates 39260 | lr 0.000159597 | gnorm 0.62 | train_wall 801 | gb_free 13.7 | wall 8177
+epoch 010 | loss 0.22 | ppl 1.16 | wps 14234.6 | ups 4.81 | wpb 2961.7 | bsz 110.3 | num_updates 39260 | lr 0.000159597 | gnorm 0.62 | train_wall 801 | gb_free 13.7 | wall 8177
+epoch 010 | loss 0.22 | ppl 1.16 | wps 14234.6 | ups 4.81 | wpb 2961.7 | bsz 110.3 | num_updates 39260 | lr 0.000159597 | gnorm 0.62 | train_wall 801 | gb_free 13.7 | wall 8177
+epoch 010 | loss 0.22 | ppl 1.16 | wps 14234.6 | ups 4.81 | wpb 2961.7 | bsz 110.3 | num_updates 39260 | lr 0.000159597 | gnorm 0.62 | train_wall 801 | gb_free 13.7 | wall 8177
+epoch 010 | loss 0.22 | ppl 1.16 | wps 14234.6 | ups 4.81 | wpb 2961.7 | bsz 110.3 | num_updates 39260 | lr 0.000159597 | gnorm 0.62 | train_wall 801 | gb_free 13.7 | wall 8177
+Start iterating over samples
+begin validation on "valid" subset
+epoch 011 | valid on 'valid' subset | loss 0.598 | ppl 1.51 | wps 44069.5 | wpb 2858.9 | bsz 108.4 | num_updates 43186 | best_loss 0.485
+epoch 011 | valid on 'valid' subset | loss 0.598 | ppl 1.51 | wps 44069.5 | wpb 2858.9 | bsz 108.4 | num_updates 43186 | best_loss 0.485
+epoch 011 | valid on 'valid' subset | loss 0.598 | ppl 1.51 | wps 44069.5 | wpb 2858.9 | bsz 108.4 | num_updates 43186 | best_loss 0.485
+epoch 011 | valid on 'valid' subset | loss 0.598 | ppl 1.51 | wps 44069.5 | wpb 2858.9 | bsz 108.4 | num_updates 43186 | best_loss 0.485
+epoch 011 | valid on 'valid' subset | loss 0.598 | ppl 1.51 | wps 44069.5 | wpb 2858.9 | bsz 108.4 | num_updates 43186 | best_loss 0.485
+epoch 011 | valid on 'valid' subset | loss 0.598 | ppl 1.51 | wps 44069.5 | wpb 2858.9 | bsz 108.4 | num_updates 43186 | best_loss 0.485
+epoch 011 | valid on 'valid' subset | loss 0.598 | ppl 1.51 | wps 44069.5 | wpb 2858.9 | bsz 108.4 | num_updates 43186 | best_loss 0.485
+epoch 011 | valid on 'valid' subset | loss 0.598 | ppl 1.51 | wps 44069.5 | wpb 2858.9 | bsz 108.4 | num_updates 43186 | best_loss 0.485
+epoch 011 | valid on 'valid' subset | loss 0.598 | ppl 1.51 | wps 44069.5 | wpb 2858.9 | bsz 108.4 | num_updates 43186 | best_loss 0.485
+epoch 011 | valid on 'valid' subset | loss 0.598 | ppl 1.51 | wps 44069.5 | wpb 2858.9 | bsz 108.4 | num_updates 43186 | best_loss 0.485
+epoch 011 | valid on 'valid' subset | loss 0.598 | ppl 1.51 | wps 44069.5 | wpb 2858.9 | bsz 108.4 | num_updates 43186 | best_loss 0.485
+early stop since valid performance hasn't improved for last 4 runs
+end of epoch 11 (average epoch stats below)
+epoch 011 | loss 0.19 | ppl 1.14 | wps 14240.1 | ups 4.81 | wpb 2961.7 | bsz 110.3 | num_updates 43186 | lr 0.00015217 | gnorm 0.623 | train_wall 801 | gb_free 13.7 | wall 8994
+epoch 011 | loss 0.19 | ppl 1.14 | wps 14240.1 | ups 4.81 | wpb 2961.7 | bsz 110.3 | num_updates 43186 | lr 0.00015217 | gnorm 0.623 | train_wall 801 | gb_free 13.7 | wall 8994
+epoch 011 | loss 0.19 | ppl 1.14 | wps 14240.1 | ups 4.81 | wpb 2961.7 | bsz 110.3 | num_updates 43186 | lr 0.00015217 | gnorm 0.623 | train_wall 801 | gb_free 13.7 | wall 8994
+epoch 011 | loss 0.19 | ppl 1.14 | wps 14240.1 | ups 4.81 | wpb 2961.7 | bsz 110.3 | num_updates 43186 | lr 0.00015217 | gnorm 0.623 | train_wall 801 | gb_free 13.7 | wall 8994
+epoch 011 | loss 0.19 | ppl 1.14 | wps 14240.1 | ups 4.81 | wpb 2961.7 | bsz 110.3 | num_updates 43186 | lr 0.00015217 | gnorm 0.623 | train_wall 801 | gb_free 13.7 | wall 8994
+epoch 011 | loss 0.19 | ppl 1.14 | wps 14240.1 | ups 4.81 | wpb 2961.7 | bsz 110.3 | num_updates 43186 | lr 0.00015217 | gnorm 0.623 | train_wall 801 | gb_free 13.7 | wall 8994
+epoch 011 | loss 0.19 | ppl 1.14 | wps 14240.1 | ups 4.81 | wpb 2961.7 | bsz 110.3 | num_updates 43186 | lr 0.00015217 | gnorm 0.623 | train_wall 801 | gb_free 13.7 | wall 8994
+epoch 011 | loss 0.19 | ppl 1.14 | wps 14240.1 | ups 4.81 | wpb 2961.7 | bsz 110.3 | num_updates 43186 | lr 0.00015217 | gnorm 0.623 | train_wall 801 | gb_free 13.7 | wall 8994
+epoch 011 | loss 0.19 | ppl 1.14 | wps 14240.1 | ups 4.81 | wpb 2961.7 | bsz 110.3 | num_updates 43186 | lr 0.00015217 | gnorm 0.623 | train_wall 801 | gb_free 13.7 | wall 8994
+epoch 011 | loss 0.19 | ppl 1.14 | wps 14240.1 | ups 4.81 | wpb 2961.7 | bsz 110.3 | num_updates 43186 | lr 0.00015217 | gnorm 0.623 | train_wall 801 | gb_free 13.7 | wall 8994
+epoch 011 | loss 0.19 | ppl 1.14 | wps 14240.1 | ups 4.81 | wpb 2961.7 | bsz 110.3 | num_updates 43186 | lr 0.00015217 | gnorm 0.623 | train_wall 801 | gb_free 13.7 | wall 8994
+done training in 8993.5 seconds

ablations/noconst/trained_x64_best_noconst/beam1.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0f7b91663e22fd7f7950507091546a1df281e3a31fb8944eeaeb8e1859b835c
+size 3801344

ablations/noconst/trained_x64_best_noconst/checkpoint_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82e62bf86ed73833a72344216deda861a86ae321a9485edb6b583591e2df2602
+size 185999604

ablations/noconst/trained_x64_best_noconst/training.log

@@ -0,0 +1,142 @@
+{'_name': None, 'common': {'_name': None, 'no_progress_bar': False, 'log_interval': 100, 'log_format': None, 'log_file': 'x64/trained_x64_best_noconst/training.log', 'aim_repo': None, 'aim_run_hash': None, 'tensorboard_logdir': None, 'wandb_project': None, 'azureml_logging': False, 'seed': 1, 'cpu': False, 'tpu': False, 'bf16': False, 'memory_efficient_bf16': False, 'fp16': False, 'memory_efficient_fp16': False, 'fp16_no_flatten_grads': False, 'fp16_init_scale': 128, 'fp16_scale_window': None, 'fp16_scale_tolerance': 0.0, 'on_cpu_convert_precision': False, 'min_loss_scale': 0.0001, 'threshold_loss_scale': None, 'amp': False, 'amp_batch_retries': 2, 'amp_init_scale': 128, 'amp_scale_window': None, 'user_dir': None, 'empty_cache_freq': 0, 'all_gather_list_size': 16384, 'model_parallel_size': 1, 'quantization_config_path': None, 'profile': False, 'reset_logging': False, 'suppress_crashes': False, 'use_plasma_view': False, 'plasma_path': '/tmp/plasma'}, 'common_eval': {'_name': None, 'path': None, 'post_process': None, 'quiet': False, 'model_overrides': '{}', 'results_path': None}, 'distributed_training': {'_name': None, 'distributed_world_size': 1, 'distributed_num_procs': 1, 'distributed_rank': 0, 'distributed_backend': 'nccl', 'distributed_init_method': None, 'distributed_port': -1, 'device_id': 0, 'distributed_no_spawn': False, 'ddp_backend': 'pytorch_ddp', 'ddp_comm_hook': 'none', 'bucket_cap_mb': 25, 'fix_batches_to_gpus': False, 'find_unused_parameters': False, 'gradient_as_bucket_view': False, 'fast_stat_sync': False, 'heartbeat_timeout': -1, 'broadcast_buffers': False, 'slowmo_momentum': None, 'slowmo_base_algorithm': 'localsgd', 'localsgd_frequency': 3, 'nprocs_per_node': 1, 'pipeline_model_parallel': False, 'pipeline_balance': None, 'pipeline_devices': None, 'pipeline_chunks': 0, 'pipeline_encoder_balance': None, 'pipeline_encoder_devices': None, 'pipeline_decoder_balance': None, 'pipeline_decoder_devices': None, 'pipeline_checkpoint': 'never', 'zero_sharding': 'none', 'fp16': False, 'memory_efficient_fp16': False, 'tpu': False, 'no_reshard_after_forward': False, 'fp32_reduce_scatter': False, 'cpu_offload': False, 'use_sharded_state': False, 'not_fsdp_flatten_parameters': False}, 'dataset': {'_name': None, 'num_workers': 1, 'skip_invalid_size_inputs_valid_test': False, 'max_tokens': 20000, 'batch_size': None, 'required_batch_size_multiple': 8, 'required_seq_len_multiple': 1, 'dataset_impl': None, 'data_buffer_size': 10, 'train_subset': 'train', 'valid_subset': 'valid', 'combine_valid_subsets': None, 'ignore_unused_valid_subsets': False, 'validate_interval': 1, 'validate_interval_updates': 0, 'validate_after_updates': 0, 'fixed_validation_seed': None, 'disable_validation': False, 'max_tokens_valid': 20000, 'batch_size_valid': None, 'max_valid_steps': None, 'curriculum': 0, 'gen_subset': 'test', 'num_shards': 1, 'shard_id': 0, 'grouped_shuffling': False, 'update_epoch_batch_itr': False, 'update_ordered_indices_seed': False}, 'optimization': {'_name': None, 'max_epoch': 0, 'max_update': 100000, 'stop_time_hours': 0.0, 'clip_norm': 0.0, 'sentence_avg': False, 'update_freq': [1], 'lr': [0.0005], 'stop_min_lr': -1.0, 'use_bmuf': False, 'skip_remainder_batch': False}, 'checkpoint': {'_name': None, 'save_dir': 'x64/trained_x64_best_noconst', 'restore_file': 'checkpoint_last.pt', 'continue_once': None, 'finetune_from_model': None, 'reset_dataloader': False, 'reset_lr_scheduler': False, 'reset_meters': False, 'reset_optimizer': False, 'optimizer_overrides': '{}', 'save_interval': 1, 'save_interval_updates': 0, 'keep_interval_updates': -1, 'keep_interval_updates_pattern': -1, 'keep_last_epochs': -1, 'keep_best_checkpoints': 3, 'no_save': False, 'no_epoch_checkpoints': True, 'no_last_checkpoints': False, 'no_save_optimizer_state': False, 'best_checkpoint_metric': 'loss', 'maximize_best_checkpoint_metric': False, 'patience': 4, 'checkpoint_suffix': '', 'checkpoint_shard_count': 1, 'load_checkpoint_on_all_dp_ranks': False, 'write_checkpoints_asynchronously': False, 'model_parallel_size': 1}, 'bmuf': {'_name': None, 'block_lr': 1.0, 'block_momentum': 0.875, 'global_sync_iter': 50, 'warmup_iterations': 500, 'use_nbm': False, 'average_sync': False, 'distributed_world_size': 1}, 'generation': {'_name': None, 'beam': 5, 'nbest': 1, 'max_len_a': 0.0, 'max_len_b': 200, 'min_len': 1, 'match_source_len': False, 'unnormalized': False, 'no_early_stop': False, 'no_beamable_mm': False, 'lenpen': 1.0, 'unkpen': 0.0, 'replace_unk': None, 'sacrebleu': False, 'score_reference': False, 'prefix_size': 0, 'no_repeat_ngram_size': 0, 'sampling': False, 'sampling_topk': -1, 'sampling_topp': -1.0, 'constraints': None, 'temperature': 1.0, 'diverse_beam_groups': -1, 'diverse_beam_strength': 0.5, 'diversity_rate': -1.0, 'print_alignment': None, 'print_step': False, 'lm_path': None, 'lm_weight': 0.0, 'iter_decode_eos_penalty': 0.0, 'iter_decode_max_iter': 10, 'iter_decode_force_max_iter': False, 'iter_decode_with_beam': 1, 'iter_decode_with_external_reranker': False, 'retain_iter_history': False, 'retain_dropout': False, 'retain_dropout_modules': None, 'decoding_format': None, 'no_seed_provided': False, 'eos_token': None}, 'eval_lm': {'_name': None, 'output_word_probs': False, 'output_word_stats': False, 'context_window': 0, 'softmax_batch': 9223372036854775807}, 'interactive': {'_name': None, 'buffer_size': 0, 'input': '-'}, 'model': Namespace(_name='transformer', activation_dropout=0.0, activation_fn='relu', adam_betas=(0.9, 0.999), adam_eps=1e-08, adaptive_input=False, adaptive_softmax_cutoff=None, adaptive_softmax_dropout=0, aim_repo=None, aim_run_hash=None, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, arch='transformer', attention_dropout=0.0, azureml_logging=False, batch_size=None, batch_size_valid=None, best_checkpoint_metric='loss', bf16=False, bpe=None, broadcast_buffers=False, bucket_cap_mb=25, checkpoint_activations=False, checkpoint_shard_count=1, checkpoint_suffix='', clip_norm=0.0, combine_valid_subsets=None, continue_once=None, cpu=False, cpu_offload=False, criterion='cross_entropy', cross_self_attention=False, curriculum=0, data='x64/tokenized_dlsm_x64_noconst', data_buffer_size=10, dataset_impl=None, ddp_backend='pytorch_ddp', ddp_comm_hook='none', decoder_attention_heads=8, decoder_embed_dim=96, decoder_embed_path=None, decoder_ffn_embed_dim=384, decoder_input_dim=96, decoder_layerdrop=0, decoder_layers=6, decoder_layers_to_keep=None, decoder_learned_pos=False, decoder_normalize_before=False, decoder_output_dim='96', device_id=0, disable_validation=False, distributed_backend='nccl', distributed_init_method=None, distributed_no_spawn=False, distributed_num_procs=1, distributed_port=-1, distributed_rank=0, distributed_world_size=1, dropout=0.05, ema_decay=0.9999, ema_fp32=False, ema_seed_model=None, ema_start_update=0, ema_update_freq=1, empty_cache_freq=0, encoder_attention_heads=8, encoder_embed_dim=384, encoder_embed_path=None, encoder_ffn_embed_dim=1536, encoder_layerdrop=0, encoder_layers=6, encoder_layers_to_keep=None, encoder_learned_pos=False, encoder_normalize_before=False, eos=2, eval_bleu=False, eval_bleu_args='{}', eval_bleu_detok='space', eval_bleu_detok_args='{}', eval_bleu_print_samples=False, eval_bleu_remove_bpe=None, eval_tokenized_bleu=False, fast_stat_sync=False, find_unused_parameters=False, finetune_from_model=None, fix_batches_to_gpus=False, fixed_validation_seed=None, fp16=False, fp16_adam_stats=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, fp32_reduce_scatter=False, gen_subset='test', gradient_as_bucket_view=False, grouped_shuffling=False, heartbeat_timeout=-1, ignore_unused_valid_subsets=False, keep_best_checkpoints=3, keep_interval_updates=-1, keep_interval_updates_pattern=-1, keep_last_epochs=-1, layernorm_embedding=False, left_pad_source=True, left_pad_target=False, load_alignments=False, load_checkpoint_on_all_dp_ranks=False, localsgd_frequency=3, log_file='x64/trained_x64_best_noconst/training.log', log_format=None, log_interval=100, lr=[0.0005], lr_scheduler='inverse_sqrt', max_epoch=0, max_source_positions=1024, max_target_positions=1024, max_tokens=20000, max_tokens_valid=20000, max_update=100000, max_valid_steps=None, maximize_best_checkpoint_metric=False, memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_cross_attention=False, no_epoch_checkpoints=True, no_last_checkpoints=False, no_progress_bar=False, no_reshard_after_forward=False, no_save=False, no_save_optimizer_state=False, no_scale_embedding=False, no_seed_provided=False, no_token_positional_embeddings=False, not_fsdp_flatten_parameters=False, nprocs_per_node=1, num_batch_buckets=0, num_shards=1, num_workers=1, offload_activations=False, on_cpu_convert_precision=False, optimizer='adam', optimizer_overrides='{}', pad=1, patience=4, pipeline_balance=None, pipeline_checkpoint='never', pipeline_chunks=0, pipeline_decoder_balance=None, pipeline_decoder_devices=None, pipeline_devices=None, pipeline_encoder_balance=None, pipeline_encoder_devices=None, pipeline_model_parallel=False, plasma_path='/tmp/plasma', profile=False, quant_noise_pq=0, quant_noise_pq_block_size=8, quant_noise_scalar=0, quantization_config_path=None, required_batch_size_multiple=8, required_seq_len_multiple=1, reset_dataloader=False, reset_logging=False, reset_lr_scheduler=False, reset_meters=False, reset_optimizer=False, restore_file='checkpoint_last.pt', save_dir='x64/trained_x64_best_noconst', save_interval=1, save_interval_updates=0, scoring='bleu', seed=1, sentence_avg=False, shard_id=0, share_all_embeddings=False, share_decoder_input_output_embed=False, skip_invalid_size_inputs_valid_test=False, skip_remainder_batch=False, slowmo_base_algorithm='localsgd', slowmo_momentum=None, source_lang=None, stop_min_lr=-1.0, stop_time_hours=0, store_ema=False, suppress_crashes=False, target_lang=None, task='translation', tensorboard_logdir=None, threshold_loss_scale=None, tie_adaptive_weights=False, tokenizer=None, tpu=False, train_subset='train', truncate_source=False, unk=3, update_epoch_batch_itr=False, update_freq=[1], update_ordered_indices_seed=False, upsample_primary=-1, use_bmuf=False, use_old_adam=False, use_plasma_view=False, use_sharded_state=False, user_dir=None, valid_subset='valid', validate_after_updates=0, validate_interval=1, validate_interval_updates=0, wandb_project=None, warmup_init_lr=-1, warmup_updates=4000, weight_decay=0.005, write_checkpoints_asynchronously=False, zero_sharding='none'), 'task': {'_name': 'translation', 'data': 'x64/tokenized_dlsm_x64_noconst', 'source_lang': None, 'target_lang': None, 'load_alignments': False, 'left_pad_source': True, 'left_pad_target': False, 'max_source_positions': 1024, 'max_target_positions': 1024, 'upsample_primary': -1, 'truncate_source': False, 'num_batch_buckets': 0, 'train_subset': 'train', 'dataset_impl': None, 'required_seq_len_multiple': 1, 'eval_bleu': False, 'eval_bleu_args': '{}', 'eval_bleu_detok': 'space', 'eval_bleu_detok_args': '{}', 'eval_tokenized_bleu': False, 'eval_bleu_remove_bpe': None, 'eval_bleu_print_samples': False}, 'criterion': {'_name': 'cross_entropy', 'sentence_avg': False}, 'optimizer': {'_name': 'adam', 'adam_betas': [0.9, 0.999], 'adam_eps': 1e-08, 'weight_decay': 0.005, 'use_old_adam': False, 'fp16_adam_stats': False, 'tpu': False, 'lr': [0.0005]}, 'lr_scheduler': {'_name': 'inverse_sqrt', 'warmup_updates': 4000, 'warmup_init_lr': -1.0, 'lr': [0.0005]}, 'scoring': {'_name': 'bleu', 'pad': 1, 'eos': 2, 'unk': 3}, 'bpe': None, 'tokenizer': None, 'ema': {'_name': None, 'store_ema': False, 'ema_decay': 0.9999, 'ema_start_update': 0, 'ema_seed_model': None, 'ema_update_freq': 1, 'ema_fp32': False}}
+TransformerModel(
+  (encoder): TransformerEncoderBase(
+    (dropout_module): FairseqDropout()
+    (embed_tokens): Embedding(128, 384, padding_idx=1)
+    (embed_positions): SinusoidalPositionalEmbedding()
+    (layers): ModuleList(
+      (0-5): 6 x TransformerEncoderLayerBase(
+        (self_attn): MultiheadAttention(
+          (dropout_module): FairseqDropout()
+          (k_proj): Linear(in_features=384, out_features=384, bias=True)
+          (v_proj): Linear(in_features=384, out_features=384, bias=True)
+          (q_proj): Linear(in_features=384, out_features=384, bias=True)
+          (out_proj): Linear(in_features=384, out_features=384, bias=True)
+        )
+        (self_attn_layer_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+        (dropout_module): FairseqDropout()
+        (activation_dropout_module): FairseqDropout()
+        (fc1): Linear(in_features=384, out_features=1536, bias=True)
+        (fc2): Linear(in_features=1536, out_features=384, bias=True)
+        (final_layer_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+      )
+    )
+  )
+  (decoder): TransformerDecoderBase(
+    (dropout_module): FairseqDropout()
+    (embed_tokens): Embedding(40, 96, padding_idx=1)
+    (embed_positions): SinusoidalPositionalEmbedding()
+    (layers): ModuleList(
+      (0-5): 6 x TransformerDecoderLayerBase(
+        (dropout_module): FairseqDropout()
+        (self_attn): MultiheadAttention(
+          (dropout_module): FairseqDropout()
+          (k_proj): Linear(in_features=96, out_features=96, bias=True)
+          (v_proj): Linear(in_features=96, out_features=96, bias=True)
+          (q_proj): Linear(in_features=96, out_features=96, bias=True)
+          (out_proj): Linear(in_features=96, out_features=96, bias=True)
+        )
+        (activation_dropout_module): FairseqDropout()
+        (self_attn_layer_norm): LayerNorm((96,), eps=1e-05, elementwise_affine=True)
+        (encoder_attn): MultiheadAttention(
+          (dropout_module): FairseqDropout()
+          (k_proj): Linear(in_features=384, out_features=96, bias=True)
+          (v_proj): Linear(in_features=384, out_features=96, bias=True)
+          (q_proj): Linear(in_features=96, out_features=96, bias=True)
+          (out_proj): Linear(in_features=96, out_features=96, bias=True)
+        )
+        (encoder_attn_layer_norm): LayerNorm((96,), eps=1e-05, elementwise_affine=True)
+        (fc1): Linear(in_features=96, out_features=384, bias=True)
+        (fc2): Linear(in_features=384, out_features=96, bias=True)
+        (final_layer_norm): LayerNorm((96,), eps=1e-05, elementwise_affine=True)
+      )
+    )
+    (output_projection): Linear(in_features=96, out_features=40, bias=False)
+  )
+)
+task: TranslationTask
+model: TransformerModel
+criterion: CrossEntropyCriterion
+num. shared model params: 22,568,640 (num. trained: 22,568,640)
+num. expert model params: 0 (num. trained: 0)
+training on 1 devices (GPUs/TPUs)
+max tokens per device = 20000 and max sentences per device = None
+Start iterating over samples
+begin validation on "valid" subset
+epoch 001 | valid on 'valid' subset | loss 1.81 | ppl 3.51 | wps 27721.3 | wpb 2679.5 | bsz 97.1 | num_updates 4211
+end of epoch 1 (average epoch stats below)
+epoch 001 | loss 2.368 | ppl 5.16 | wps 9077.7 | ups 3.25 | wpb 2793.3 | bsz 100.7 | num_updates 4211 | lr 0.000487312 | gnorm 2.758 | train_wall 1275 | gb_free 13.1 | wall 1296
+Start iterating over samples
+begin validation on "valid" subset
+epoch 002 | valid on 'valid' subset | loss 1.272 | ppl 2.42 | wps 27624.5 | wpb 2679.5 | bsz 97.1 | num_updates 8422 | best_loss 1.272
+epoch 002 | valid on 'valid' subset | loss 1.272 | ppl 2.42 | wps 27624.5 | wpb 2679.5 | bsz 97.1 | num_updates 8422 | best_loss 1.272
+end of epoch 2 (average epoch stats below)
+epoch 002 | loss 1.507 | ppl 2.84 | wps 9061.9 | ups 3.24 | wpb 2793.3 | bsz 100.7 | num_updates 8422 | lr 0.000344582 | gnorm 1.357 | train_wall 1277 | gb_free 12.9 | wall 2594
+epoch 002 | loss 1.507 | ppl 2.84 | wps 9061.9 | ups 3.24 | wpb 2793.3 | bsz 100.7 | num_updates 8422 | lr 0.000344582 | gnorm 1.357 | train_wall 1277 | gb_free 12.9 | wall 2594
+Start iterating over samples
+begin validation on "valid" subset
+epoch 003 | valid on 'valid' subset | loss 1.17 | ppl 2.25 | wps 27577.4 | wpb 2679.5 | bsz 97.1 | num_updates 12633 | best_loss 1.17
+epoch 003 | valid on 'valid' subset | loss 1.17 | ppl 2.25 | wps 27577.4 | wpb 2679.5 | bsz 97.1 | num_updates 12633 | best_loss 1.17
+epoch 003 | valid on 'valid' subset | loss 1.17 | ppl 2.25 | wps 27577.4 | wpb 2679.5 | bsz 97.1 | num_updates 12633 | best_loss 1.17
+end of epoch 3 (average epoch stats below)
+epoch 003 | loss 1.155 | ppl 2.23 | wps 9060.7 | ups 3.24 | wpb 2793.3 | bsz 100.7 | num_updates 12633 | lr 0.00028135 | gnorm 1.172 | train_wall 1277 | gb_free 12.8 | wall 3893
+epoch 003 | loss 1.155 | ppl 2.23 | wps 9060.7 | ups 3.24 | wpb 2793.3 | bsz 100.7 | num_updates 12633 | lr 0.00028135 | gnorm 1.172 | train_wall 1277 | gb_free 12.8 | wall 3893
+epoch 003 | loss 1.155 | ppl 2.23 | wps 9060.7 | ups 3.24 | wpb 2793.3 | bsz 100.7 | num_updates 12633 | lr 0.00028135 | gnorm 1.172 | train_wall 1277 | gb_free 12.8 | wall 3893
+Start iterating over samples
+begin validation on "valid" subset
+epoch 004 | valid on 'valid' subset | loss 1.201 | ppl 2.3 | wps 27719.2 | wpb 2679.5 | bsz 97.1 | num_updates 16844 | best_loss 1.17
+epoch 004 | valid on 'valid' subset | loss 1.201 | ppl 2.3 | wps 27719.2 | wpb 2679.5 | bsz 97.1 | num_updates 16844 | best_loss 1.17
+epoch 004 | valid on 'valid' subset | loss 1.201 | ppl 2.3 | wps 27719.2 | wpb 2679.5 | bsz 97.1 | num_updates 16844 | best_loss 1.17
+epoch 004 | valid on 'valid' subset | loss 1.201 | ppl 2.3 | wps 27719.2 | wpb 2679.5 | bsz 97.1 | num_updates 16844 | best_loss 1.17
+end of epoch 4 (average epoch stats below)
+epoch 004 | loss 0.976 | ppl 1.97 | wps 9063.1 | ups 3.24 | wpb 2793.3 | bsz 100.7 | num_updates 16844 | lr 0.000243656 | gnorm 1.093 | train_wall 1277 | gb_free 13.1 | wall 5190
+epoch 004 | loss 0.976 | ppl 1.97 | wps 9063.1 | ups 3.24 | wpb 2793.3 | bsz 100.7 | num_updates 16844 | lr 0.000243656 | gnorm 1.093 | train_wall 1277 | gb_free 13.1 | wall 5190
+epoch 004 | loss 0.976 | ppl 1.97 | wps 9063.1 | ups 3.24 | wpb 2793.3 | bsz 100.7 | num_updates 16844 | lr 0.000243656 | gnorm 1.093 | train_wall 1277 | gb_free 13.1 | wall 5190
+epoch 004 | loss 0.976 | ppl 1.97 | wps 9063.1 | ups 3.24 | wpb 2793.3 | bsz 100.7 | num_updates 16844 | lr 0.000243656 | gnorm 1.093 | train_wall 1277 | gb_free 13.1 | wall 5190
+Start iterating over samples
+begin validation on "valid" subset
+epoch 005 | valid on 'valid' subset | loss 1.267 | ppl 2.41 | wps 27584.2 | wpb 2679.5 | bsz 97.1 | num_updates 21055 | best_loss 1.17
+epoch 005 | valid on 'valid' subset | loss 1.267 | ppl 2.41 | wps 27584.2 | wpb 2679.5 | bsz 97.1 | num_updates 21055 | best_loss 1.17
+epoch 005 | valid on 'valid' subset | loss 1.267 | ppl 2.41 | wps 27584.2 | wpb 2679.5 | bsz 97.1 | num_updates 21055 | best_loss 1.17
+epoch 005 | valid on 'valid' subset | loss 1.267 | ppl 2.41 | wps 27584.2 | wpb 2679.5 | bsz 97.1 | num_updates 21055 | best_loss 1.17
+epoch 005 | valid on 'valid' subset | loss 1.267 | ppl 2.41 | wps 27584.2 | wpb 2679.5 | bsz 97.1 | num_updates 21055 | best_loss 1.17
+end of epoch 5 (average epoch stats below)
+epoch 005 | loss 0.847 | ppl 1.8 | wps 9051.5 | ups 3.24 | wpb 2793.3 | bsz 100.7 | num_updates 21055 | lr 0.000217933 | gnorm 1.075 | train_wall 1279 | gb_free 12.7 | wall 6490
+epoch 005 | loss 0.847 | ppl 1.8 | wps 9051.5 | ups 3.24 | wpb 2793.3 | bsz 100.7 | num_updates 21055 | lr 0.000217933 | gnorm 1.075 | train_wall 1279 | gb_free 12.7 | wall 6490
+epoch 005 | loss 0.847 | ppl 1.8 | wps 9051.5 | ups 3.24 | wpb 2793.3 | bsz 100.7 | num_updates 21055 | lr 0.000217933 | gnorm 1.075 | train_wall 1279 | gb_free 12.7 | wall 6490
+epoch 005 | loss 0.847 | ppl 1.8 | wps 9051.5 | ups 3.24 | wpb 2793.3 | bsz 100.7 | num_updates 21055 | lr 0.000217933 | gnorm 1.075 | train_wall 1279 | gb_free 12.7 | wall 6490
+epoch 005 | loss 0.847 | ppl 1.8 | wps 9051.5 | ups 3.24 | wpb 2793.3 | bsz 100.7 | num_updates 21055 | lr 0.000217933 | gnorm 1.075 | train_wall 1279 | gb_free 12.7 | wall 6490
+Start iterating over samples
+begin validation on "valid" subset
+epoch 006 | valid on 'valid' subset | loss 1.371 | ppl 2.59 | wps 27709.9 | wpb 2679.5 | bsz 97.1 | num_updates 25266 | best_loss 1.17
+epoch 006 | valid on 'valid' subset | loss 1.371 | ppl 2.59 | wps 27709.9 | wpb 2679.5 | bsz 97.1 | num_updates 25266 | best_loss 1.17
+epoch 006 | valid on 'valid' subset | loss 1.371 | ppl 2.59 | wps 27709.9 | wpb 2679.5 | bsz 97.1 | num_updates 25266 | best_loss 1.17
+epoch 006 | valid on 'valid' subset | loss 1.371 | ppl 2.59 | wps 27709.9 | wpb 2679.5 | bsz 97.1 | num_updates 25266 | best_loss 1.17
+epoch 006 | valid on 'valid' subset | loss 1.371 | ppl 2.59 | wps 27709.9 | wpb 2679.5 | bsz 97.1 | num_updates 25266 | best_loss 1.17
+epoch 006 | valid on 'valid' subset | loss 1.371 | ppl 2.59 | wps 27709.9 | wpb 2679.5 | bsz 97.1 | num_updates 25266 | best_loss 1.17
+end of epoch 6 (average epoch stats below)
+epoch 006 | loss 0.737 | ppl 1.67 | wps 9050.9 | ups 3.24 | wpb 2793.3 | bsz 100.7 | num_updates 25266 | lr 0.000198944 | gnorm 1.103 | train_wall 1278 | gb_free 12.9 | wall 7790
+epoch 006 | loss 0.737 | ppl 1.67 | wps 9050.9 | ups 3.24 | wpb 2793.3 | bsz 100.7 | num_updates 25266 | lr 0.000198944 | gnorm 1.103 | train_wall 1278 | gb_free 12.9 | wall 7790
+epoch 006 | loss 0.737 | ppl 1.67 | wps 9050.9 | ups 3.24 | wpb 2793.3 | bsz 100.7 | num_updates 25266 | lr 0.000198944 | gnorm 1.103 | train_wall 1278 | gb_free 12.9 | wall 7790
+epoch 006 | loss 0.737 | ppl 1.67 | wps 9050.9 | ups 3.24 | wpb 2793.3 | bsz 100.7 | num_updates 25266 | lr 0.000198944 | gnorm 1.103 | train_wall 1278 | gb_free 12.9 | wall 7790
+epoch 006 | loss 0.737 | ppl 1.67 | wps 9050.9 | ups 3.24 | wpb 2793.3 | bsz 100.7 | num_updates 25266 | lr 0.000198944 | gnorm 1.103 | train_wall 1278 | gb_free 12.9 | wall 7790
+epoch 006 | loss 0.737 | ppl 1.67 | wps 9050.9 | ups 3.24 | wpb 2793.3 | bsz 100.7 | num_updates 25266 | lr 0.000198944 | gnorm 1.103 | train_wall 1278 | gb_free 12.9 | wall 7790
+Start iterating over samples
+begin validation on "valid" subset
+epoch 007 | valid on 'valid' subset | loss 1.475 | ppl 2.78 | wps 27733.8 | wpb 2679.5 | bsz 97.1 | num_updates 29477 | best_loss 1.17
+epoch 007 | valid on 'valid' subset | loss 1.475 | ppl 2.78 | wps 27733.8 | wpb 2679.5 | bsz 97.1 | num_updates 29477 | best_loss 1.17
+epoch 007 | valid on 'valid' subset | loss 1.475 | ppl 2.78 | wps 27733.8 | wpb 2679.5 | bsz 97.1 | num_updates 29477 | best_loss 1.17
+epoch 007 | valid on 'valid' subset | loss 1.475 | ppl 2.78 | wps 27733.8 | wpb 2679.5 | bsz 97.1 | num_updates 29477 | best_loss 1.17
+epoch 007 | valid on 'valid' subset | loss 1.475 | ppl 2.78 | wps 27733.8 | wpb 2679.5 | bsz 97.1 | num_updates 29477 | best_loss 1.17
+epoch 007 | valid on 'valid' subset | loss 1.475 | ppl 2.78 | wps 27733.8 | wpb 2679.5 | bsz 97.1 | num_updates 29477 | best_loss 1.17
+epoch 007 | valid on 'valid' subset | loss 1.475 | ppl 2.78 | wps 27733.8 | wpb 2679.5 | bsz 97.1 | num_updates 29477 | best_loss 1.17
+early stop since valid performance hasn't improved for last 4 runs
+end of epoch 7 (average epoch stats below)
+epoch 007 | loss 0.644 | ppl 1.56 | wps 9051.9 | ups 3.24 | wpb 2793.3 | bsz 100.7 | num_updates 29477 | lr 0.000184187 | gnorm 1.147 | train_wall 1277 | gb_free 12.9 | wall 9089
+epoch 007 | loss 0.644 | ppl 1.56 | wps 9051.9 | ups 3.24 | wpb 2793.3 | bsz 100.7 | num_updates 29477 | lr 0.000184187 | gnorm 1.147 | train_wall 1277 | gb_free 12.9 | wall 9089
+epoch 007 | loss 0.644 | ppl 1.56 | wps 9051.9 | ups 3.24 | wpb 2793.3 | bsz 100.7 | num_updates 29477 | lr 0.000184187 | gnorm 1.147 | train_wall 1277 | gb_free 12.9 | wall 9089
+epoch 007 | loss 0.644 | ppl 1.56 | wps 9051.9 | ups 3.24 | wpb 2793.3 | bsz 100.7 | num_updates 29477 | lr 0.000184187 | gnorm 1.147 | train_wall 1277 | gb_free 12.9 | wall 9089
+epoch 007 | loss 0.644 | ppl 1.56 | wps 9051.9 | ups 3.24 | wpb 2793.3 | bsz 100.7 | num_updates 29477 | lr 0.000184187 | gnorm 1.147 | train_wall 1277 | gb_free 12.9 | wall 9089
+epoch 007 | loss 0.644 | ppl 1.56 | wps 9051.9 | ups 3.24 | wpb 2793.3 | bsz 100.7 | num_updates 29477 | lr 0.000184187 | gnorm 1.147 | train_wall 1277 | gb_free 12.9 | wall 9089
+epoch 007 | loss 0.644 | ppl 1.56 | wps 9051.9 | ups 3.24 | wpb 2793.3 | bsz 100.7 | num_updates 29477 | lr 0.000184187 | gnorm 1.147 | train_wall 1277 | gb_free 12.9 | wall 9089
+done training in 9088.6 seconds

ablations/postfix/tokenized_postfix.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c91cb78de78a69a17dc9e41d0d9e1881e09a63f05a3f27e6ef65405bcaef8e2
+size 121303562

ablations/postfix/trained_aarch64_best_postfix/beam1.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d87d55da3a5b55535dadead1bf429a4dbff610bea36df3283ea40973c4cf16a
+size 2538195

ablations/postfix/trained_aarch64_best_postfix/checkpoint_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec8724cbe0491471936204fadd4d30a8382581e02534189053cf415ac101f7f5
+size 186184500

ablations/postfix/trained_aarch64_best_postfix/training.log

@@ -0,0 +1,590 @@
+{'_name': None, 'common': {'_name': None, 'no_progress_bar': False, 'log_interval': 100, 'log_format': None, 'log_file': 'aarch64/trained_aarch64_best_postfix/training.log', 'aim_repo': None, 'aim_run_hash': None, 'tensorboard_logdir': None, 'wandb_project': None, 'azureml_logging': False, 'seed': 1, 'cpu': False, 'tpu': False, 'bf16': False, 'memory_efficient_bf16': False, 'fp16': False, 'memory_efficient_fp16': False, 'fp16_no_flatten_grads': False, 'fp16_init_scale': 128, 'fp16_scale_window': None, 'fp16_scale_tolerance': 0.0, 'on_cpu_convert_precision': False, 'min_loss_scale': 0.0001, 'threshold_loss_scale': None, 'amp': False, 'amp_batch_retries': 2, 'amp_init_scale': 128, 'amp_scale_window': None, 'user_dir': None, 'empty_cache_freq': 0, 'all_gather_list_size': 16384, 'model_parallel_size': 1, 'quantization_config_path': None, 'profile': False, 'reset_logging': False, 'suppress_crashes': False, 'use_plasma_view': False, 'plasma_path': '/tmp/plasma'}, 'common_eval': {'_name': None, 'path': None, 'post_process': None, 'quiet': False, 'model_overrides': '{}', 'results_path': None}, 'distributed_training': {'_name': None, 'distributed_world_size': 1, 'distributed_num_procs': 1, 'distributed_rank': 0, 'distributed_backend': 'nccl', 'distributed_init_method': None, 'distributed_port': -1, 'device_id': 0, 'distributed_no_spawn': False, 'ddp_backend': 'pytorch_ddp', 'ddp_comm_hook': 'none', 'bucket_cap_mb': 25, 'fix_batches_to_gpus': False, 'find_unused_parameters': False, 'gradient_as_bucket_view': False, 'fast_stat_sync': False, 'heartbeat_timeout': -1, 'broadcast_buffers': False, 'slowmo_momentum': None, 'slowmo_base_algorithm': 'localsgd', 'localsgd_frequency': 3, 'nprocs_per_node': 1, 'pipeline_model_parallel': False, 'pipeline_balance': None, 'pipeline_devices': None, 'pipeline_chunks': 0, 'pipeline_encoder_balance': None, 'pipeline_encoder_devices': None, 'pipeline_decoder_balance': None, 'pipeline_decoder_devices': None, 'pipeline_checkpoint': 'never', 'zero_sharding': 'none', 'fp16': False, 'memory_efficient_fp16': False, 'tpu': False, 'no_reshard_after_forward': False, 'fp32_reduce_scatter': False, 'cpu_offload': False, 'use_sharded_state': False, 'not_fsdp_flatten_parameters': False}, 'dataset': {'_name': None, 'num_workers': 1, 'skip_invalid_size_inputs_valid_test': False, 'max_tokens': 20000, 'batch_size': None, 'required_batch_size_multiple': 8, 'required_seq_len_multiple': 1, 'dataset_impl': None, 'data_buffer_size': 10, 'train_subset': 'train', 'valid_subset': 'valid', 'combine_valid_subsets': None, 'ignore_unused_valid_subsets': False, 'validate_interval': 1, 'validate_interval_updates': 0, 'validate_after_updates': 0, 'fixed_validation_seed': None, 'disable_validation': False, 'max_tokens_valid': 20000, 'batch_size_valid': None, 'max_valid_steps': None, 'curriculum': 0, 'gen_subset': 'test', 'num_shards': 1, 'shard_id': 0, 'grouped_shuffling': False, 'update_epoch_batch_itr': False, 'update_ordered_indices_seed': False}, 'optimization': {'_name': None, 'max_epoch': 0, 'max_update': 100000, 'stop_time_hours': 0.0, 'clip_norm': 0.0, 'sentence_avg': False, 'update_freq': [1], 'lr': [0.0005], 'stop_min_lr': -1.0, 'use_bmuf': False, 'skip_remainder_batch': False}, 'checkpoint': {'_name': None, 'save_dir': 'aarch64/trained_aarch64_best_postfix', 'restore_file': 'checkpoint_last.pt', 'continue_once': None, 'finetune_from_model': None, 'reset_dataloader': False, 'reset_lr_scheduler': False, 'reset_meters': False, 'reset_optimizer': False, 'optimizer_overrides': '{}', 'save_interval': 1, 'save_interval_updates': 0, 'keep_interval_updates': -1, 'keep_interval_updates_pattern': -1, 'keep_last_epochs': -1, 'keep_best_checkpoints': 3, 'no_save': False, 'no_epoch_checkpoints': True, 'no_last_checkpoints': False, 'no_save_optimizer_state': False, 'best_checkpoint_metric': 'loss', 'maximize_best_checkpoint_metric': False, 'patience': 4, 'checkpoint_suffix': '', 'checkpoint_shard_count': 1, 'load_checkpoint_on_all_dp_ranks': False, 'write_checkpoints_asynchronously': False, 'model_parallel_size': 1}, 'bmuf': {'_name': None, 'block_lr': 1.0, 'block_momentum': 0.875, 'global_sync_iter': 50, 'warmup_iterations': 500, 'use_nbm': False, 'average_sync': False, 'distributed_world_size': 1}, 'generation': {'_name': None, 'beam': 5, 'nbest': 1, 'max_len_a': 0.0, 'max_len_b': 200, 'min_len': 1, 'match_source_len': False, 'unnormalized': False, 'no_early_stop': False, 'no_beamable_mm': False, 'lenpen': 1.0, 'unkpen': 0.0, 'replace_unk': None, 'sacrebleu': False, 'score_reference': False, 'prefix_size': 0, 'no_repeat_ngram_size': 0, 'sampling': False, 'sampling_topk': -1, 'sampling_topp': -1.0, 'constraints': None, 'temperature': 1.0, 'diverse_beam_groups': -1, 'diverse_beam_strength': 0.5, 'diversity_rate': -1.0, 'print_alignment': None, 'print_step': False, 'lm_path': None, 'lm_weight': 0.0, 'iter_decode_eos_penalty': 0.0, 'iter_decode_max_iter': 10, 'iter_decode_force_max_iter': False, 'iter_decode_with_beam': 1, 'iter_decode_with_external_reranker': False, 'retain_iter_history': False, 'retain_dropout': False, 'retain_dropout_modules': None, 'decoding_format': None, 'no_seed_provided': False, 'eos_token': None}, 'eval_lm': {'_name': None, 'output_word_probs': False, 'output_word_stats': False, 'context_window': 0, 'softmax_batch': 9223372036854775807}, 'interactive': {'_name': None, 'buffer_size': 0, 'input': '-'}, 'model': Namespace(_name='transformer', activation_dropout=0.0, activation_fn='relu', adam_betas=(0.9, 0.999), adam_eps=1e-08, adaptive_input=False, adaptive_softmax_cutoff=None, adaptive_softmax_dropout=0, aim_repo=None, aim_run_hash=None, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, arch='transformer', attention_dropout=0.0, azureml_logging=False, batch_size=None, batch_size_valid=None, best_checkpoint_metric='loss', bf16=False, bpe=None, broadcast_buffers=False, bucket_cap_mb=25, checkpoint_activations=False, checkpoint_shard_count=1, checkpoint_suffix='', clip_norm=0.0, combine_valid_subsets=None, continue_once=None, cpu=False, cpu_offload=False, criterion='cross_entropy', cross_self_attention=False, curriculum=0, data='aarch64/tokenized_dlsm_aarch64_postfix', data_buffer_size=10, dataset_impl=None, ddp_backend='pytorch_ddp', ddp_comm_hook='none', decoder_attention_heads=8, decoder_embed_dim=96, decoder_embed_path=None, decoder_ffn_embed_dim=384, decoder_input_dim=96, decoder_layerdrop=0, decoder_layers=6, decoder_layers_to_keep=None, decoder_learned_pos=False, decoder_normalize_before=False, decoder_output_dim='96', device_id=0, disable_validation=False, distributed_backend='nccl', distributed_init_method=None, distributed_no_spawn=False, distributed_num_procs=1, distributed_port=-1, distributed_rank=0, distributed_world_size=1, dropout=0.1, ema_decay=0.9999, ema_fp32=False, ema_seed_model=None, ema_start_update=0, ema_update_freq=1, empty_cache_freq=0, encoder_attention_heads=8, encoder_embed_dim=384, encoder_embed_path=None, encoder_ffn_embed_dim=1536, encoder_layerdrop=0, encoder_layers=6, encoder_layers_to_keep=None, encoder_learned_pos=False, encoder_normalize_before=False, eos=2, eval_bleu=False, eval_bleu_args='{}', eval_bleu_detok='space', eval_bleu_detok_args='{}', eval_bleu_print_samples=False, eval_bleu_remove_bpe=None, eval_tokenized_bleu=False, fast_stat_sync=False, find_unused_parameters=False, finetune_from_model=None, fix_batches_to_gpus=False, fixed_validation_seed=None, fp16=False, fp16_adam_stats=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, fp32_reduce_scatter=False, gen_subset='test', gradient_as_bucket_view=False, grouped_shuffling=False, heartbeat_timeout=-1, ignore_unused_valid_subsets=False, keep_best_checkpoints=3, keep_interval_updates=-1, keep_interval_updates_pattern=-1, keep_last_epochs=-1, layernorm_embedding=False, left_pad_source=True, left_pad_target=False, load_alignments=False, load_checkpoint_on_all_dp_ranks=False, localsgd_frequency=3, log_file='aarch64/trained_aarch64_best_postfix/training.log', log_format=None, log_interval=100, lr=[0.0005], lr_scheduler='inverse_sqrt', max_epoch=0, max_source_positions=1024, max_target_positions=1024, max_tokens=20000, max_tokens_valid=20000, max_update=100000, max_valid_steps=None, maximize_best_checkpoint_metric=False, memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_cross_attention=False, no_epoch_checkpoints=True, no_last_checkpoints=False, no_progress_bar=False, no_reshard_after_forward=False, no_save=False, no_save_optimizer_state=False, no_scale_embedding=False, no_seed_provided=False, no_token_positional_embeddings=False, not_fsdp_flatten_parameters=False, nprocs_per_node=1, num_batch_buckets=0, num_shards=1, num_workers=1, offload_activations=False, on_cpu_convert_precision=False, optimizer='adam', optimizer_overrides='{}', pad=1, patience=4, pipeline_balance=None, pipeline_checkpoint='never', pipeline_chunks=0, pipeline_decoder_balance=None, pipeline_decoder_devices=None, pipeline_devices=None, pipeline_encoder_balance=None, pipeline_encoder_devices=None, pipeline_model_parallel=False, plasma_path='/tmp/plasma', profile=False, quant_noise_pq=0, quant_noise_pq_block_size=8, quant_noise_scalar=0, quantization_config_path=None, required_batch_size_multiple=8, required_seq_len_multiple=1, reset_dataloader=False, reset_logging=False, reset_lr_scheduler=False, reset_meters=False, reset_optimizer=False, restore_file='checkpoint_last.pt', save_dir='aarch64/trained_aarch64_best_postfix', save_interval=1, save_interval_updates=0, scoring='bleu', seed=1, sentence_avg=False, shard_id=0, share_all_embeddings=False, share_decoder_input_output_embed=False, skip_invalid_size_inputs_valid_test=False, skip_remainder_batch=False, slowmo_base_algorithm='localsgd', slowmo_momentum=None, source_lang=None, stop_min_lr=-1.0, stop_time_hours=0, store_ema=False, suppress_crashes=False, target_lang=None, task='translation', tensorboard_logdir=None, threshold_loss_scale=None, tie_adaptive_weights=False, tokenizer=None, tpu=False, train_subset='train', truncate_source=False, unk=3, update_epoch_batch_itr=False, update_freq=[1], update_ordered_indices_seed=False, upsample_primary=-1, use_bmuf=False, use_old_adam=False, use_plasma_view=False, use_sharded_state=False, user_dir=None, valid_subset='valid', validate_after_updates=0, validate_interval=1, validate_interval_updates=0, wandb_project=None, warmup_init_lr=-1, warmup_updates=4000, weight_decay=0.001, write_checkpoints_asynchronously=False, zero_sharding='none'), 'task': {'_name': 'translation', 'data': 'aarch64/tokenized_dlsm_aarch64_postfix', 'source_lang': None, 'target_lang': None, 'load_alignments': False, 'left_pad_source': True, 'left_pad_target': False, 'max_source_positions': 1024, 'max_target_positions': 1024, 'upsample_primary': -1, 'truncate_source': False, 'num_batch_buckets': 0, 'train_subset': 'train', 'dataset_impl': None, 'required_seq_len_multiple': 1, 'eval_bleu': False, 'eval_bleu_args': '{}', 'eval_bleu_detok': 'space', 'eval_bleu_detok_args': '{}', 'eval_tokenized_bleu': False, 'eval_bleu_remove_bpe': None, 'eval_bleu_print_samples': False}, 'criterion': {'_name': 'cross_entropy', 'sentence_avg': False}, 'optimizer': {'_name': 'adam', 'adam_betas': [0.9, 0.999], 'adam_eps': 1e-08, 'weight_decay': 0.001, 'use_old_adam': False, 'fp16_adam_stats': False, 'tpu': False, 'lr': [0.0005]}, 'lr_scheduler': {'_name': 'inverse_sqrt', 'warmup_updates': 4000, 'warmup_init_lr': -1.0, 'lr': [0.0005]}, 'scoring': {'_name': 'bleu', 'pad': 1, 'eos': 2, 'unk': 3}, 'bpe': None, 'tokenizer': None, 'ema': {'_name': None, 'store_ema': False, 'ema_decay': 0.9999, 'ema_start_update': 0, 'ema_seed_model': None, 'ema_update_freq': 1, 'ema_fp32': False}}
+TransformerModel(
+  (encoder): TransformerEncoderBase(
+    (dropout_module): FairseqDropout()
+    (embed_tokens): Embedding(168, 384, padding_idx=1)
+    (embed_positions): SinusoidalPositionalEmbedding()
+    (layers): ModuleList(
+      (0-5): 6 x TransformerEncoderLayerBase(
+        (self_attn): MultiheadAttention(
+          (dropout_module): FairseqDropout()
+          (k_proj): Linear(in_features=384, out_features=384, bias=True)
+          (v_proj): Linear(in_features=384, out_features=384, bias=True)
+          (q_proj): Linear(in_features=384, out_features=384, bias=True)
+          (out_proj): Linear(in_features=384, out_features=384, bias=True)
+        )
+        (self_attn_layer_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+        (dropout_module): FairseqDropout()
+        (activation_dropout_module): FairseqDropout()
+        (fc1): Linear(in_features=384, out_features=1536, bias=True)
+        (fc2): Linear(in_features=1536, out_features=384, bias=True)
+        (final_layer_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+      )
+    )
+  )
+  (decoder): TransformerDecoderBase(
+    (dropout_module): FairseqDropout()
+    (embed_tokens): Embedding(40, 96, padding_idx=1)
+    (embed_positions): SinusoidalPositionalEmbedding()
+    (layers): ModuleList(
+      (0-5): 6 x TransformerDecoderLayerBase(
+        (dropout_module): FairseqDropout()
+        (self_attn): MultiheadAttention(
+          (dropout_module): FairseqDropout()
+          (k_proj): Linear(in_features=96, out_features=96, bias=True)
+          (v_proj): Linear(in_features=96, out_features=96, bias=True)
+          (q_proj): Linear(in_features=96, out_features=96, bias=True)
+          (out_proj): Linear(in_features=96, out_features=96, bias=True)
+        )
+        (activation_dropout_module): FairseqDropout()
+        (self_attn_layer_norm): LayerNorm((96,), eps=1e-05, elementwise_affine=True)
+        (encoder_attn): MultiheadAttention(
+          (dropout_module): FairseqDropout()
+          (k_proj): Linear(in_features=384, out_features=96, bias=True)
+          (v_proj): Linear(in_features=384, out_features=96, bias=True)
+          (q_proj): Linear(in_features=96, out_features=96, bias=True)
+          (out_proj): Linear(in_features=96, out_features=96, bias=True)
+        )
+        (encoder_attn_layer_norm): LayerNorm((96,), eps=1e-05, elementwise_affine=True)
+        (fc1): Linear(in_features=96, out_features=384, bias=True)
+        (fc2): Linear(in_features=384, out_features=96, bias=True)
+        (final_layer_norm): LayerNorm((96,), eps=1e-05, elementwise_affine=True)
+      )
+    )
+    (output_projection): Linear(in_features=96, out_features=40, bias=False)
+  )
+)
+task: TranslationTask
+model: TransformerModel
+criterion: CrossEntropyCriterion
+num. shared model params: 22,584,000 (num. trained: 22,584,000)
+num. expert model params: 0 (num. trained: 0)
+training on 1 devices (GPUs/TPUs)
+max tokens per device = 20000 and max sentences per device = None
+Start iterating over samples
+begin validation on "valid" subset
+epoch 001 | valid on 'valid' subset | loss 1.037 | ppl 2.05 | wps 21510.2 | wpb 1943.9 | bsz 93.9 | num_updates 4943
+end of epoch 1 (average epoch stats below)
+epoch 001 | loss 1.718 | ppl 3.29 | wps 6524 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 4943 | lr 0.000449785 | gnorm 2.84 | train_wall 1470 | gb_free 13 | wall 1494
+Start iterating over samples
+begin validation on "valid" subset
+epoch 002 | valid on 'valid' subset | loss 0.526 | ppl 1.44 | wps 21452 | wpb 1943.9 | bsz 93.9 | num_updates 9886 | best_loss 0.526
+epoch 002 | valid on 'valid' subset | loss 0.526 | ppl 1.44 | wps 21452 | wpb 1943.9 | bsz 93.9 | num_updates 9886 | best_loss 0.526
+end of epoch 2 (average epoch stats below)
+epoch 002 | loss 0.745 | ppl 1.68 | wps 6532.6 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 9886 | lr 0.000318046 | gnorm 1.305 | train_wall 1468 | gb_free 13 | wall 2986
+epoch 002 | loss 0.745 | ppl 1.68 | wps 6532.6 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 9886 | lr 0.000318046 | gnorm 1.305 | train_wall 1468 | gb_free 13 | wall 2986
+Start iterating over samples
+begin validation on "valid" subset
+epoch 003 | valid on 'valid' subset | loss 0.347 | ppl 1.27 | wps 21573.1 | wpb 1943.9 | bsz 93.9 | num_updates 14829 | best_loss 0.347
+epoch 003 | valid on 'valid' subset | loss 0.347 | ppl 1.27 | wps 21573.1 | wpb 1943.9 | bsz 93.9 | num_updates 14829 | best_loss 0.347
+epoch 003 | valid on 'valid' subset | loss 0.347 | ppl 1.27 | wps 21573.1 | wpb 1943.9 | bsz 93.9 | num_updates 14829 | best_loss 0.347
+end of epoch 3 (average epoch stats below)
+epoch 003 | loss 0.402 | ppl 1.32 | wps 6533.6 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 14829 | lr 0.000259683 | gnorm 0.979 | train_wall 1467 | gb_free 12.7 | wall 4478
+epoch 003 | loss 0.402 | ppl 1.32 | wps 6533.6 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 14829 | lr 0.000259683 | gnorm 0.979 | train_wall 1467 | gb_free 12.7 | wall 4478
+epoch 003 | loss 0.402 | ppl 1.32 | wps 6533.6 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 14829 | lr 0.000259683 | gnorm 0.979 | train_wall 1467 | gb_free 12.7 | wall 4478
+Start iterating over samples
+begin validation on "valid" subset
+epoch 004 | valid on 'valid' subset | loss 0.264 | ppl 1.2 | wps 21501.3 | wpb 1943.9 | bsz 93.9 | num_updates 19772 | best_loss 0.264
+epoch 004 | valid on 'valid' subset | loss 0.264 | ppl 1.2 | wps 21501.3 | wpb 1943.9 | bsz 93.9 | num_updates 19772 | best_loss 0.264
+epoch 004 | valid on 'valid' subset | loss 0.264 | ppl 1.2 | wps 21501.3 | wpb 1943.9 | bsz 93.9 | num_updates 19772 | best_loss 0.264
+epoch 004 | valid on 'valid' subset | loss 0.264 | ppl 1.2 | wps 21501.3 | wpb 1943.9 | bsz 93.9 | num_updates 19772 | best_loss 0.264
+end of epoch 4 (average epoch stats below)
+epoch 004 | loss 0.24 | ppl 1.18 | wps 6540.8 | ups 3.32 | wpb 1971.8 | bsz 95.4 | num_updates 19772 | lr 0.000224892 | gnorm 0.813 | train_wall 1466 | gb_free 13 | wall 5968
+epoch 004 | loss 0.24 | ppl 1.18 | wps 6540.8 | ups 3.32 | wpb 1971.8 | bsz 95.4 | num_updates 19772 | lr 0.000224892 | gnorm 0.813 | train_wall 1466 | gb_free 13 | wall 5968
+epoch 004 | loss 0.24 | ppl 1.18 | wps 6540.8 | ups 3.32 | wpb 1971.8 | bsz 95.4 | num_updates 19772 | lr 0.000224892 | gnorm 0.813 | train_wall 1466 | gb_free 13 | wall 5968
+epoch 004 | loss 0.24 | ppl 1.18 | wps 6540.8 | ups 3.32 | wpb 1971.8 | bsz 95.4 | num_updates 19772 | lr 0.000224892 | gnorm 0.813 | train_wall 1466 | gb_free 13 | wall 5968
+Start iterating over samples
+begin validation on "valid" subset
+epoch 005 | valid on 'valid' subset | loss 0.194 | ppl 1.14 | wps 21456.9 | wpb 1943.9 | bsz 93.9 | num_updates 24715 | best_loss 0.194
+epoch 005 | valid on 'valid' subset | loss 0.194 | ppl 1.14 | wps 21456.9 | wpb 1943.9 | bsz 93.9 | num_updates 24715 | best_loss 0.194
+epoch 005 | valid on 'valid' subset | loss 0.194 | ppl 1.14 | wps 21456.9 | wpb 1943.9 | bsz 93.9 | num_updates 24715 | best_loss 0.194
+epoch 005 | valid on 'valid' subset | loss 0.194 | ppl 1.14 | wps 21456.9 | wpb 1943.9 | bsz 93.9 | num_updates 24715 | best_loss 0.194
+epoch 005 | valid on 'valid' subset | loss 0.194 | ppl 1.14 | wps 21456.9 | wpb 1943.9 | bsz 93.9 | num_updates 24715 | best_loss 0.194
+end of epoch 5 (average epoch stats below)
+epoch 005 | loss 0.14 | ppl 1.1 | wps 6454.8 | ups 3.27 | wpb 1971.8 | bsz 95.4 | num_updates 24715 | lr 0.00020115 | gnorm 0.705 | train_wall 1469 | gb_free 12.9 | wall 7478
+epoch 005 | loss 0.14 | ppl 1.1 | wps 6454.8 | ups 3.27 | wpb 1971.8 | bsz 95.4 | num_updates 24715 | lr 0.00020115 | gnorm 0.705 | train_wall 1469 | gb_free 12.9 | wall 7478
+epoch 005 | loss 0.14 | ppl 1.1 | wps 6454.8 | ups 3.27 | wpb 1971.8 | bsz 95.4 | num_updates 24715 | lr 0.00020115 | gnorm 0.705 | train_wall 1469 | gb_free 12.9 | wall 7478
+epoch 005 | loss 0.14 | ppl 1.1 | wps 6454.8 | ups 3.27 | wpb 1971.8 | bsz 95.4 | num_updates 24715 | lr 0.00020115 | gnorm 0.705 | train_wall 1469 | gb_free 12.9 | wall 7478
+epoch 005 | loss 0.14 | ppl 1.1 | wps 6454.8 | ups 3.27 | wpb 1971.8 | bsz 95.4 | num_updates 24715 | lr 0.00020115 | gnorm 0.705 | train_wall 1469 | gb_free 12.9 | wall 7478
+Start iterating over samples
+begin validation on "valid" subset
+epoch 006 | valid on 'valid' subset | loss 0.146 | ppl 1.11 | wps 21466.7 | wpb 1943.9 | bsz 93.9 | num_updates 29658 | best_loss 0.146
+epoch 006 | valid on 'valid' subset | loss 0.146 | ppl 1.11 | wps 21466.7 | wpb 1943.9 | bsz 93.9 | num_updates 29658 | best_loss 0.146
+epoch 006 | valid on 'valid' subset | loss 0.146 | ppl 1.11 | wps 21466.7 | wpb 1943.9 | bsz 93.9 | num_updates 29658 | best_loss 0.146
+epoch 006 | valid on 'valid' subset | loss 0.146 | ppl 1.11 | wps 21466.7 | wpb 1943.9 | bsz 93.9 | num_updates 29658 | best_loss 0.146
+epoch 006 | valid on 'valid' subset | loss 0.146 | ppl 1.11 | wps 21466.7 | wpb 1943.9 | bsz 93.9 | num_updates 29658 | best_loss 0.146
+epoch 006 | valid on 'valid' subset | loss 0.146 | ppl 1.11 | wps 21466.7 | wpb 1943.9 | bsz 93.9 | num_updates 29658 | best_loss 0.146
+end of epoch 6 (average epoch stats below)
+epoch 006 | loss 0.081 | ppl 1.06 | wps 6502.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 29658 | lr 0.000183624 | gnorm 0.609 | train_wall 1474 | gb_free 12.8 | wall 8977
+epoch 006 | loss 0.081 | ppl 1.06 | wps 6502.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 29658 | lr 0.000183624 | gnorm 0.609 | train_wall 1474 | gb_free 12.8 | wall 8977
+epoch 006 | loss 0.081 | ppl 1.06 | wps 6502.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 29658 | lr 0.000183624 | gnorm 0.609 | train_wall 1474 | gb_free 12.8 | wall 8977
+epoch 006 | loss 0.081 | ppl 1.06 | wps 6502.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 29658 | lr 0.000183624 | gnorm 0.609 | train_wall 1474 | gb_free 12.8 | wall 8977
+epoch 006 | loss 0.081 | ppl 1.06 | wps 6502.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 29658 | lr 0.000183624 | gnorm 0.609 | train_wall 1474 | gb_free 12.8 | wall 8977
+epoch 006 | loss 0.081 | ppl 1.06 | wps 6502.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 29658 | lr 0.000183624 | gnorm 0.609 | train_wall 1474 | gb_free 12.8 | wall 8977
+Start iterating over samples
+begin validation on "valid" subset
+epoch 007 | valid on 'valid' subset | loss 0.138 | ppl 1.1 | wps 21354.2 | wpb 1943.9 | bsz 93.9 | num_updates 34601 | best_loss 0.138
+epoch 007 | valid on 'valid' subset | loss 0.138 | ppl 1.1 | wps 21354.2 | wpb 1943.9 | bsz 93.9 | num_updates 34601 | best_loss 0.138
+epoch 007 | valid on 'valid' subset | loss 0.138 | ppl 1.1 | wps 21354.2 | wpb 1943.9 | bsz 93.9 | num_updates 34601 | best_loss 0.138
+epoch 007 | valid on 'valid' subset | loss 0.138 | ppl 1.1 | wps 21354.2 | wpb 1943.9 | bsz 93.9 | num_updates 34601 | best_loss 0.138
+epoch 007 | valid on 'valid' subset | loss 0.138 | ppl 1.1 | wps 21354.2 | wpb 1943.9 | bsz 93.9 | num_updates 34601 | best_loss 0.138
+epoch 007 | valid on 'valid' subset | loss 0.138 | ppl 1.1 | wps 21354.2 | wpb 1943.9 | bsz 93.9 | num_updates 34601 | best_loss 0.138
+epoch 007 | valid on 'valid' subset | loss 0.138 | ppl 1.1 | wps 21354.2 | wpb 1943.9 | bsz 93.9 | num_updates 34601 | best_loss 0.138
+end of epoch 7 (average epoch stats below)
+epoch 007 | loss 0.051 | ppl 1.04 | wps 6479.2 | ups 3.29 | wpb 1971.8 | bsz 95.4 | num_updates 34601 | lr 0.000170003 | gnorm 0.544 | train_wall 1478 | gb_free 13.2 | wall 10481
+epoch 007 | loss 0.051 | ppl 1.04 | wps 6479.2 | ups 3.29 | wpb 1971.8 | bsz 95.4 | num_updates 34601 | lr 0.000170003 | gnorm 0.544 | train_wall 1478 | gb_free 13.2 | wall 10481
+epoch 007 | loss 0.051 | ppl 1.04 | wps 6479.2 | ups 3.29 | wpb 1971.8 | bsz 95.4 | num_updates 34601 | lr 0.000170003 | gnorm 0.544 | train_wall 1478 | gb_free 13.2 | wall 10481
+epoch 007 | loss 0.051 | ppl 1.04 | wps 6479.2 | ups 3.29 | wpb 1971.8 | bsz 95.4 | num_updates 34601 | lr 0.000170003 | gnorm 0.544 | train_wall 1478 | gb_free 13.2 | wall 10481
+epoch 007 | loss 0.051 | ppl 1.04 | wps 6479.2 | ups 3.29 | wpb 1971.8 | bsz 95.4 | num_updates 34601 | lr 0.000170003 | gnorm 0.544 | train_wall 1478 | gb_free 13.2 | wall 10481
+epoch 007 | loss 0.051 | ppl 1.04 | wps 6479.2 | ups 3.29 | wpb 1971.8 | bsz 95.4 | num_updates 34601 | lr 0.000170003 | gnorm 0.544 | train_wall 1478 | gb_free 13.2 | wall 10481
+epoch 007 | loss 0.051 | ppl 1.04 | wps 6479.2 | ups 3.29 | wpb 1971.8 | bsz 95.4 | num_updates 34601 | lr 0.000170003 | gnorm 0.544 | train_wall 1478 | gb_free 13.2 | wall 10481
+Start iterating over samples
+begin validation on "valid" subset
+epoch 008 | valid on 'valid' subset | loss 0.111 | ppl 1.08 | wps 21464.3 | wpb 1943.9 | bsz 93.9 | num_updates 39544 | best_loss 0.111
+epoch 008 | valid on 'valid' subset | loss 0.111 | ppl 1.08 | wps 21464.3 | wpb 1943.9 | bsz 93.9 | num_updates 39544 | best_loss 0.111
+epoch 008 | valid on 'valid' subset | loss 0.111 | ppl 1.08 | wps 21464.3 | wpb 1943.9 | bsz 93.9 | num_updates 39544 | best_loss 0.111
+epoch 008 | valid on 'valid' subset | loss 0.111 | ppl 1.08 | wps 21464.3 | wpb 1943.9 | bsz 93.9 | num_updates 39544 | best_loss 0.111
+epoch 008 | valid on 'valid' subset | loss 0.111 | ppl 1.08 | wps 21464.3 | wpb 1943.9 | bsz 93.9 | num_updates 39544 | best_loss 0.111
+epoch 008 | valid on 'valid' subset | loss 0.111 | ppl 1.08 | wps 21464.3 | wpb 1943.9 | bsz 93.9 | num_updates 39544 | best_loss 0.111
+epoch 008 | valid on 'valid' subset | loss 0.111 | ppl 1.08 | wps 21464.3 | wpb 1943.9 | bsz 93.9 | num_updates 39544 | best_loss 0.111
+epoch 008 | valid on 'valid' subset | loss 0.111 | ppl 1.08 | wps 21464.3 | wpb 1943.9 | bsz 93.9 | num_updates 39544 | best_loss 0.111
+end of epoch 8 (average epoch stats below)
+epoch 008 | loss 0.034 | ppl 1.02 | wps 6517.4 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 39544 | lr 0.000159023 | gnorm 0.47 | train_wall 1471 | gb_free 12 | wall 11977
+epoch 008 | loss 0.034 | ppl 1.02 | wps 6517.4 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 39544 | lr 0.000159023 | gnorm 0.47 | train_wall 1471 | gb_free 12 | wall 11977
+epoch 008 | loss 0.034 | ppl 1.02 | wps 6517.4 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 39544 | lr 0.000159023 | gnorm 0.47 | train_wall 1471 | gb_free 12 | wall 11977
+epoch 008 | loss 0.034 | ppl 1.02 | wps 6517.4 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 39544 | lr 0.000159023 | gnorm 0.47 | train_wall 1471 | gb_free 12 | wall 11977
+epoch 008 | loss 0.034 | ppl 1.02 | wps 6517.4 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 39544 | lr 0.000159023 | gnorm 0.47 | train_wall 1471 | gb_free 12 | wall 11977
+epoch 008 | loss 0.034 | ppl 1.02 | wps 6517.4 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 39544 | lr 0.000159023 | gnorm 0.47 | train_wall 1471 | gb_free 12 | wall 11977
+epoch 008 | loss 0.034 | ppl 1.02 | wps 6517.4 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 39544 | lr 0.000159023 | gnorm 0.47 | train_wall 1471 | gb_free 12 | wall 11977
+epoch 008 | loss 0.034 | ppl 1.02 | wps 6517.4 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 39544 | lr 0.000159023 | gnorm 0.47 | train_wall 1471 | gb_free 12 | wall 11977
+Start iterating over samples
+begin validation on "valid" subset
+epoch 009 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21374 | wpb 1943.9 | bsz 93.9 | num_updates 44487 | best_loss 0.11
+epoch 009 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21374 | wpb 1943.9 | bsz 93.9 | num_updates 44487 | best_loss 0.11
+epoch 009 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21374 | wpb 1943.9 | bsz 93.9 | num_updates 44487 | best_loss 0.11
+epoch 009 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21374 | wpb 1943.9 | bsz 93.9 | num_updates 44487 | best_loss 0.11
+epoch 009 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21374 | wpb 1943.9 | bsz 93.9 | num_updates 44487 | best_loss 0.11
+epoch 009 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21374 | wpb 1943.9 | bsz 93.9 | num_updates 44487 | best_loss 0.11
+epoch 009 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21374 | wpb 1943.9 | bsz 93.9 | num_updates 44487 | best_loss 0.11
+epoch 009 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21374 | wpb 1943.9 | bsz 93.9 | num_updates 44487 | best_loss 0.11
+epoch 009 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21374 | wpb 1943.9 | bsz 93.9 | num_updates 44487 | best_loss 0.11
+end of epoch 9 (average epoch stats below)
+epoch 009 | loss 0.026 | ppl 1.02 | wps 6516.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 44487 | lr 0.000149928 | gnorm 0.429 | train_wall 1471 | gb_free 12.6 | wall 13472
+epoch 009 | loss 0.026 | ppl 1.02 | wps 6516.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 44487 | lr 0.000149928 | gnorm 0.429 | train_wall 1471 | gb_free 12.6 | wall 13472
+epoch 009 | loss 0.026 | ppl 1.02 | wps 6516.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 44487 | lr 0.000149928 | gnorm 0.429 | train_wall 1471 | gb_free 12.6 | wall 13472
+epoch 009 | loss 0.026 | ppl 1.02 | wps 6516.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 44487 | lr 0.000149928 | gnorm 0.429 | train_wall 1471 | gb_free 12.6 | wall 13472
+epoch 009 | loss 0.026 | ppl 1.02 | wps 6516.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 44487 | lr 0.000149928 | gnorm 0.429 | train_wall 1471 | gb_free 12.6 | wall 13472
+epoch 009 | loss 0.026 | ppl 1.02 | wps 6516.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 44487 | lr 0.000149928 | gnorm 0.429 | train_wall 1471 | gb_free 12.6 | wall 13472
+epoch 009 | loss 0.026 | ppl 1.02 | wps 6516.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 44487 | lr 0.000149928 | gnorm 0.429 | train_wall 1471 | gb_free 12.6 | wall 13472
+epoch 009 | loss 0.026 | ppl 1.02 | wps 6516.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 44487 | lr 0.000149928 | gnorm 0.429 | train_wall 1471 | gb_free 12.6 | wall 13472
+epoch 009 | loss 0.026 | ppl 1.02 | wps 6516.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 44487 | lr 0.000149928 | gnorm 0.429 | train_wall 1471 | gb_free 12.6 | wall 13472
+Start iterating over samples
+begin validation on "valid" subset
+epoch 010 | valid on 'valid' subset | loss 0.113 | ppl 1.08 | wps 21458.6 | wpb 1943.9 | bsz 93.9 | num_updates 49430 | best_loss 0.11
+epoch 010 | valid on 'valid' subset | loss 0.113 | ppl 1.08 | wps 21458.6 | wpb 1943.9 | bsz 93.9 | num_updates 49430 | best_loss 0.11
+epoch 010 | valid on 'valid' subset | loss 0.113 | ppl 1.08 | wps 21458.6 | wpb 1943.9 | bsz 93.9 | num_updates 49430 | best_loss 0.11
+epoch 010 | valid on 'valid' subset | loss 0.113 | ppl 1.08 | wps 21458.6 | wpb 1943.9 | bsz 93.9 | num_updates 49430 | best_loss 0.11
+epoch 010 | valid on 'valid' subset | loss 0.113 | ppl 1.08 | wps 21458.6 | wpb 1943.9 | bsz 93.9 | num_updates 49430 | best_loss 0.11
+epoch 010 | valid on 'valid' subset | loss 0.113 | ppl 1.08 | wps 21458.6 | wpb 1943.9 | bsz 93.9 | num_updates 49430 | best_loss 0.11
+epoch 010 | valid on 'valid' subset | loss 0.113 | ppl 1.08 | wps 21458.6 | wpb 1943.9 | bsz 93.9 | num_updates 49430 | best_loss 0.11
+epoch 010 | valid on 'valid' subset | loss 0.113 | ppl 1.08 | wps 21458.6 | wpb 1943.9 | bsz 93.9 | num_updates 49430 | best_loss 0.11
+epoch 010 | valid on 'valid' subset | loss 0.113 | ppl 1.08 | wps 21458.6 | wpb 1943.9 | bsz 93.9 | num_updates 49430 | best_loss 0.11
+epoch 010 | valid on 'valid' subset | loss 0.113 | ppl 1.08 | wps 21458.6 | wpb 1943.9 | bsz 93.9 | num_updates 49430 | best_loss 0.11
+end of epoch 10 (average epoch stats below)
+epoch 010 | loss 0.02 | ppl 1.01 | wps 6514.4 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 49430 | lr 0.000142234 | gnorm 0.394 | train_wall 1472 | gb_free 12.7 | wall 14968
+epoch 010 | loss 0.02 | ppl 1.01 | wps 6514.4 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 49430 | lr 0.000142234 | gnorm 0.394 | train_wall 1472 | gb_free 12.7 | wall 14968
+epoch 010 | loss 0.02 | ppl 1.01 | wps 6514.4 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 49430 | lr 0.000142234 | gnorm 0.394 | train_wall 1472 | gb_free 12.7 | wall 14968
+epoch 010 | loss 0.02 | ppl 1.01 | wps 6514.4 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 49430 | lr 0.000142234 | gnorm 0.394 | train_wall 1472 | gb_free 12.7 | wall 14968
+epoch 010 | loss 0.02 | ppl 1.01 | wps 6514.4 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 49430 | lr 0.000142234 | gnorm 0.394 | train_wall 1472 | gb_free 12.7 | wall 14968
+epoch 010 | loss 0.02 | ppl 1.01 | wps 6514.4 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 49430 | lr 0.000142234 | gnorm 0.394 | train_wall 1472 | gb_free 12.7 | wall 14968
+epoch 010 | loss 0.02 | ppl 1.01 | wps 6514.4 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 49430 | lr 0.000142234 | gnorm 0.394 | train_wall 1472 | gb_free 12.7 | wall 14968
+epoch 010 | loss 0.02 | ppl 1.01 | wps 6514.4 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 49430 | lr 0.000142234 | gnorm 0.394 | train_wall 1472 | gb_free 12.7 | wall 14968
+epoch 010 | loss 0.02 | ppl 1.01 | wps 6514.4 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 49430 | lr 0.000142234 | gnorm 0.394 | train_wall 1472 | gb_free 12.7 | wall 14968
+epoch 010 | loss 0.02 | ppl 1.01 | wps 6514.4 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 49430 | lr 0.000142234 | gnorm 0.394 | train_wall 1472 | gb_free 12.7 | wall 14968
+Start iterating over samples
+begin validation on "valid" subset
+epoch 011 | valid on 'valid' subset | loss 0.123 | ppl 1.09 | wps 21419.1 | wpb 1943.9 | bsz 93.9 | num_updates 54373 | best_loss 0.11
+epoch 011 | valid on 'valid' subset | loss 0.123 | ppl 1.09 | wps 21419.1 | wpb 1943.9 | bsz 93.9 | num_updates 54373 | best_loss 0.11
+epoch 011 | valid on 'valid' subset | loss 0.123 | ppl 1.09 | wps 21419.1 | wpb 1943.9 | bsz 93.9 | num_updates 54373 | best_loss 0.11
+epoch 011 | valid on 'valid' subset | loss 0.123 | ppl 1.09 | wps 21419.1 | wpb 1943.9 | bsz 93.9 | num_updates 54373 | best_loss 0.11
+epoch 011 | valid on 'valid' subset | loss 0.123 | ppl 1.09 | wps 21419.1 | wpb 1943.9 | bsz 93.9 | num_updates 54373 | best_loss 0.11
+epoch 011 | valid on 'valid' subset | loss 0.123 | ppl 1.09 | wps 21419.1 | wpb 1943.9 | bsz 93.9 | num_updates 54373 | best_loss 0.11
+epoch 011 | valid on 'valid' subset | loss 0.123 | ppl 1.09 | wps 21419.1 | wpb 1943.9 | bsz 93.9 | num_updates 54373 | best_loss 0.11
+epoch 011 | valid on 'valid' subset | loss 0.123 | ppl 1.09 | wps 21419.1 | wpb 1943.9 | bsz 93.9 | num_updates 54373 | best_loss 0.11
+epoch 011 | valid on 'valid' subset | loss 0.123 | ppl 1.09 | wps 21419.1 | wpb 1943.9 | bsz 93.9 | num_updates 54373 | best_loss 0.11
+epoch 011 | valid on 'valid' subset | loss 0.123 | ppl 1.09 | wps 21419.1 | wpb 1943.9 | bsz 93.9 | num_updates 54373 | best_loss 0.11
+epoch 011 | valid on 'valid' subset | loss 0.123 | ppl 1.09 | wps 21419.1 | wpb 1943.9 | bsz 93.9 | num_updates 54373 | best_loss 0.11
+end of epoch 11 (average epoch stats below)
+epoch 011 | loss 0.017 | ppl 1.01 | wps 6514.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 54373 | lr 0.000135615 | gnorm 0.361 | train_wall 1472 | gb_free 12.8 | wall 16464
+epoch 011 | loss 0.017 | ppl 1.01 | wps 6514.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 54373 | lr 0.000135615 | gnorm 0.361 | train_wall 1472 | gb_free 12.8 | wall 16464
+epoch 011 | loss 0.017 | ppl 1.01 | wps 6514.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 54373 | lr 0.000135615 | gnorm 0.361 | train_wall 1472 | gb_free 12.8 | wall 16464
+epoch 011 | loss 0.017 | ppl 1.01 | wps 6514.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 54373 | lr 0.000135615 | gnorm 0.361 | train_wall 1472 | gb_free 12.8 | wall 16464
+epoch 011 | loss 0.017 | ppl 1.01 | wps 6514.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 54373 | lr 0.000135615 | gnorm 0.361 | train_wall 1472 | gb_free 12.8 | wall 16464
+epoch 011 | loss 0.017 | ppl 1.01 | wps 6514.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 54373 | lr 0.000135615 | gnorm 0.361 | train_wall 1472 | gb_free 12.8 | wall 16464
+epoch 011 | loss 0.017 | ppl 1.01 | wps 6514.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 54373 | lr 0.000135615 | gnorm 0.361 | train_wall 1472 | gb_free 12.8 | wall 16464
+epoch 011 | loss 0.017 | ppl 1.01 | wps 6514.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 54373 | lr 0.000135615 | gnorm 0.361 | train_wall 1472 | gb_free 12.8 | wall 16464
+epoch 011 | loss 0.017 | ppl 1.01 | wps 6514.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 54373 | lr 0.000135615 | gnorm 0.361 | train_wall 1472 | gb_free 12.8 | wall 16464
+epoch 011 | loss 0.017 | ppl 1.01 | wps 6514.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 54373 | lr 0.000135615 | gnorm 0.361 | train_wall 1472 | gb_free 12.8 | wall 16464
+epoch 011 | loss 0.017 | ppl 1.01 | wps 6514.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 54373 | lr 0.000135615 | gnorm 0.361 | train_wall 1472 | gb_free 12.8 | wall 16464
+Start iterating over samples
+begin validation on "valid" subset
+epoch 012 | valid on 'valid' subset | loss 0.108 | ppl 1.08 | wps 21329.7 | wpb 1943.9 | bsz 93.9 | num_updates 59316 | best_loss 0.108
+epoch 012 | valid on 'valid' subset | loss 0.108 | ppl 1.08 | wps 21329.7 | wpb 1943.9 | bsz 93.9 | num_updates 59316 | best_loss 0.108
+epoch 012 | valid on 'valid' subset | loss 0.108 | ppl 1.08 | wps 21329.7 | wpb 1943.9 | bsz 93.9 | num_updates 59316 | best_loss 0.108
+epoch 012 | valid on 'valid' subset | loss 0.108 | ppl 1.08 | wps 21329.7 | wpb 1943.9 | bsz 93.9 | num_updates 59316 | best_loss 0.108
+epoch 012 | valid on 'valid' subset | loss 0.108 | ppl 1.08 | wps 21329.7 | wpb 1943.9 | bsz 93.9 | num_updates 59316 | best_loss 0.108
+epoch 012 | valid on 'valid' subset | loss 0.108 | ppl 1.08 | wps 21329.7 | wpb 1943.9 | bsz 93.9 | num_updates 59316 | best_loss 0.108
+epoch 012 | valid on 'valid' subset | loss 0.108 | ppl 1.08 | wps 21329.7 | wpb 1943.9 | bsz 93.9 | num_updates 59316 | best_loss 0.108
+epoch 012 | valid on 'valid' subset | loss 0.108 | ppl 1.08 | wps 21329.7 | wpb 1943.9 | bsz 93.9 | num_updates 59316 | best_loss 0.108
+epoch 012 | valid on 'valid' subset | loss 0.108 | ppl 1.08 | wps 21329.7 | wpb 1943.9 | bsz 93.9 | num_updates 59316 | best_loss 0.108
+epoch 012 | valid on 'valid' subset | loss 0.108 | ppl 1.08 | wps 21329.7 | wpb 1943.9 | bsz 93.9 | num_updates 59316 | best_loss 0.108
+epoch 012 | valid on 'valid' subset | loss 0.108 | ppl 1.08 | wps 21329.7 | wpb 1943.9 | bsz 93.9 | num_updates 59316 | best_loss 0.108
+epoch 012 | valid on 'valid' subset | loss 0.108 | ppl 1.08 | wps 21329.7 | wpb 1943.9 | bsz 93.9 | num_updates 59316 | best_loss 0.108
+end of epoch 12 (average epoch stats below)
+epoch 012 | loss 0.014 | ppl 1.01 | wps 6511.6 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 59316 | lr 0.000129842 | gnorm 0.334 | train_wall 1472 | gb_free 12.8 | wall 17961
+epoch 012 | loss 0.014 | ppl 1.01 | wps 6511.6 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 59316 | lr 0.000129842 | gnorm 0.334 | train_wall 1472 | gb_free 12.8 | wall 17961
+epoch 012 | loss 0.014 | ppl 1.01 | wps 6511.6 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 59316 | lr 0.000129842 | gnorm 0.334 | train_wall 1472 | gb_free 12.8 | wall 17961
+epoch 012 | loss 0.014 | ppl 1.01 | wps 6511.6 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 59316 | lr 0.000129842 | gnorm 0.334 | train_wall 1472 | gb_free 12.8 | wall 17961
+epoch 012 | loss 0.014 | ppl 1.01 | wps 6511.6 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 59316 | lr 0.000129842 | gnorm 0.334 | train_wall 1472 | gb_free 12.8 | wall 17961
+epoch 012 | loss 0.014 | ppl 1.01 | wps 6511.6 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 59316 | lr 0.000129842 | gnorm 0.334 | train_wall 1472 | gb_free 12.8 | wall 17961
+epoch 012 | loss 0.014 | ppl 1.01 | wps 6511.6 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 59316 | lr 0.000129842 | gnorm 0.334 | train_wall 1472 | gb_free 12.8 | wall 17961
+epoch 012 | loss 0.014 | ppl 1.01 | wps 6511.6 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 59316 | lr 0.000129842 | gnorm 0.334 | train_wall 1472 | gb_free 12.8 | wall 17961
+epoch 012 | loss 0.014 | ppl 1.01 | wps 6511.6 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 59316 | lr 0.000129842 | gnorm 0.334 | train_wall 1472 | gb_free 12.8 | wall 17961
+epoch 012 | loss 0.014 | ppl 1.01 | wps 6511.6 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 59316 | lr 0.000129842 | gnorm 0.334 | train_wall 1472 | gb_free 12.8 | wall 17961
+epoch 012 | loss 0.014 | ppl 1.01 | wps 6511.6 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 59316 | lr 0.000129842 | gnorm 0.334 | train_wall 1472 | gb_free 12.8 | wall 17961
+epoch 012 | loss 0.014 | ppl 1.01 | wps 6511.6 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 59316 | lr 0.000129842 | gnorm 0.334 | train_wall 1472 | gb_free 12.8 | wall 17961
+Start iterating over samples
+begin validation on "valid" subset
+epoch 013 | valid on 'valid' subset | loss 0.108 | ppl 1.08 | wps 21359.7 | wpb 1943.9 | bsz 93.9 | num_updates 64259 | best_loss 0.108
+epoch 013 | valid on 'valid' subset | loss 0.108 | ppl 1.08 | wps 21359.7 | wpb 1943.9 | bsz 93.9 | num_updates 64259 | best_loss 0.108
+epoch 013 | valid on 'valid' subset | loss 0.108 | ppl 1.08 | wps 21359.7 | wpb 1943.9 | bsz 93.9 | num_updates 64259 | best_loss 0.108
+epoch 013 | valid on 'valid' subset | loss 0.108 | ppl 1.08 | wps 21359.7 | wpb 1943.9 | bsz 93.9 | num_updates 64259 | best_loss 0.108
+epoch 013 | valid on 'valid' subset | loss 0.108 | ppl 1.08 | wps 21359.7 | wpb 1943.9 | bsz 93.9 | num_updates 64259 | best_loss 0.108
+epoch 013 | valid on 'valid' subset | loss 0.108 | ppl 1.08 | wps 21359.7 | wpb 1943.9 | bsz 93.9 | num_updates 64259 | best_loss 0.108
+epoch 013 | valid on 'valid' subset | loss 0.108 | ppl 1.08 | wps 21359.7 | wpb 1943.9 | bsz 93.9 | num_updates 64259 | best_loss 0.108
+epoch 013 | valid on 'valid' subset | loss 0.108 | ppl 1.08 | wps 21359.7 | wpb 1943.9 | bsz 93.9 | num_updates 64259 | best_loss 0.108
+epoch 013 | valid on 'valid' subset | loss 0.108 | ppl 1.08 | wps 21359.7 | wpb 1943.9 | bsz 93.9 | num_updates 64259 | best_loss 0.108
+epoch 013 | valid on 'valid' subset | loss 0.108 | ppl 1.08 | wps 21359.7 | wpb 1943.9 | bsz 93.9 | num_updates 64259 | best_loss 0.108
+epoch 013 | valid on 'valid' subset | loss 0.108 | ppl 1.08 | wps 21359.7 | wpb 1943.9 | bsz 93.9 | num_updates 64259 | best_loss 0.108
+epoch 013 | valid on 'valid' subset | loss 0.108 | ppl 1.08 | wps 21359.7 | wpb 1943.9 | bsz 93.9 | num_updates 64259 | best_loss 0.108
+epoch 013 | valid on 'valid' subset | loss 0.108 | ppl 1.08 | wps 21359.7 | wpb 1943.9 | bsz 93.9 | num_updates 64259 | best_loss 0.108
+end of epoch 13 (average epoch stats below)
+epoch 013 | loss 0.012 | ppl 1.01 | wps 6506.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 64259 | lr 0.000124748 | gnorm 0.323 | train_wall 1474 | gb_free 13 | wall 19459
+epoch 013 | loss 0.012 | ppl 1.01 | wps 6506.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 64259 | lr 0.000124748 | gnorm 0.323 | train_wall 1474 | gb_free 13 | wall 19459
+epoch 013 | loss 0.012 | ppl 1.01 | wps 6506.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 64259 | lr 0.000124748 | gnorm 0.323 | train_wall 1474 | gb_free 13 | wall 19459
+epoch 013 | loss 0.012 | ppl 1.01 | wps 6506.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 64259 | lr 0.000124748 | gnorm 0.323 | train_wall 1474 | gb_free 13 | wall 19459
+epoch 013 | loss 0.012 | ppl 1.01 | wps 6506.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 64259 | lr 0.000124748 | gnorm 0.323 | train_wall 1474 | gb_free 13 | wall 19459
+epoch 013 | loss 0.012 | ppl 1.01 | wps 6506.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 64259 | lr 0.000124748 | gnorm 0.323 | train_wall 1474 | gb_free 13 | wall 19459
+epoch 013 | loss 0.012 | ppl 1.01 | wps 6506.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 64259 | lr 0.000124748 | gnorm 0.323 | train_wall 1474 | gb_free 13 | wall 19459
+epoch 013 | loss 0.012 | ppl 1.01 | wps 6506.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 64259 | lr 0.000124748 | gnorm 0.323 | train_wall 1474 | gb_free 13 | wall 19459
+epoch 013 | loss 0.012 | ppl 1.01 | wps 6506.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 64259 | lr 0.000124748 | gnorm 0.323 | train_wall 1474 | gb_free 13 | wall 19459
+epoch 013 | loss 0.012 | ppl 1.01 | wps 6506.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 64259 | lr 0.000124748 | gnorm 0.323 | train_wall 1474 | gb_free 13 | wall 19459
+epoch 013 | loss 0.012 | ppl 1.01 | wps 6506.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 64259 | lr 0.000124748 | gnorm 0.323 | train_wall 1474 | gb_free 13 | wall 19459
+epoch 013 | loss 0.012 | ppl 1.01 | wps 6506.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 64259 | lr 0.000124748 | gnorm 0.323 | train_wall 1474 | gb_free 13 | wall 19459
+epoch 013 | loss 0.012 | ppl 1.01 | wps 6506.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 64259 | lr 0.000124748 | gnorm 0.323 | train_wall 1474 | gb_free 13 | wall 19459
+Start iterating over samples
+begin validation on "valid" subset
+epoch 014 | valid on 'valid' subset | loss 0.113 | ppl 1.08 | wps 21403.9 | wpb 1943.9 | bsz 93.9 | num_updates 69202 | best_loss 0.108
+epoch 014 | valid on 'valid' subset | loss 0.113 | ppl 1.08 | wps 21403.9 | wpb 1943.9 | bsz 93.9 | num_updates 69202 | best_loss 0.108
+epoch 014 | valid on 'valid' subset | loss 0.113 | ppl 1.08 | wps 21403.9 | wpb 1943.9 | bsz 93.9 | num_updates 69202 | best_loss 0.108
+epoch 014 | valid on 'valid' subset | loss 0.113 | ppl 1.08 | wps 21403.9 | wpb 1943.9 | bsz 93.9 | num_updates 69202 | best_loss 0.108
+epoch 014 | valid on 'valid' subset | loss 0.113 | ppl 1.08 | wps 21403.9 | wpb 1943.9 | bsz 93.9 | num_updates 69202 | best_loss 0.108
+epoch 014 | valid on 'valid' subset | loss 0.113 | ppl 1.08 | wps 21403.9 | wpb 1943.9 | bsz 93.9 | num_updates 69202 | best_loss 0.108
+epoch 014 | valid on 'valid' subset | loss 0.113 | ppl 1.08 | wps 21403.9 | wpb 1943.9 | bsz 93.9 | num_updates 69202 | best_loss 0.108
+epoch 014 | valid on 'valid' subset | loss 0.113 | ppl 1.08 | wps 21403.9 | wpb 1943.9 | bsz 93.9 | num_updates 69202 | best_loss 0.108
+epoch 014 | valid on 'valid' subset | loss 0.113 | ppl 1.08 | wps 21403.9 | wpb 1943.9 | bsz 93.9 | num_updates 69202 | best_loss 0.108
+epoch 014 | valid on 'valid' subset | loss 0.113 | ppl 1.08 | wps 21403.9 | wpb 1943.9 | bsz 93.9 | num_updates 69202 | best_loss 0.108
+epoch 014 | valid on 'valid' subset | loss 0.113 | ppl 1.08 | wps 21403.9 | wpb 1943.9 | bsz 93.9 | num_updates 69202 | best_loss 0.108
+epoch 014 | valid on 'valid' subset | loss 0.113 | ppl 1.08 | wps 21403.9 | wpb 1943.9 | bsz 93.9 | num_updates 69202 | best_loss 0.108
+epoch 014 | valid on 'valid' subset | loss 0.113 | ppl 1.08 | wps 21403.9 | wpb 1943.9 | bsz 93.9 | num_updates 69202 | best_loss 0.108
+epoch 014 | valid on 'valid' subset | loss 0.113 | ppl 1.08 | wps 21403.9 | wpb 1943.9 | bsz 93.9 | num_updates 69202 | best_loss 0.108
+end of epoch 14 (average epoch stats below)
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6512.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 69202 | lr 0.00012021 | gnorm 0.295 | train_wall 1473 | gb_free 12.8 | wall 20956
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6512.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 69202 | lr 0.00012021 | gnorm 0.295 | train_wall 1473 | gb_free 12.8 | wall 20956
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6512.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 69202 | lr 0.00012021 | gnorm 0.295 | train_wall 1473 | gb_free 12.8 | wall 20956
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6512.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 69202 | lr 0.00012021 | gnorm 0.295 | train_wall 1473 | gb_free 12.8 | wall 20956
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6512.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 69202 | lr 0.00012021 | gnorm 0.295 | train_wall 1473 | gb_free 12.8 | wall 20956
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6512.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 69202 | lr 0.00012021 | gnorm 0.295 | train_wall 1473 | gb_free 12.8 | wall 20956
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6512.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 69202 | lr 0.00012021 | gnorm 0.295 | train_wall 1473 | gb_free 12.8 | wall 20956
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6512.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 69202 | lr 0.00012021 | gnorm 0.295 | train_wall 1473 | gb_free 12.8 | wall 20956
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6512.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 69202 | lr 0.00012021 | gnorm 0.295 | train_wall 1473 | gb_free 12.8 | wall 20956
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6512.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 69202 | lr 0.00012021 | gnorm 0.295 | train_wall 1473 | gb_free 12.8 | wall 20956
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6512.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 69202 | lr 0.00012021 | gnorm 0.295 | train_wall 1473 | gb_free 12.8 | wall 20956
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6512.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 69202 | lr 0.00012021 | gnorm 0.295 | train_wall 1473 | gb_free 12.8 | wall 20956
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6512.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 69202 | lr 0.00012021 | gnorm 0.295 | train_wall 1473 | gb_free 12.8 | wall 20956
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6512.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 69202 | lr 0.00012021 | gnorm 0.295 | train_wall 1473 | gb_free 12.8 | wall 20956
+Start iterating over samples
+begin validation on "valid" subset
+epoch 015 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21394.9 | wpb 1943.9 | bsz 93.9 | num_updates 74145 | best_loss 0.106
+epoch 015 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21394.9 | wpb 1943.9 | bsz 93.9 | num_updates 74145 | best_loss 0.106
+epoch 015 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21394.9 | wpb 1943.9 | bsz 93.9 | num_updates 74145 | best_loss 0.106
+epoch 015 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21394.9 | wpb 1943.9 | bsz 93.9 | num_updates 74145 | best_loss 0.106
+epoch 015 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21394.9 | wpb 1943.9 | bsz 93.9 | num_updates 74145 | best_loss 0.106
+epoch 015 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21394.9 | wpb 1943.9 | bsz 93.9 | num_updates 74145 | best_loss 0.106
+epoch 015 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21394.9 | wpb 1943.9 | bsz 93.9 | num_updates 74145 | best_loss 0.106
+epoch 015 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21394.9 | wpb 1943.9 | bsz 93.9 | num_updates 74145 | best_loss 0.106
+epoch 015 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21394.9 | wpb 1943.9 | bsz 93.9 | num_updates 74145 | best_loss 0.106
+epoch 015 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21394.9 | wpb 1943.9 | bsz 93.9 | num_updates 74145 | best_loss 0.106
+epoch 015 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21394.9 | wpb 1943.9 | bsz 93.9 | num_updates 74145 | best_loss 0.106
+epoch 015 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21394.9 | wpb 1943.9 | bsz 93.9 | num_updates 74145 | best_loss 0.106
+epoch 015 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21394.9 | wpb 1943.9 | bsz 93.9 | num_updates 74145 | best_loss 0.106
+epoch 015 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21394.9 | wpb 1943.9 | bsz 93.9 | num_updates 74145 | best_loss 0.106
+epoch 015 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21394.9 | wpb 1943.9 | bsz 93.9 | num_updates 74145 | best_loss 0.106
+end of epoch 15 (average epoch stats below)
+epoch 015 | loss 0.009 | ppl 1.01 | wps 6509.6 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 74145 | lr 0.000116134 | gnorm 0.276 | train_wall 1473 | gb_free 12.8 | wall 22453
+epoch 015 | loss 0.009 | ppl 1.01 | wps 6509.6 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 74145 | lr 0.000116134 | gnorm 0.276 | train_wall 1473 | gb_free 12.8 | wall 22453
+epoch 015 | loss 0.009 | ppl 1.01 | wps 6509.6 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 74145 | lr 0.000116134 | gnorm 0.276 | train_wall 1473 | gb_free 12.8 | wall 22453
+epoch 015 | loss 0.009 | ppl 1.01 | wps 6509.6 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 74145 | lr 0.000116134 | gnorm 0.276 | train_wall 1473 | gb_free 12.8 | wall 22453
+epoch 015 | loss 0.009 | ppl 1.01 | wps 6509.6 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 74145 | lr 0.000116134 | gnorm 0.276 | train_wall 1473 | gb_free 12.8 | wall 22453
+epoch 015 | loss 0.009 | ppl 1.01 | wps 6509.6 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 74145 | lr 0.000116134 | gnorm 0.276 | train_wall 1473 | gb_free 12.8 | wall 22453
+epoch 015 | loss 0.009 | ppl 1.01 | wps 6509.6 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 74145 | lr 0.000116134 | gnorm 0.276 | train_wall 1473 | gb_free 12.8 | wall 22453
+epoch 015 | loss 0.009 | ppl 1.01 | wps 6509.6 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 74145 | lr 0.000116134 | gnorm 0.276 | train_wall 1473 | gb_free 12.8 | wall 22453
+epoch 015 | loss 0.009 | ppl 1.01 | wps 6509.6 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 74145 | lr 0.000116134 | gnorm 0.276 | train_wall 1473 | gb_free 12.8 | wall 22453
+epoch 015 | loss 0.009 | ppl 1.01 | wps 6509.6 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 74145 | lr 0.000116134 | gnorm 0.276 | train_wall 1473 | gb_free 12.8 | wall 22453
+epoch 015 | loss 0.009 | ppl 1.01 | wps 6509.6 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 74145 | lr 0.000116134 | gnorm 0.276 | train_wall 1473 | gb_free 12.8 | wall 22453
+epoch 015 | loss 0.009 | ppl 1.01 | wps 6509.6 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 74145 | lr 0.000116134 | gnorm 0.276 | train_wall 1473 | gb_free 12.8 | wall 22453
+epoch 015 | loss 0.009 | ppl 1.01 | wps 6509.6 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 74145 | lr 0.000116134 | gnorm 0.276 | train_wall 1473 | gb_free 12.8 | wall 22453
+epoch 015 | loss 0.009 | ppl 1.01 | wps 6509.6 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 74145 | lr 0.000116134 | gnorm 0.276 | train_wall 1473 | gb_free 12.8 | wall 22453
+epoch 015 | loss 0.009 | ppl 1.01 | wps 6509.6 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 74145 | lr 0.000116134 | gnorm 0.276 | train_wall 1473 | gb_free 12.8 | wall 22453
+Start iterating over samples
+begin validation on "valid" subset
+epoch 016 | valid on 'valid' subset | loss 0.102 | ppl 1.07 | wps 21274.8 | wpb 1943.9 | bsz 93.9 | num_updates 79088 | best_loss 0.102
+epoch 016 | valid on 'valid' subset | loss 0.102 | ppl 1.07 | wps 21274.8 | wpb 1943.9 | bsz 93.9 | num_updates 79088 | best_loss 0.102
+epoch 016 | valid on 'valid' subset | loss 0.102 | ppl 1.07 | wps 21274.8 | wpb 1943.9 | bsz 93.9 | num_updates 79088 | best_loss 0.102
+epoch 016 | valid on 'valid' subset | loss 0.102 | ppl 1.07 | wps 21274.8 | wpb 1943.9 | bsz 93.9 | num_updates 79088 | best_loss 0.102
+epoch 016 | valid on 'valid' subset | loss 0.102 | ppl 1.07 | wps 21274.8 | wpb 1943.9 | bsz 93.9 | num_updates 79088 | best_loss 0.102
+epoch 016 | valid on 'valid' subset | loss 0.102 | ppl 1.07 | wps 21274.8 | wpb 1943.9 | bsz 93.9 | num_updates 79088 | best_loss 0.102
+epoch 016 | valid on 'valid' subset | loss 0.102 | ppl 1.07 | wps 21274.8 | wpb 1943.9 | bsz 93.9 | num_updates 79088 | best_loss 0.102
+epoch 016 | valid on 'valid' subset | loss 0.102 | ppl 1.07 | wps 21274.8 | wpb 1943.9 | bsz 93.9 | num_updates 79088 | best_loss 0.102
+epoch 016 | valid on 'valid' subset | loss 0.102 | ppl 1.07 | wps 21274.8 | wpb 1943.9 | bsz 93.9 | num_updates 79088 | best_loss 0.102
+epoch 016 | valid on 'valid' subset | loss 0.102 | ppl 1.07 | wps 21274.8 | wpb 1943.9 | bsz 93.9 | num_updates 79088 | best_loss 0.102
+epoch 016 | valid on 'valid' subset | loss 0.102 | ppl 1.07 | wps 21274.8 | wpb 1943.9 | bsz 93.9 | num_updates 79088 | best_loss 0.102
+epoch 016 | valid on 'valid' subset | loss 0.102 | ppl 1.07 | wps 21274.8 | wpb 1943.9 | bsz 93.9 | num_updates 79088 | best_loss 0.102
+epoch 016 | valid on 'valid' subset | loss 0.102 | ppl 1.07 | wps 21274.8 | wpb 1943.9 | bsz 93.9 | num_updates 79088 | best_loss 0.102
+epoch 016 | valid on 'valid' subset | loss 0.102 | ppl 1.07 | wps 21274.8 | wpb 1943.9 | bsz 93.9 | num_updates 79088 | best_loss 0.102
+epoch 016 | valid on 'valid' subset | loss 0.102 | ppl 1.07 | wps 21274.8 | wpb 1943.9 | bsz 93.9 | num_updates 79088 | best_loss 0.102
+epoch 016 | valid on 'valid' subset | loss 0.102 | ppl 1.07 | wps 21274.8 | wpb 1943.9 | bsz 93.9 | num_updates 79088 | best_loss 0.102
+end of epoch 16 (average epoch stats below)
+epoch 016 | loss 0.008 | ppl 1.01 | wps 6507.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 79088 | lr 0.000112446 | gnorm 0.268 | train_wall 1473 | gb_free 12.9 | wall 23951
+epoch 016 | loss 0.008 | ppl 1.01 | wps 6507.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 79088 | lr 0.000112446 | gnorm 0.268 | train_wall 1473 | gb_free 12.9 | wall 23951
+epoch 016 | loss 0.008 | ppl 1.01 | wps 6507.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 79088 | lr 0.000112446 | gnorm 0.268 | train_wall 1473 | gb_free 12.9 | wall 23951
+epoch 016 | loss 0.008 | ppl 1.01 | wps 6507.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 79088 | lr 0.000112446 | gnorm 0.268 | train_wall 1473 | gb_free 12.9 | wall 23951
+epoch 016 | loss 0.008 | ppl 1.01 | wps 6507.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 79088 | lr 0.000112446 | gnorm 0.268 | train_wall 1473 | gb_free 12.9 | wall 23951
+epoch 016 | loss 0.008 | ppl 1.01 | wps 6507.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 79088 | lr 0.000112446 | gnorm 0.268 | train_wall 1473 | gb_free 12.9 | wall 23951
+epoch 016 | loss 0.008 | ppl 1.01 | wps 6507.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 79088 | lr 0.000112446 | gnorm 0.268 | train_wall 1473 | gb_free 12.9 | wall 23951
+epoch 016 | loss 0.008 | ppl 1.01 | wps 6507.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 79088 | lr 0.000112446 | gnorm 0.268 | train_wall 1473 | gb_free 12.9 | wall 23951
+epoch 016 | loss 0.008 | ppl 1.01 | wps 6507.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 79088 | lr 0.000112446 | gnorm 0.268 | train_wall 1473 | gb_free 12.9 | wall 23951
+epoch 016 | loss 0.008 | ppl 1.01 | wps 6507.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 79088 | lr 0.000112446 | gnorm 0.268 | train_wall 1473 | gb_free 12.9 | wall 23951
+epoch 016 | loss 0.008 | ppl 1.01 | wps 6507.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 79088 | lr 0.000112446 | gnorm 0.268 | train_wall 1473 | gb_free 12.9 | wall 23951
+epoch 016 | loss 0.008 | ppl 1.01 | wps 6507.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 79088 | lr 0.000112446 | gnorm 0.268 | train_wall 1473 | gb_free 12.9 | wall 23951
+epoch 016 | loss 0.008 | ppl 1.01 | wps 6507.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 79088 | lr 0.000112446 | gnorm 0.268 | train_wall 1473 | gb_free 12.9 | wall 23951
+epoch 016 | loss 0.008 | ppl 1.01 | wps 6507.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 79088 | lr 0.000112446 | gnorm 0.268 | train_wall 1473 | gb_free 12.9 | wall 23951
+epoch 016 | loss 0.008 | ppl 1.01 | wps 6507.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 79088 | lr 0.000112446 | gnorm 0.268 | train_wall 1473 | gb_free 12.9 | wall 23951
+epoch 016 | loss 0.008 | ppl 1.01 | wps 6507.7 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 79088 | lr 0.000112446 | gnorm 0.268 | train_wall 1473 | gb_free 12.9 | wall 23951
+Start iterating over samples
+begin validation on "valid" subset
+epoch 017 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21457.6 | wpb 1943.9 | bsz 93.9 | num_updates 84031 | best_loss 0.102
+epoch 017 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21457.6 | wpb 1943.9 | bsz 93.9 | num_updates 84031 | best_loss 0.102
+epoch 017 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21457.6 | wpb 1943.9 | bsz 93.9 | num_updates 84031 | best_loss 0.102
+epoch 017 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21457.6 | wpb 1943.9 | bsz 93.9 | num_updates 84031 | best_loss 0.102
+epoch 017 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21457.6 | wpb 1943.9 | bsz 93.9 | num_updates 84031 | best_loss 0.102
+epoch 017 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21457.6 | wpb 1943.9 | bsz 93.9 | num_updates 84031 | best_loss 0.102
+epoch 017 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21457.6 | wpb 1943.9 | bsz 93.9 | num_updates 84031 | best_loss 0.102
+epoch 017 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21457.6 | wpb 1943.9 | bsz 93.9 | num_updates 84031 | best_loss 0.102
+epoch 017 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21457.6 | wpb 1943.9 | bsz 93.9 | num_updates 84031 | best_loss 0.102
+epoch 017 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21457.6 | wpb 1943.9 | bsz 93.9 | num_updates 84031 | best_loss 0.102
+epoch 017 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21457.6 | wpb 1943.9 | bsz 93.9 | num_updates 84031 | best_loss 0.102
+epoch 017 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21457.6 | wpb 1943.9 | bsz 93.9 | num_updates 84031 | best_loss 0.102
+epoch 017 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21457.6 | wpb 1943.9 | bsz 93.9 | num_updates 84031 | best_loss 0.102
+epoch 017 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21457.6 | wpb 1943.9 | bsz 93.9 | num_updates 84031 | best_loss 0.102
+epoch 017 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21457.6 | wpb 1943.9 | bsz 93.9 | num_updates 84031 | best_loss 0.102
+epoch 017 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21457.6 | wpb 1943.9 | bsz 93.9 | num_updates 84031 | best_loss 0.102
+epoch 017 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21457.6 | wpb 1943.9 | bsz 93.9 | num_updates 84031 | best_loss 0.102
+end of epoch 17 (average epoch stats below)
+epoch 017 | loss 0.007 | ppl 1.01 | wps 6509.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 84031 | lr 0.000109089 | gnorm 0.246 | train_wall 1473 | gb_free 13 | wall 25448
+epoch 017 | loss 0.007 | ppl 1.01 | wps 6509.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 84031 | lr 0.000109089 | gnorm 0.246 | train_wall 1473 | gb_free 13 | wall 25448
+epoch 017 | loss 0.007 | ppl 1.01 | wps 6509.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 84031 | lr 0.000109089 | gnorm 0.246 | train_wall 1473 | gb_free 13 | wall 25448
+epoch 017 | loss 0.007 | ppl 1.01 | wps 6509.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 84031 | lr 0.000109089 | gnorm 0.246 | train_wall 1473 | gb_free 13 | wall 25448
+epoch 017 | loss 0.007 | ppl 1.01 | wps 6509.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 84031 | lr 0.000109089 | gnorm 0.246 | train_wall 1473 | gb_free 13 | wall 25448
+epoch 017 | loss 0.007 | ppl 1.01 | wps 6509.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 84031 | lr 0.000109089 | gnorm 0.246 | train_wall 1473 | gb_free 13 | wall 25448
+epoch 017 | loss 0.007 | ppl 1.01 | wps 6509.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 84031 | lr 0.000109089 | gnorm 0.246 | train_wall 1473 | gb_free 13 | wall 25448
+epoch 017 | loss 0.007 | ppl 1.01 | wps 6509.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 84031 | lr 0.000109089 | gnorm 0.246 | train_wall 1473 | gb_free 13 | wall 25448
+epoch 017 | loss 0.007 | ppl 1.01 | wps 6509.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 84031 | lr 0.000109089 | gnorm 0.246 | train_wall 1473 | gb_free 13 | wall 25448
+epoch 017 | loss 0.007 | ppl 1.01 | wps 6509.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 84031 | lr 0.000109089 | gnorm 0.246 | train_wall 1473 | gb_free 13 | wall 25448
+epoch 017 | loss 0.007 | ppl 1.01 | wps 6509.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 84031 | lr 0.000109089 | gnorm 0.246 | train_wall 1473 | gb_free 13 | wall 25448
+epoch 017 | loss 0.007 | ppl 1.01 | wps 6509.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 84031 | lr 0.000109089 | gnorm 0.246 | train_wall 1473 | gb_free 13 | wall 25448
+epoch 017 | loss 0.007 | ppl 1.01 | wps 6509.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 84031 | lr 0.000109089 | gnorm 0.246 | train_wall 1473 | gb_free 13 | wall 25448
+epoch 017 | loss 0.007 | ppl 1.01 | wps 6509.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 84031 | lr 0.000109089 | gnorm 0.246 | train_wall 1473 | gb_free 13 | wall 25448
+epoch 017 | loss 0.007 | ppl 1.01 | wps 6509.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 84031 | lr 0.000109089 | gnorm 0.246 | train_wall 1473 | gb_free 13 | wall 25448
+epoch 017 | loss 0.007 | ppl 1.01 | wps 6509.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 84031 | lr 0.000109089 | gnorm 0.246 | train_wall 1473 | gb_free 13 | wall 25448
+epoch 017 | loss 0.007 | ppl 1.01 | wps 6509.5 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 84031 | lr 0.000109089 | gnorm 0.246 | train_wall 1473 | gb_free 13 | wall 25448
+Start iterating over samples
+begin validation on "valid" subset
+epoch 018 | valid on 'valid' subset | loss 0.097 | ppl 1.07 | wps 21448.5 | wpb 1943.9 | bsz 93.9 | num_updates 88974 | best_loss 0.097
+epoch 018 | valid on 'valid' subset | loss 0.097 | ppl 1.07 | wps 21448.5 | wpb 1943.9 | bsz 93.9 | num_updates 88974 | best_loss 0.097
+epoch 018 | valid on 'valid' subset | loss 0.097 | ppl 1.07 | wps 21448.5 | wpb 1943.9 | bsz 93.9 | num_updates 88974 | best_loss 0.097
+epoch 018 | valid on 'valid' subset | loss 0.097 | ppl 1.07 | wps 21448.5 | wpb 1943.9 | bsz 93.9 | num_updates 88974 | best_loss 0.097
+epoch 018 | valid on 'valid' subset | loss 0.097 | ppl 1.07 | wps 21448.5 | wpb 1943.9 | bsz 93.9 | num_updates 88974 | best_loss 0.097
+epoch 018 | valid on 'valid' subset | loss 0.097 | ppl 1.07 | wps 21448.5 | wpb 1943.9 | bsz 93.9 | num_updates 88974 | best_loss 0.097
+epoch 018 | valid on 'valid' subset | loss 0.097 | ppl 1.07 | wps 21448.5 | wpb 1943.9 | bsz 93.9 | num_updates 88974 | best_loss 0.097
+epoch 018 | valid on 'valid' subset | loss 0.097 | ppl 1.07 | wps 21448.5 | wpb 1943.9 | bsz 93.9 | num_updates 88974 | best_loss 0.097
+epoch 018 | valid on 'valid' subset | loss 0.097 | ppl 1.07 | wps 21448.5 | wpb 1943.9 | bsz 93.9 | num_updates 88974 | best_loss 0.097
+epoch 018 | valid on 'valid' subset | loss 0.097 | ppl 1.07 | wps 21448.5 | wpb 1943.9 | bsz 93.9 | num_updates 88974 | best_loss 0.097
+epoch 018 | valid on 'valid' subset | loss 0.097 | ppl 1.07 | wps 21448.5 | wpb 1943.9 | bsz 93.9 | num_updates 88974 | best_loss 0.097
+epoch 018 | valid on 'valid' subset | loss 0.097 | ppl 1.07 | wps 21448.5 | wpb 1943.9 | bsz 93.9 | num_updates 88974 | best_loss 0.097
+epoch 018 | valid on 'valid' subset | loss 0.097 | ppl 1.07 | wps 21448.5 | wpb 1943.9 | bsz 93.9 | num_updates 88974 | best_loss 0.097
+epoch 018 | valid on 'valid' subset | loss 0.097 | ppl 1.07 | wps 21448.5 | wpb 1943.9 | bsz 93.9 | num_updates 88974 | best_loss 0.097
+epoch 018 | valid on 'valid' subset | loss 0.097 | ppl 1.07 | wps 21448.5 | wpb 1943.9 | bsz 93.9 | num_updates 88974 | best_loss 0.097
+epoch 018 | valid on 'valid' subset | loss 0.097 | ppl 1.07 | wps 21448.5 | wpb 1943.9 | bsz 93.9 | num_updates 88974 | best_loss 0.097
+epoch 018 | valid on 'valid' subset | loss 0.097 | ppl 1.07 | wps 21448.5 | wpb 1943.9 | bsz 93.9 | num_updates 88974 | best_loss 0.097
+epoch 018 | valid on 'valid' subset | loss 0.097 | ppl 1.07 | wps 21448.5 | wpb 1943.9 | bsz 93.9 | num_updates 88974 | best_loss 0.097
+end of epoch 18 (average epoch stats below)
+epoch 018 | loss 0.007 | ppl 1 | wps 6510.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 88974 | lr 0.000106015 | gnorm 0.238 | train_wall 1473 | gb_free 12.9 | wall 26945
+epoch 018 | loss 0.007 | ppl 1 | wps 6510.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 88974 | lr 0.000106015 | gnorm 0.238 | train_wall 1473 | gb_free 12.9 | wall 26945
+epoch 018 | loss 0.007 | ppl 1 | wps 6510.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 88974 | lr 0.000106015 | gnorm 0.238 | train_wall 1473 | gb_free 12.9 | wall 26945
+epoch 018 | loss 0.007 | ppl 1 | wps 6510.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 88974 | lr 0.000106015 | gnorm 0.238 | train_wall 1473 | gb_free 12.9 | wall 26945
+epoch 018 | loss 0.007 | ppl 1 | wps 6510.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 88974 | lr 0.000106015 | gnorm 0.238 | train_wall 1473 | gb_free 12.9 | wall 26945
+epoch 018 | loss 0.007 | ppl 1 | wps 6510.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 88974 | lr 0.000106015 | gnorm 0.238 | train_wall 1473 | gb_free 12.9 | wall 26945
+epoch 018 | loss 0.007 | ppl 1 | wps 6510.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 88974 | lr 0.000106015 | gnorm 0.238 | train_wall 1473 | gb_free 12.9 | wall 26945
+epoch 018 | loss 0.007 | ppl 1 | wps 6510.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 88974 | lr 0.000106015 | gnorm 0.238 | train_wall 1473 | gb_free 12.9 | wall 26945
+epoch 018 | loss 0.007 | ppl 1 | wps 6510.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 88974 | lr 0.000106015 | gnorm 0.238 | train_wall 1473 | gb_free 12.9 | wall 26945
+epoch 018 | loss 0.007 | ppl 1 | wps 6510.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 88974 | lr 0.000106015 | gnorm 0.238 | train_wall 1473 | gb_free 12.9 | wall 26945
+epoch 018 | loss 0.007 | ppl 1 | wps 6510.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 88974 | lr 0.000106015 | gnorm 0.238 | train_wall 1473 | gb_free 12.9 | wall 26945
+epoch 018 | loss 0.007 | ppl 1 | wps 6510.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 88974 | lr 0.000106015 | gnorm 0.238 | train_wall 1473 | gb_free 12.9 | wall 26945
+epoch 018 | loss 0.007 | ppl 1 | wps 6510.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 88974 | lr 0.000106015 | gnorm 0.238 | train_wall 1473 | gb_free 12.9 | wall 26945
+epoch 018 | loss 0.007 | ppl 1 | wps 6510.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 88974 | lr 0.000106015 | gnorm 0.238 | train_wall 1473 | gb_free 12.9 | wall 26945
+epoch 018 | loss 0.007 | ppl 1 | wps 6510.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 88974 | lr 0.000106015 | gnorm 0.238 | train_wall 1473 | gb_free 12.9 | wall 26945
+epoch 018 | loss 0.007 | ppl 1 | wps 6510.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 88974 | lr 0.000106015 | gnorm 0.238 | train_wall 1473 | gb_free 12.9 | wall 26945
+epoch 018 | loss 0.007 | ppl 1 | wps 6510.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 88974 | lr 0.000106015 | gnorm 0.238 | train_wall 1473 | gb_free 12.9 | wall 26945
+epoch 018 | loss 0.007 | ppl 1 | wps 6510.3 | ups 3.3 | wpb 1971.8 | bsz 95.4 | num_updates 88974 | lr 0.000106015 | gnorm 0.238 | train_wall 1473 | gb_free 12.9 | wall 26945
+Start iterating over samples
+begin validation on "valid" subset
+epoch 019 | valid on 'valid' subset | loss 0.098 | ppl 1.07 | wps 21517.5 | wpb 1943.9 | bsz 93.9 | num_updates 93917 | best_loss 0.097
+epoch 019 | valid on 'valid' subset | loss 0.098 | ppl 1.07 | wps 21517.5 | wpb 1943.9 | bsz 93.9 | num_updates 93917 | best_loss 0.097
+epoch 019 | valid on 'valid' subset | loss 0.098 | ppl 1.07 | wps 21517.5 | wpb 1943.9 | bsz 93.9 | num_updates 93917 | best_loss 0.097
+epoch 019 | valid on 'valid' subset | loss 0.098 | ppl 1.07 | wps 21517.5 | wpb 1943.9 | bsz 93.9 | num_updates 93917 | best_loss 0.097
+epoch 019 | valid on 'valid' subset | loss 0.098 | ppl 1.07 | wps 21517.5 | wpb 1943.9 | bsz 93.9 | num_updates 93917 | best_loss 0.097
+epoch 019 | valid on 'valid' subset | loss 0.098 | ppl 1.07 | wps 21517.5 | wpb 1943.9 | bsz 93.9 | num_updates 93917 | best_loss 0.097
+epoch 019 | valid on 'valid' subset | loss 0.098 | ppl 1.07 | wps 21517.5 | wpb 1943.9 | bsz 93.9 | num_updates 93917 | best_loss 0.097
+epoch 019 | valid on 'valid' subset | loss 0.098 | ppl 1.07 | wps 21517.5 | wpb 1943.9 | bsz 93.9 | num_updates 93917 | best_loss 0.097
+epoch 019 | valid on 'valid' subset | loss 0.098 | ppl 1.07 | wps 21517.5 | wpb 1943.9 | bsz 93.9 | num_updates 93917 | best_loss 0.097
+epoch 019 | valid on 'valid' subset | loss 0.098 | ppl 1.07 | wps 21517.5 | wpb 1943.9 | bsz 93.9 | num_updates 93917 | best_loss 0.097
+epoch 019 | valid on 'valid' subset | loss 0.098 | ppl 1.07 | wps 21517.5 | wpb 1943.9 | bsz 93.9 | num_updates 93917 | best_loss 0.097
+epoch 019 | valid on 'valid' subset | loss 0.098 | ppl 1.07 | wps 21517.5 | wpb 1943.9 | bsz 93.9 | num_updates 93917 | best_loss 0.097
+epoch 019 | valid on 'valid' subset | loss 0.098 | ppl 1.07 | wps 21517.5 | wpb 1943.9 | bsz 93.9 | num_updates 93917 | best_loss 0.097
+epoch 019 | valid on 'valid' subset | loss 0.098 | ppl 1.07 | wps 21517.5 | wpb 1943.9 | bsz 93.9 | num_updates 93917 | best_loss 0.097
+epoch 019 | valid on 'valid' subset | loss 0.098 | ppl 1.07 | wps 21517.5 | wpb 1943.9 | bsz 93.9 | num_updates 93917 | best_loss 0.097
+epoch 019 | valid on 'valid' subset | loss 0.098 | ppl 1.07 | wps 21517.5 | wpb 1943.9 | bsz 93.9 | num_updates 93917 | best_loss 0.097
+epoch 019 | valid on 'valid' subset | loss 0.098 | ppl 1.07 | wps 21517.5 | wpb 1943.9 | bsz 93.9 | num_updates 93917 | best_loss 0.097
+epoch 019 | valid on 'valid' subset | loss 0.098 | ppl 1.07 | wps 21517.5 | wpb 1943.9 | bsz 93.9 | num_updates 93917 | best_loss 0.097
+epoch 019 | valid on 'valid' subset | loss 0.098 | ppl 1.07 | wps 21517.5 | wpb 1943.9 | bsz 93.9 | num_updates 93917 | best_loss 0.097
+end of epoch 19 (average epoch stats below)
+epoch 019 | loss 0.006 | ppl 1 | wps 6530.4 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 93917 | lr 0.000103188 | gnorm 0.226 | train_wall 1468 | gb_free 12.7 | wall 28438
+epoch 019 | loss 0.006 | ppl 1 | wps 6530.4 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 93917 | lr 0.000103188 | gnorm 0.226 | train_wall 1468 | gb_free 12.7 | wall 28438
+epoch 019 | loss 0.006 | ppl 1 | wps 6530.4 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 93917 | lr 0.000103188 | gnorm 0.226 | train_wall 1468 | gb_free 12.7 | wall 28438
+epoch 019 | loss 0.006 | ppl 1 | wps 6530.4 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 93917 | lr 0.000103188 | gnorm 0.226 | train_wall 1468 | gb_free 12.7 | wall 28438
+epoch 019 | loss 0.006 | ppl 1 | wps 6530.4 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 93917 | lr 0.000103188 | gnorm 0.226 | train_wall 1468 | gb_free 12.7 | wall 28438
+epoch 019 | loss 0.006 | ppl 1 | wps 6530.4 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 93917 | lr 0.000103188 | gnorm 0.226 | train_wall 1468 | gb_free 12.7 | wall 28438
+epoch 019 | loss 0.006 | ppl 1 | wps 6530.4 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 93917 | lr 0.000103188 | gnorm 0.226 | train_wall 1468 | gb_free 12.7 | wall 28438
+epoch 019 | loss 0.006 | ppl 1 | wps 6530.4 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 93917 | lr 0.000103188 | gnorm 0.226 | train_wall 1468 | gb_free 12.7 | wall 28438
+epoch 019 | loss 0.006 | ppl 1 | wps 6530.4 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 93917 | lr 0.000103188 | gnorm 0.226 | train_wall 1468 | gb_free 12.7 | wall 28438
+epoch 019 | loss 0.006 | ppl 1 | wps 6530.4 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 93917 | lr 0.000103188 | gnorm 0.226 | train_wall 1468 | gb_free 12.7 | wall 28438
+epoch 019 | loss 0.006 | ppl 1 | wps 6530.4 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 93917 | lr 0.000103188 | gnorm 0.226 | train_wall 1468 | gb_free 12.7 | wall 28438
+epoch 019 | loss 0.006 | ppl 1 | wps 6530.4 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 93917 | lr 0.000103188 | gnorm 0.226 | train_wall 1468 | gb_free 12.7 | wall 28438
+epoch 019 | loss 0.006 | ppl 1 | wps 6530.4 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 93917 | lr 0.000103188 | gnorm 0.226 | train_wall 1468 | gb_free 12.7 | wall 28438
+epoch 019 | loss 0.006 | ppl 1 | wps 6530.4 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 93917 | lr 0.000103188 | gnorm 0.226 | train_wall 1468 | gb_free 12.7 | wall 28438
+epoch 019 | loss 0.006 | ppl 1 | wps 6530.4 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 93917 | lr 0.000103188 | gnorm 0.226 | train_wall 1468 | gb_free 12.7 | wall 28438
+epoch 019 | loss 0.006 | ppl 1 | wps 6530.4 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 93917 | lr 0.000103188 | gnorm 0.226 | train_wall 1468 | gb_free 12.7 | wall 28438
+epoch 019 | loss 0.006 | ppl 1 | wps 6530.4 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 93917 | lr 0.000103188 | gnorm 0.226 | train_wall 1468 | gb_free 12.7 | wall 28438
+epoch 019 | loss 0.006 | ppl 1 | wps 6530.4 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 93917 | lr 0.000103188 | gnorm 0.226 | train_wall 1468 | gb_free 12.7 | wall 28438
+epoch 019 | loss 0.006 | ppl 1 | wps 6530.4 | ups 3.31 | wpb 1971.8 | bsz 95.4 | num_updates 93917 | lr 0.000103188 | gnorm 0.226 | train_wall 1468 | gb_free 12.7 | wall 28438
+Start iterating over samples
+begin validation on "valid" subset
+epoch 020 | valid on 'valid' subset | loss 0.1 | ppl 1.07 | wps 21536.6 | wpb 1943.9 | bsz 93.9 | num_updates 98860 | best_loss 0.097
+epoch 020 | valid on 'valid' subset | loss 0.1 | ppl 1.07 | wps 21536.6 | wpb 1943.9 | bsz 93.9 | num_updates 98860 | best_loss 0.097
+epoch 020 | valid on 'valid' subset | loss 0.1 | ppl 1.07 | wps 21536.6 | wpb 1943.9 | bsz 93.9 | num_updates 98860 | best_loss 0.097
+epoch 020 | valid on 'valid' subset | loss 0.1 | ppl 1.07 | wps 21536.6 | wpb 1943.9 | bsz 93.9 | num_updates 98860 | best_loss 0.097
+epoch 020 | valid on 'valid' subset | loss 0.1 | ppl 1.07 | wps 21536.6 | wpb 1943.9 | bsz 93.9 | num_updates 98860 | best_loss 0.097
+epoch 020 | valid on 'valid' subset | loss 0.1 | ppl 1.07 | wps 21536.6 | wpb 1943.9 | bsz 93.9 | num_updates 98860 | best_loss 0.097
+epoch 020 | valid on 'valid' subset | loss 0.1 | ppl 1.07 | wps 21536.6 | wpb 1943.9 | bsz 93.9 | num_updates 98860 | best_loss 0.097
+epoch 020 | valid on 'valid' subset | loss 0.1 | ppl 1.07 | wps 21536.6 | wpb 1943.9 | bsz 93.9 | num_updates 98860 | best_loss 0.097
+epoch 020 | valid on 'valid' subset | loss 0.1 | ppl 1.07 | wps 21536.6 | wpb 1943.9 | bsz 93.9 | num_updates 98860 | best_loss 0.097
+epoch 020 | valid on 'valid' subset | loss 0.1 | ppl 1.07 | wps 21536.6 | wpb 1943.9 | bsz 93.9 | num_updates 98860 | best_loss 0.097
+epoch 020 | valid on 'valid' subset | loss 0.1 | ppl 1.07 | wps 21536.6 | wpb 1943.9 | bsz 93.9 | num_updates 98860 | best_loss 0.097
+epoch 020 | valid on 'valid' subset | loss 0.1 | ppl 1.07 | wps 21536.6 | wpb 1943.9 | bsz 93.9 | num_updates 98860 | best_loss 0.097
+epoch 020 | valid on 'valid' subset | loss 0.1 | ppl 1.07 | wps 21536.6 | wpb 1943.9 | bsz 93.9 | num_updates 98860 | best_loss 0.097
+epoch 020 | valid on 'valid' subset | loss 0.1 | ppl 1.07 | wps 21536.6 | wpb 1943.9 | bsz 93.9 | num_updates 98860 | best_loss 0.097
+epoch 020 | valid on 'valid' subset | loss 0.1 | ppl 1.07 | wps 21536.6 | wpb 1943.9 | bsz 93.9 | num_updates 98860 | best_loss 0.097
+epoch 020 | valid on 'valid' subset | loss 0.1 | ppl 1.07 | wps 21536.6 | wpb 1943.9 | bsz 93.9 | num_updates 98860 | best_loss 0.097
+epoch 020 | valid on 'valid' subset | loss 0.1 | ppl 1.07 | wps 21536.6 | wpb 1943.9 | bsz 93.9 | num_updates 98860 | best_loss 0.097
+epoch 020 | valid on 'valid' subset | loss 0.1 | ppl 1.07 | wps 21536.6 | wpb 1943.9 | bsz 93.9 | num_updates 98860 | best_loss 0.097
+epoch 020 | valid on 'valid' subset | loss 0.1 | ppl 1.07 | wps 21536.6 | wpb 1943.9 | bsz 93.9 | num_updates 98860 | best_loss 0.097
+epoch 020 | valid on 'valid' subset | loss 0.1 | ppl 1.07 | wps 21536.6 | wpb 1943.9 | bsz 93.9 | num_updates 98860 | best_loss 0.097
+end of epoch 20 (average epoch stats below)
+epoch 020 | loss 0.006 | ppl 1 | wps 6546.6 | ups 3.32 | wpb 1971.8 | bsz 95.4 | num_updates 98860 | lr 0.000100575 | gnorm 0.214 | train_wall 1465 | gb_free 13 | wall 29927
+epoch 020 | loss 0.006 | ppl 1 | wps 6546.6 | ups 3.32 | wpb 1971.8 | bsz 95.4 | num_updates 98860 | lr 0.000100575 | gnorm 0.214 | train_wall 1465 | gb_free 13 | wall 29927
+epoch 020 | loss 0.006 | ppl 1 | wps 6546.6 | ups 3.32 | wpb 1971.8 | bsz 95.4 | num_updates 98860 | lr 0.000100575 | gnorm 0.214 | train_wall 1465 | gb_free 13 | wall 29927
+epoch 020 | loss 0.006 | ppl 1 | wps 6546.6 | ups 3.32 | wpb 1971.8 | bsz 95.4 | num_updates 98860 | lr 0.000100575 | gnorm 0.214 | train_wall 1465 | gb_free 13 | wall 29927
+epoch 020 | loss 0.006 | ppl 1 | wps 6546.6 | ups 3.32 | wpb 1971.8 | bsz 95.4 | num_updates 98860 | lr 0.000100575 | gnorm 0.214 | train_wall 1465 | gb_free 13 | wall 29927
+epoch 020 | loss 0.006 | ppl 1 | wps 6546.6 | ups 3.32 | wpb 1971.8 | bsz 95.4 | num_updates 98860 | lr 0.000100575 | gnorm 0.214 | train_wall 1465 | gb_free 13 | wall 29927
+epoch 020 | loss 0.006 | ppl 1 | wps 6546.6 | ups 3.32 | wpb 1971.8 | bsz 95.4 | num_updates 98860 | lr 0.000100575 | gnorm 0.214 | train_wall 1465 | gb_free 13 | wall 29927
+epoch 020 | loss 0.006 | ppl 1 | wps 6546.6 | ups 3.32 | wpb 1971.8 | bsz 95.4 | num_updates 98860 | lr 0.000100575 | gnorm 0.214 | train_wall 1465 | gb_free 13 | wall 29927
+epoch 020 | loss 0.006 | ppl 1 | wps 6546.6 | ups 3.32 | wpb 1971.8 | bsz 95.4 | num_updates 98860 | lr 0.000100575 | gnorm 0.214 | train_wall 1465 | gb_free 13 | wall 29927
+epoch 020 | loss 0.006 | ppl 1 | wps 6546.6 | ups 3.32 | wpb 1971.8 | bsz 95.4 | num_updates 98860 | lr 0.000100575 | gnorm 0.214 | train_wall 1465 | gb_free 13 | wall 29927
+epoch 020 | loss 0.006 | ppl 1 | wps 6546.6 | ups 3.32 | wpb 1971.8 | bsz 95.4 | num_updates 98860 | lr 0.000100575 | gnorm 0.214 | train_wall 1465 | gb_free 13 | wall 29927
+epoch 020 | loss 0.006 | ppl 1 | wps 6546.6 | ups 3.32 | wpb 1971.8 | bsz 95.4 | num_updates 98860 | lr 0.000100575 | gnorm 0.214 | train_wall 1465 | gb_free 13 | wall 29927
+epoch 020 | loss 0.006 | ppl 1 | wps 6546.6 | ups 3.32 | wpb 1971.8 | bsz 95.4 | num_updates 98860 | lr 0.000100575 | gnorm 0.214 | train_wall 1465 | gb_free 13 | wall 29927
+epoch 020 | loss 0.006 | ppl 1 | wps 6546.6 | ups 3.32 | wpb 1971.8 | bsz 95.4 | num_updates 98860 | lr 0.000100575 | gnorm 0.214 | train_wall 1465 | gb_free 13 | wall 29927
+epoch 020 | loss 0.006 | ppl 1 | wps 6546.6 | ups 3.32 | wpb 1971.8 | bsz 95.4 | num_updates 98860 | lr 0.000100575 | gnorm 0.214 | train_wall 1465 | gb_free 13 | wall 29927
+epoch 020 | loss 0.006 | ppl 1 | wps 6546.6 | ups 3.32 | wpb 1971.8 | bsz 95.4 | num_updates 98860 | lr 0.000100575 | gnorm 0.214 | train_wall 1465 | gb_free 13 | wall 29927
+epoch 020 | loss 0.006 | ppl 1 | wps 6546.6 | ups 3.32 | wpb 1971.8 | bsz 95.4 | num_updates 98860 | lr 0.000100575 | gnorm 0.214 | train_wall 1465 | gb_free 13 | wall 29927
+epoch 020 | loss 0.006 | ppl 1 | wps 6546.6 | ups 3.32 | wpb 1971.8 | bsz 95.4 | num_updates 98860 | lr 0.000100575 | gnorm 0.214 | train_wall 1465 | gb_free 13 | wall 29927
+epoch 020 | loss 0.006 | ppl 1 | wps 6546.6 | ups 3.32 | wpb 1971.8 | bsz 95.4 | num_updates 98860 | lr 0.000100575 | gnorm 0.214 | train_wall 1465 | gb_free 13 | wall 29927
+epoch 020 | loss 0.006 | ppl 1 | wps 6546.6 | ups 3.32 | wpb 1971.8 | bsz 95.4 | num_updates 98860 | lr 0.000100575 | gnorm 0.214 | train_wall 1465 | gb_free 13 | wall 29927
+Start iterating over samples
+Stopping training due to num_updates: 100000 >= max_update: 100000
+begin validation on "valid" subset
+epoch 021 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21391.8 | wpb 1943.9 | bsz 93.9 | num_updates 100000 | best_loss 0.097
+epoch 021 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21391.8 | wpb 1943.9 | bsz 93.9 | num_updates 100000 | best_loss 0.097
+epoch 021 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21391.8 | wpb 1943.9 | bsz 93.9 | num_updates 100000 | best_loss 0.097
+epoch 021 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21391.8 | wpb 1943.9 | bsz 93.9 | num_updates 100000 | best_loss 0.097
+epoch 021 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21391.8 | wpb 1943.9 | bsz 93.9 | num_updates 100000 | best_loss 0.097
+epoch 021 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21391.8 | wpb 1943.9 | bsz 93.9 | num_updates 100000 | best_loss 0.097
+epoch 021 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21391.8 | wpb 1943.9 | bsz 93.9 | num_updates 100000 | best_loss 0.097
+epoch 021 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21391.8 | wpb 1943.9 | bsz 93.9 | num_updates 100000 | best_loss 0.097
+epoch 021 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21391.8 | wpb 1943.9 | bsz 93.9 | num_updates 100000 | best_loss 0.097
+epoch 021 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21391.8 | wpb 1943.9 | bsz 93.9 | num_updates 100000 | best_loss 0.097
+epoch 021 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21391.8 | wpb 1943.9 | bsz 93.9 | num_updates 100000 | best_loss 0.097
+epoch 021 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21391.8 | wpb 1943.9 | bsz 93.9 | num_updates 100000 | best_loss 0.097
+epoch 021 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21391.8 | wpb 1943.9 | bsz 93.9 | num_updates 100000 | best_loss 0.097
+epoch 021 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21391.8 | wpb 1943.9 | bsz 93.9 | num_updates 100000 | best_loss 0.097
+epoch 021 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21391.8 | wpb 1943.9 | bsz 93.9 | num_updates 100000 | best_loss 0.097
+epoch 021 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21391.8 | wpb 1943.9 | bsz 93.9 | num_updates 100000 | best_loss 0.097
+epoch 021 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21391.8 | wpb 1943.9 | bsz 93.9 | num_updates 100000 | best_loss 0.097
+epoch 021 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21391.8 | wpb 1943.9 | bsz 93.9 | num_updates 100000 | best_loss 0.097
+epoch 021 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21391.8 | wpb 1943.9 | bsz 93.9 | num_updates 100000 | best_loss 0.097
+epoch 021 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21391.8 | wpb 1943.9 | bsz 93.9 | num_updates 100000 | best_loss 0.097
+epoch 021 | valid on 'valid' subset | loss 0.106 | ppl 1.08 | wps 21391.8 | wpb 1943.9 | bsz 93.9 | num_updates 100000 | best_loss 0.097
+end of epoch 21 (average epoch stats below)
+epoch 021 | loss 0.006 | ppl 1 | wps 6330.8 | ups 3.22 | wpb 1963.6 | bsz 95 | num_updates 100000 | lr 0.0001 | gnorm 0.235 | train_wall 339 | gb_free 12.5 | wall 30280
+epoch 021 | loss 0.006 | ppl 1 | wps 6330.8 | ups 3.22 | wpb 1963.6 | bsz 95 | num_updates 100000 | lr 0.0001 | gnorm 0.235 | train_wall 339 | gb_free 12.5 | wall 30280
+epoch 021 | loss 0.006 | ppl 1 | wps 6330.8 | ups 3.22 | wpb 1963.6 | bsz 95 | num_updates 100000 | lr 0.0001 | gnorm 0.235 | train_wall 339 | gb_free 12.5 | wall 30280
+epoch 021 | loss 0.006 | ppl 1 | wps 6330.8 | ups 3.22 | wpb 1963.6 | bsz 95 | num_updates 100000 | lr 0.0001 | gnorm 0.235 | train_wall 339 | gb_free 12.5 | wall 30280
+epoch 021 | loss 0.006 | ppl 1 | wps 6330.8 | ups 3.22 | wpb 1963.6 | bsz 95 | num_updates 100000 | lr 0.0001 | gnorm 0.235 | train_wall 339 | gb_free 12.5 | wall 30280
+epoch 021 | loss 0.006 | ppl 1 | wps 6330.8 | ups 3.22 | wpb 1963.6 | bsz 95 | num_updates 100000 | lr 0.0001 | gnorm 0.235 | train_wall 339 | gb_free 12.5 | wall 30280
+epoch 021 | loss 0.006 | ppl 1 | wps 6330.8 | ups 3.22 | wpb 1963.6 | bsz 95 | num_updates 100000 | lr 0.0001 | gnorm 0.235 | train_wall 339 | gb_free 12.5 | wall 30280
+epoch 021 | loss 0.006 | ppl 1 | wps 6330.8 | ups 3.22 | wpb 1963.6 | bsz 95 | num_updates 100000 | lr 0.0001 | gnorm 0.235 | train_wall 339 | gb_free 12.5 | wall 30280
+epoch 021 | loss 0.006 | ppl 1 | wps 6330.8 | ups 3.22 | wpb 1963.6 | bsz 95 | num_updates 100000 | lr 0.0001 | gnorm 0.235 | train_wall 339 | gb_free 12.5 | wall 30280
+epoch 021 | loss 0.006 | ppl 1 | wps 6330.8 | ups 3.22 | wpb 1963.6 | bsz 95 | num_updates 100000 | lr 0.0001 | gnorm 0.235 | train_wall 339 | gb_free 12.5 | wall 30280
+epoch 021 | loss 0.006 | ppl 1 | wps 6330.8 | ups 3.22 | wpb 1963.6 | bsz 95 | num_updates 100000 | lr 0.0001 | gnorm 0.235 | train_wall 339 | gb_free 12.5 | wall 30280
+epoch 021 | loss 0.006 | ppl 1 | wps 6330.8 | ups 3.22 | wpb 1963.6 | bsz 95 | num_updates 100000 | lr 0.0001 | gnorm 0.235 | train_wall 339 | gb_free 12.5 | wall 30280
+epoch 021 | loss 0.006 | ppl 1 | wps 6330.8 | ups 3.22 | wpb 1963.6 | bsz 95 | num_updates 100000 | lr 0.0001 | gnorm 0.235 | train_wall 339 | gb_free 12.5 | wall 30280
+epoch 021 | loss 0.006 | ppl 1 | wps 6330.8 | ups 3.22 | wpb 1963.6 | bsz 95 | num_updates 100000 | lr 0.0001 | gnorm 0.235 | train_wall 339 | gb_free 12.5 | wall 30280
+epoch 021 | loss 0.006 | ppl 1 | wps 6330.8 | ups 3.22 | wpb 1963.6 | bsz 95 | num_updates 100000 | lr 0.0001 | gnorm 0.235 | train_wall 339 | gb_free 12.5 | wall 30280
+epoch 021 | loss 0.006 | ppl 1 | wps 6330.8 | ups 3.22 | wpb 1963.6 | bsz 95 | num_updates 100000 | lr 0.0001 | gnorm 0.235 | train_wall 339 | gb_free 12.5 | wall 30280
+epoch 021 | loss 0.006 | ppl 1 | wps 6330.8 | ups 3.22 | wpb 1963.6 | bsz 95 | num_updates 100000 | lr 0.0001 | gnorm 0.235 | train_wall 339 | gb_free 12.5 | wall 30280
+epoch 021 | loss 0.006 | ppl 1 | wps 6330.8 | ups 3.22 | wpb 1963.6 | bsz 95 | num_updates 100000 | lr 0.0001 | gnorm 0.235 | train_wall 339 | gb_free 12.5 | wall 30280
+epoch 021 | loss 0.006 | ppl 1 | wps 6330.8 | ups 3.22 | wpb 1963.6 | bsz 95 | num_updates 100000 | lr 0.0001 | gnorm 0.235 | train_wall 339 | gb_free 12.5 | wall 30280
+epoch 021 | loss 0.006 | ppl 1 | wps 6330.8 | ups 3.22 | wpb 1963.6 | bsz 95 | num_updates 100000 | lr 0.0001 | gnorm 0.235 | train_wall 339 | gb_free 12.5 | wall 30280
+epoch 021 | loss 0.006 | ppl 1 | wps 6330.8 | ups 3.22 | wpb 1963.6 | bsz 95 | num_updates 100000 | lr 0.0001 | gnorm 0.235 | train_wall 339 | gb_free 12.5 | wall 30280
+done training in 30279.7 seconds

ablations/postfix/trained_arm32_best_postfix/beam1.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65006183aa175b82c3d02c31a6c5ae1b5990df997a5661640bf78cc42aa803c4
+size 2223373

ablations/postfix/trained_arm32_best_postfix/checkpoint_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b72d91990f0117dfdb5d61cceff6c279405e56ee786115a656584cdd9f0bae2
+size 124676620

ablations/postfix/trained_x64_best_postfix/beam1.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ef2c6fb16c63beccb0fda4a30a7b71ad213945996583cdfa7998e66e734bd67
+size 2732416

ablations/postfix/trained_x64_best_postfix/checkpoint_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:038881962540bf4707fb4ac19c18e7e3669f986feb243cf53267063de088ebc5
+size 186036468

ablations/postfix/trained_x64_best_postfix/training.log

@@ -0,0 +1,385 @@
+{'_name': None, 'common': {'_name': None, 'no_progress_bar': False, 'log_interval': 100, 'log_format': None, 'log_file': 'x64/trained_x64_best_postfix/training.log', 'aim_repo': None, 'aim_run_hash': None, 'tensorboard_logdir': None, 'wandb_project': None, 'azureml_logging': False, 'seed': 1, 'cpu': False, 'tpu': False, 'bf16': False, 'memory_efficient_bf16': False, 'fp16': False, 'memory_efficient_fp16': False, 'fp16_no_flatten_grads': False, 'fp16_init_scale': 128, 'fp16_scale_window': None, 'fp16_scale_tolerance': 0.0, 'on_cpu_convert_precision': False, 'min_loss_scale': 0.0001, 'threshold_loss_scale': None, 'amp': False, 'amp_batch_retries': 2, 'amp_init_scale': 128, 'amp_scale_window': None, 'user_dir': None, 'empty_cache_freq': 0, 'all_gather_list_size': 16384, 'model_parallel_size': 1, 'quantization_config_path': None, 'profile': False, 'reset_logging': False, 'suppress_crashes': False, 'use_plasma_view': False, 'plasma_path': '/tmp/plasma'}, 'common_eval': {'_name': None, 'path': None, 'post_process': None, 'quiet': False, 'model_overrides': '{}', 'results_path': None}, 'distributed_training': {'_name': None, 'distributed_world_size': 1, 'distributed_num_procs': 1, 'distributed_rank': 0, 'distributed_backend': 'nccl', 'distributed_init_method': None, 'distributed_port': -1, 'device_id': 0, 'distributed_no_spawn': False, 'ddp_backend': 'pytorch_ddp', 'ddp_comm_hook': 'none', 'bucket_cap_mb': 25, 'fix_batches_to_gpus': False, 'find_unused_parameters': False, 'gradient_as_bucket_view': False, 'fast_stat_sync': False, 'heartbeat_timeout': -1, 'broadcast_buffers': False, 'slowmo_momentum': None, 'slowmo_base_algorithm': 'localsgd', 'localsgd_frequency': 3, 'nprocs_per_node': 1, 'pipeline_model_parallel': False, 'pipeline_balance': None, 'pipeline_devices': None, 'pipeline_chunks': 0, 'pipeline_encoder_balance': None, 'pipeline_encoder_devices': None, 'pipeline_decoder_balance': None, 'pipeline_decoder_devices': None, 'pipeline_checkpoint': 'never', 'zero_sharding': 'none', 'fp16': False, 'memory_efficient_fp16': False, 'tpu': False, 'no_reshard_after_forward': False, 'fp32_reduce_scatter': False, 'cpu_offload': False, 'use_sharded_state': False, 'not_fsdp_flatten_parameters': False}, 'dataset': {'_name': None, 'num_workers': 1, 'skip_invalid_size_inputs_valid_test': False, 'max_tokens': 20000, 'batch_size': None, 'required_batch_size_multiple': 8, 'required_seq_len_multiple': 1, 'dataset_impl': None, 'data_buffer_size': 10, 'train_subset': 'train', 'valid_subset': 'valid', 'combine_valid_subsets': None, 'ignore_unused_valid_subsets': False, 'validate_interval': 1, 'validate_interval_updates': 0, 'validate_after_updates': 0, 'fixed_validation_seed': None, 'disable_validation': False, 'max_tokens_valid': 20000, 'batch_size_valid': None, 'max_valid_steps': None, 'curriculum': 0, 'gen_subset': 'test', 'num_shards': 1, 'shard_id': 0, 'grouped_shuffling': False, 'update_epoch_batch_itr': False, 'update_ordered_indices_seed': False}, 'optimization': {'_name': None, 'max_epoch': 0, 'max_update': 100000, 'stop_time_hours': 0.0, 'clip_norm': 0.0, 'sentence_avg': False, 'update_freq': [1], 'lr': [0.0005], 'stop_min_lr': -1.0, 'use_bmuf': False, 'skip_remainder_batch': False}, 'checkpoint': {'_name': None, 'save_dir': 'x64/trained_x64_best_postfix', 'restore_file': 'checkpoint_last.pt', 'continue_once': None, 'finetune_from_model': None, 'reset_dataloader': False, 'reset_lr_scheduler': False, 'reset_meters': False, 'reset_optimizer': False, 'optimizer_overrides': '{}', 'save_interval': 1, 'save_interval_updates': 0, 'keep_interval_updates': -1, 'keep_interval_updates_pattern': -1, 'keep_last_epochs': -1, 'keep_best_checkpoints': 3, 'no_save': False, 'no_epoch_checkpoints': True, 'no_last_checkpoints': False, 'no_save_optimizer_state': False, 'best_checkpoint_metric': 'loss', 'maximize_best_checkpoint_metric': False, 'patience': 4, 'checkpoint_suffix': '', 'checkpoint_shard_count': 1, 'load_checkpoint_on_all_dp_ranks': False, 'write_checkpoints_asynchronously': False, 'model_parallel_size': 1}, 'bmuf': {'_name': None, 'block_lr': 1.0, 'block_momentum': 0.875, 'global_sync_iter': 50, 'warmup_iterations': 500, 'use_nbm': False, 'average_sync': False, 'distributed_world_size': 1}, 'generation': {'_name': None, 'beam': 5, 'nbest': 1, 'max_len_a': 0.0, 'max_len_b': 200, 'min_len': 1, 'match_source_len': False, 'unnormalized': False, 'no_early_stop': False, 'no_beamable_mm': False, 'lenpen': 1.0, 'unkpen': 0.0, 'replace_unk': None, 'sacrebleu': False, 'score_reference': False, 'prefix_size': 0, 'no_repeat_ngram_size': 0, 'sampling': False, 'sampling_topk': -1, 'sampling_topp': -1.0, 'constraints': None, 'temperature': 1.0, 'diverse_beam_groups': -1, 'diverse_beam_strength': 0.5, 'diversity_rate': -1.0, 'print_alignment': None, 'print_step': False, 'lm_path': None, 'lm_weight': 0.0, 'iter_decode_eos_penalty': 0.0, 'iter_decode_max_iter': 10, 'iter_decode_force_max_iter': False, 'iter_decode_with_beam': 1, 'iter_decode_with_external_reranker': False, 'retain_iter_history': False, 'retain_dropout': False, 'retain_dropout_modules': None, 'decoding_format': None, 'no_seed_provided': False, 'eos_token': None}, 'eval_lm': {'_name': None, 'output_word_probs': False, 'output_word_stats': False, 'context_window': 0, 'softmax_batch': 9223372036854775807}, 'interactive': {'_name': None, 'buffer_size': 0, 'input': '-'}, 'model': Namespace(_name='transformer', activation_dropout=0.0, activation_fn='relu', adam_betas=(0.9, 0.999), adam_eps=1e-08, adaptive_input=False, adaptive_softmax_cutoff=None, adaptive_softmax_dropout=0, aim_repo=None, aim_run_hash=None, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, arch='transformer', attention_dropout=0.0, azureml_logging=False, batch_size=None, batch_size_valid=None, best_checkpoint_metric='loss', bf16=False, bpe=None, broadcast_buffers=False, bucket_cap_mb=25, checkpoint_activations=False, checkpoint_shard_count=1, checkpoint_suffix='', clip_norm=0.0, combine_valid_subsets=None, continue_once=None, cpu=False, cpu_offload=False, criterion='cross_entropy', cross_self_attention=False, curriculum=0, data='x64/tokenized_dlsm_x64_postfix', data_buffer_size=10, dataset_impl=None, ddp_backend='pytorch_ddp', ddp_comm_hook='none', decoder_attention_heads=8, decoder_embed_dim=96, decoder_embed_path=None, decoder_ffn_embed_dim=384, decoder_input_dim=96, decoder_layerdrop=0, decoder_layers=6, decoder_layers_to_keep=None, decoder_learned_pos=False, decoder_normalize_before=False, decoder_output_dim='96', device_id=0, disable_validation=False, distributed_backend='nccl', distributed_init_method=None, distributed_no_spawn=False, distributed_num_procs=1, distributed_port=-1, distributed_rank=0, distributed_world_size=1, dropout=0.05, ema_decay=0.9999, ema_fp32=False, ema_seed_model=None, ema_start_update=0, ema_update_freq=1, empty_cache_freq=0, encoder_attention_heads=8, encoder_embed_dim=384, encoder_embed_path=None, encoder_ffn_embed_dim=1536, encoder_layerdrop=0, encoder_layers=6, encoder_layers_to_keep=None, encoder_learned_pos=False, encoder_normalize_before=False, eos=2, eval_bleu=False, eval_bleu_args='{}', eval_bleu_detok='space', eval_bleu_detok_args='{}', eval_bleu_print_samples=False, eval_bleu_remove_bpe=None, eval_tokenized_bleu=False, fast_stat_sync=False, find_unused_parameters=False, finetune_from_model=None, fix_batches_to_gpus=False, fixed_validation_seed=None, fp16=False, fp16_adam_stats=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, fp32_reduce_scatter=False, gen_subset='test', gradient_as_bucket_view=False, grouped_shuffling=False, heartbeat_timeout=-1, ignore_unused_valid_subsets=False, keep_best_checkpoints=3, keep_interval_updates=-1, keep_interval_updates_pattern=-1, keep_last_epochs=-1, layernorm_embedding=False, left_pad_source=True, left_pad_target=False, load_alignments=False, load_checkpoint_on_all_dp_ranks=False, localsgd_frequency=3, log_file='x64/trained_x64_best_postfix/training.log', log_format=None, log_interval=100, lr=[0.0005], lr_scheduler='inverse_sqrt', max_epoch=0, max_source_positions=1024, max_target_positions=1024, max_tokens=20000, max_tokens_valid=20000, max_update=100000, max_valid_steps=None, maximize_best_checkpoint_metric=False, memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_cross_attention=False, no_epoch_checkpoints=True, no_last_checkpoints=False, no_progress_bar=False, no_reshard_after_forward=False, no_save=False, no_save_optimizer_state=False, no_scale_embedding=False, no_seed_provided=False, no_token_positional_embeddings=False, not_fsdp_flatten_parameters=False, nprocs_per_node=1, num_batch_buckets=0, num_shards=1, num_workers=1, offload_activations=False, on_cpu_convert_precision=False, optimizer='adam', optimizer_overrides='{}', pad=1, patience=4, pipeline_balance=None, pipeline_checkpoint='never', pipeline_chunks=0, pipeline_decoder_balance=None, pipeline_decoder_devices=None, pipeline_devices=None, pipeline_encoder_balance=None, pipeline_encoder_devices=None, pipeline_model_parallel=False, plasma_path='/tmp/plasma', profile=False, quant_noise_pq=0, quant_noise_pq_block_size=8, quant_noise_scalar=0, quantization_config_path=None, required_batch_size_multiple=8, required_seq_len_multiple=1, reset_dataloader=False, reset_logging=False, reset_lr_scheduler=False, reset_meters=False, reset_optimizer=False, restore_file='checkpoint_last.pt', save_dir='x64/trained_x64_best_postfix', save_interval=1, save_interval_updates=0, scoring='bleu', seed=1, sentence_avg=False, shard_id=0, share_all_embeddings=False, share_decoder_input_output_embed=False, skip_invalid_size_inputs_valid_test=False, skip_remainder_batch=False, slowmo_base_algorithm='localsgd', slowmo_momentum=None, source_lang=None, stop_min_lr=-1.0, stop_time_hours=0, store_ema=False, suppress_crashes=False, target_lang=None, task='translation', tensorboard_logdir=None, threshold_loss_scale=None, tie_adaptive_weights=False, tokenizer=None, tpu=False, train_subset='train', truncate_source=False, unk=3, update_epoch_batch_itr=False, update_freq=[1], update_ordered_indices_seed=False, upsample_primary=-1, use_bmuf=False, use_old_adam=False, use_plasma_view=False, use_sharded_state=False, user_dir=None, valid_subset='valid', validate_after_updates=0, validate_interval=1, validate_interval_updates=0, wandb_project=None, warmup_init_lr=-1, warmup_updates=4000, weight_decay=0.005, write_checkpoints_asynchronously=False, zero_sharding='none'), 'task': {'_name': 'translation', 'data': 'x64/tokenized_dlsm_x64_postfix', 'source_lang': None, 'target_lang': None, 'load_alignments': False, 'left_pad_source': True, 'left_pad_target': False, 'max_source_positions': 1024, 'max_target_positions': 1024, 'upsample_primary': -1, 'truncate_source': False, 'num_batch_buckets': 0, 'train_subset': 'train', 'dataset_impl': None, 'required_seq_len_multiple': 1, 'eval_bleu': False, 'eval_bleu_args': '{}', 'eval_bleu_detok': 'space', 'eval_bleu_detok_args': '{}', 'eval_tokenized_bleu': False, 'eval_bleu_remove_bpe': None, 'eval_bleu_print_samples': False}, 'criterion': {'_name': 'cross_entropy', 'sentence_avg': False}, 'optimizer': {'_name': 'adam', 'adam_betas': [0.9, 0.999], 'adam_eps': 1e-08, 'weight_decay': 0.005, 'use_old_adam': False, 'fp16_adam_stats': False, 'tpu': False, 'lr': [0.0005]}, 'lr_scheduler': {'_name': 'inverse_sqrt', 'warmup_updates': 4000, 'warmup_init_lr': -1.0, 'lr': [0.0005]}, 'scoring': {'_name': 'bleu', 'pad': 1, 'eos': 2, 'unk': 3}, 'bpe': None, 'tokenizer': None, 'ema': {'_name': None, 'store_ema': False, 'ema_decay': 0.9999, 'ema_start_update': 0, 'ema_seed_model': None, 'ema_update_freq': 1, 'ema_fp32': False}}
+TransformerModel(
+  (encoder): TransformerEncoderBase(
+    (dropout_module): FairseqDropout()
+    (embed_tokens): Embedding(136, 384, padding_idx=1)
+    (embed_positions): SinusoidalPositionalEmbedding()
+    (layers): ModuleList(
+      (0-5): 6 x TransformerEncoderLayerBase(
+        (self_attn): MultiheadAttention(
+          (dropout_module): FairseqDropout()
+          (k_proj): Linear(in_features=384, out_features=384, bias=True)
+          (v_proj): Linear(in_features=384, out_features=384, bias=True)
+          (q_proj): Linear(in_features=384, out_features=384, bias=True)
+          (out_proj): Linear(in_features=384, out_features=384, bias=True)
+        )
+        (self_attn_layer_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+        (dropout_module): FairseqDropout()
+        (activation_dropout_module): FairseqDropout()
+        (fc1): Linear(in_features=384, out_features=1536, bias=True)
+        (fc2): Linear(in_features=1536, out_features=384, bias=True)
+        (final_layer_norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)
+      )
+    )
+  )
+  (decoder): TransformerDecoderBase(
+    (dropout_module): FairseqDropout()
+    (embed_tokens): Embedding(40, 96, padding_idx=1)
+    (embed_positions): SinusoidalPositionalEmbedding()
+    (layers): ModuleList(
+      (0-5): 6 x TransformerDecoderLayerBase(
+        (dropout_module): FairseqDropout()
+        (self_attn): MultiheadAttention(
+          (dropout_module): FairseqDropout()
+          (k_proj): Linear(in_features=96, out_features=96, bias=True)
+          (v_proj): Linear(in_features=96, out_features=96, bias=True)
+          (q_proj): Linear(in_features=96, out_features=96, bias=True)
+          (out_proj): Linear(in_features=96, out_features=96, bias=True)
+        )
+        (activation_dropout_module): FairseqDropout()
+        (self_attn_layer_norm): LayerNorm((96,), eps=1e-05, elementwise_affine=True)
+        (encoder_attn): MultiheadAttention(
+          (dropout_module): FairseqDropout()
+          (k_proj): Linear(in_features=384, out_features=96, bias=True)
+          (v_proj): Linear(in_features=384, out_features=96, bias=True)
+          (q_proj): Linear(in_features=96, out_features=96, bias=True)
+          (out_proj): Linear(in_features=96, out_features=96, bias=True)
+        )
+        (encoder_attn_layer_norm): LayerNorm((96,), eps=1e-05, elementwise_affine=True)
+        (fc1): Linear(in_features=96, out_features=384, bias=True)
+        (fc2): Linear(in_features=384, out_features=96, bias=True)
+        (final_layer_norm): LayerNorm((96,), eps=1e-05, elementwise_affine=True)
+      )
+    )
+    (output_projection): Linear(in_features=96, out_features=40, bias=False)
+  )
+)
+task: TranslationTask
+model: TransformerModel
+criterion: CrossEntropyCriterion
+num. shared model params: 22,571,712 (num. trained: 22,571,712)
+num. expert model params: 0 (num. trained: 0)
+training on 1 devices (GPUs/TPUs)
+max tokens per device = 20000 and max sentences per device = None
+Start iterating over samples
+begin validation on "valid" subset
+epoch 001 | valid on 'valid' subset | loss 0.843 | ppl 1.79 | wps 21756.6 | wpb 1954.1 | bsz 93.8 | num_updates 4852
+end of epoch 1 (average epoch stats below)
+epoch 001 | loss 1.658 | ppl 3.16 | wps 6582.3 | ups 3.32 | wpb 1984.3 | bsz 95.9 | num_updates 4852 | lr 0.000453983 | gnorm 2.832 | train_wall 1440 | gb_free 13 | wall 1463
+Start iterating over samples
+begin validation on "valid" subset
+epoch 002 | valid on 'valid' subset | loss 0.391 | ppl 1.31 | wps 21776.9 | wpb 1954.1 | bsz 93.8 | num_updates 9704 | best_loss 0.391
+epoch 002 | valid on 'valid' subset | loss 0.391 | ppl 1.31 | wps 21776.9 | wpb 1954.1 | bsz 93.8 | num_updates 9704 | best_loss 0.391
+end of epoch 2 (average epoch stats below)
+epoch 002 | loss 0.56 | ppl 1.47 | wps 6573.7 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 9704 | lr 0.000321014 | gnorm 1.203 | train_wall 1441 | gb_free 12.8 | wall 2928
+epoch 002 | loss 0.56 | ppl 1.47 | wps 6573.7 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 9704 | lr 0.000321014 | gnorm 1.203 | train_wall 1441 | gb_free 12.8 | wall 2928
+Start iterating over samples
+begin validation on "valid" subset
+epoch 003 | valid on 'valid' subset | loss 0.242 | ppl 1.18 | wps 21774.3 | wpb 1954.1 | bsz 93.8 | num_updates 14556 | best_loss 0.242
+epoch 003 | valid on 'valid' subset | loss 0.242 | ppl 1.18 | wps 21774.3 | wpb 1954.1 | bsz 93.8 | num_updates 14556 | best_loss 0.242
+epoch 003 | valid on 'valid' subset | loss 0.242 | ppl 1.18 | wps 21774.3 | wpb 1954.1 | bsz 93.8 | num_updates 14556 | best_loss 0.242
+end of epoch 3 (average epoch stats below)
+epoch 003 | loss 0.277 | ppl 1.21 | wps 6573.3 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 14556 | lr 0.000262107 | gnorm 0.892 | train_wall 1441 | gb_free 12.9 | wall 4393
+epoch 003 | loss 0.277 | ppl 1.21 | wps 6573.3 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 14556 | lr 0.000262107 | gnorm 0.892 | train_wall 1441 | gb_free 12.9 | wall 4393
+epoch 003 | loss 0.277 | ppl 1.21 | wps 6573.3 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 14556 | lr 0.000262107 | gnorm 0.892 | train_wall 1441 | gb_free 12.9 | wall 4393
+Start iterating over samples
+begin validation on "valid" subset
+epoch 004 | valid on 'valid' subset | loss 0.144 | ppl 1.11 | wps 21742.1 | wpb 1954.1 | bsz 93.8 | num_updates 19408 | best_loss 0.144
+epoch 004 | valid on 'valid' subset | loss 0.144 | ppl 1.11 | wps 21742.1 | wpb 1954.1 | bsz 93.8 | num_updates 19408 | best_loss 0.144
+epoch 004 | valid on 'valid' subset | loss 0.144 | ppl 1.11 | wps 21742.1 | wpb 1954.1 | bsz 93.8 | num_updates 19408 | best_loss 0.144
+epoch 004 | valid on 'valid' subset | loss 0.144 | ppl 1.11 | wps 21742.1 | wpb 1954.1 | bsz 93.8 | num_updates 19408 | best_loss 0.144
+end of epoch 4 (average epoch stats below)
+epoch 004 | loss 0.151 | ppl 1.11 | wps 6570 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 19408 | lr 0.000226992 | gnorm 0.728 | train_wall 1442 | gb_free 13 | wall 5858
+epoch 004 | loss 0.151 | ppl 1.11 | wps 6570 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 19408 | lr 0.000226992 | gnorm 0.728 | train_wall 1442 | gb_free 13 | wall 5858
+epoch 004 | loss 0.151 | ppl 1.11 | wps 6570 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 19408 | lr 0.000226992 | gnorm 0.728 | train_wall 1442 | gb_free 13 | wall 5858
+epoch 004 | loss 0.151 | ppl 1.11 | wps 6570 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 19408 | lr 0.000226992 | gnorm 0.728 | train_wall 1442 | gb_free 13 | wall 5858
+Start iterating over samples
+begin validation on "valid" subset
+epoch 005 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21789.1 | wpb 1954.1 | bsz 93.8 | num_updates 24260 | best_loss 0.11
+epoch 005 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21789.1 | wpb 1954.1 | bsz 93.8 | num_updates 24260 | best_loss 0.11
+epoch 005 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21789.1 | wpb 1954.1 | bsz 93.8 | num_updates 24260 | best_loss 0.11
+epoch 005 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21789.1 | wpb 1954.1 | bsz 93.8 | num_updates 24260 | best_loss 0.11
+epoch 005 | valid on 'valid' subset | loss 0.11 | ppl 1.08 | wps 21789.1 | wpb 1954.1 | bsz 93.8 | num_updates 24260 | best_loss 0.11
+end of epoch 5 (average epoch stats below)
+epoch 005 | loss 0.087 | ppl 1.06 | wps 6567.9 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 24260 | lr 0.000203027 | gnorm 0.626 | train_wall 1443 | gb_free 12.8 | wall 7324
+epoch 005 | loss 0.087 | ppl 1.06 | wps 6567.9 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 24260 | lr 0.000203027 | gnorm 0.626 | train_wall 1443 | gb_free 12.8 | wall 7324
+epoch 005 | loss 0.087 | ppl 1.06 | wps 6567.9 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 24260 | lr 0.000203027 | gnorm 0.626 | train_wall 1443 | gb_free 12.8 | wall 7324
+epoch 005 | loss 0.087 | ppl 1.06 | wps 6567.9 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 24260 | lr 0.000203027 | gnorm 0.626 | train_wall 1443 | gb_free 12.8 | wall 7324
+epoch 005 | loss 0.087 | ppl 1.06 | wps 6567.9 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 24260 | lr 0.000203027 | gnorm 0.626 | train_wall 1443 | gb_free 12.8 | wall 7324
+Start iterating over samples
+begin validation on "valid" subset
+epoch 006 | valid on 'valid' subset | loss 0.104 | ppl 1.07 | wps 21782.7 | wpb 1954.1 | bsz 93.8 | num_updates 29112 | best_loss 0.104
+epoch 006 | valid on 'valid' subset | loss 0.104 | ppl 1.07 | wps 21782.7 | wpb 1954.1 | bsz 93.8 | num_updates 29112 | best_loss 0.104
+epoch 006 | valid on 'valid' subset | loss 0.104 | ppl 1.07 | wps 21782.7 | wpb 1954.1 | bsz 93.8 | num_updates 29112 | best_loss 0.104
+epoch 006 | valid on 'valid' subset | loss 0.104 | ppl 1.07 | wps 21782.7 | wpb 1954.1 | bsz 93.8 | num_updates 29112 | best_loss 0.104
+epoch 006 | valid on 'valid' subset | loss 0.104 | ppl 1.07 | wps 21782.7 | wpb 1954.1 | bsz 93.8 | num_updates 29112 | best_loss 0.104
+epoch 006 | valid on 'valid' subset | loss 0.104 | ppl 1.07 | wps 21782.7 | wpb 1954.1 | bsz 93.8 | num_updates 29112 | best_loss 0.104
+end of epoch 6 (average epoch stats below)
+epoch 006 | loss 0.055 | ppl 1.04 | wps 6564.9 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 29112 | lr 0.000185338 | gnorm 0.541 | train_wall 1443 | gb_free 13 | wall 8790
+epoch 006 | loss 0.055 | ppl 1.04 | wps 6564.9 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 29112 | lr 0.000185338 | gnorm 0.541 | train_wall 1443 | gb_free 13 | wall 8790
+epoch 006 | loss 0.055 | ppl 1.04 | wps 6564.9 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 29112 | lr 0.000185338 | gnorm 0.541 | train_wall 1443 | gb_free 13 | wall 8790
+epoch 006 | loss 0.055 | ppl 1.04 | wps 6564.9 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 29112 | lr 0.000185338 | gnorm 0.541 | train_wall 1443 | gb_free 13 | wall 8790
+epoch 006 | loss 0.055 | ppl 1.04 | wps 6564.9 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 29112 | lr 0.000185338 | gnorm 0.541 | train_wall 1443 | gb_free 13 | wall 8790
+epoch 006 | loss 0.055 | ppl 1.04 | wps 6564.9 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 29112 | lr 0.000185338 | gnorm 0.541 | train_wall 1443 | gb_free 13 | wall 8790
+Start iterating over samples
+begin validation on "valid" subset
+epoch 007 | valid on 'valid' subset | loss 0.087 | ppl 1.06 | wps 21687 | wpb 1954.1 | bsz 93.8 | num_updates 33964 | best_loss 0.087
+epoch 007 | valid on 'valid' subset | loss 0.087 | ppl 1.06 | wps 21687 | wpb 1954.1 | bsz 93.8 | num_updates 33964 | best_loss 0.087
+epoch 007 | valid on 'valid' subset | loss 0.087 | ppl 1.06 | wps 21687 | wpb 1954.1 | bsz 93.8 | num_updates 33964 | best_loss 0.087
+epoch 007 | valid on 'valid' subset | loss 0.087 | ppl 1.06 | wps 21687 | wpb 1954.1 | bsz 93.8 | num_updates 33964 | best_loss 0.087
+epoch 007 | valid on 'valid' subset | loss 0.087 | ppl 1.06 | wps 21687 | wpb 1954.1 | bsz 93.8 | num_updates 33964 | best_loss 0.087
+epoch 007 | valid on 'valid' subset | loss 0.087 | ppl 1.06 | wps 21687 | wpb 1954.1 | bsz 93.8 | num_updates 33964 | best_loss 0.087
+epoch 007 | valid on 'valid' subset | loss 0.087 | ppl 1.06 | wps 21687 | wpb 1954.1 | bsz 93.8 | num_updates 33964 | best_loss 0.087
+end of epoch 7 (average epoch stats below)
+epoch 007 | loss 0.038 | ppl 1.03 | wps 6567.8 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 33964 | lr 0.000171589 | gnorm 0.486 | train_wall 1442 | gb_free 13.3 | wall 10256
+epoch 007 | loss 0.038 | ppl 1.03 | wps 6567.8 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 33964 | lr 0.000171589 | gnorm 0.486 | train_wall 1442 | gb_free 13.3 | wall 10256
+epoch 007 | loss 0.038 | ppl 1.03 | wps 6567.8 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 33964 | lr 0.000171589 | gnorm 0.486 | train_wall 1442 | gb_free 13.3 | wall 10256
+epoch 007 | loss 0.038 | ppl 1.03 | wps 6567.8 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 33964 | lr 0.000171589 | gnorm 0.486 | train_wall 1442 | gb_free 13.3 | wall 10256
+epoch 007 | loss 0.038 | ppl 1.03 | wps 6567.8 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 33964 | lr 0.000171589 | gnorm 0.486 | train_wall 1442 | gb_free 13.3 | wall 10256
+epoch 007 | loss 0.038 | ppl 1.03 | wps 6567.8 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 33964 | lr 0.000171589 | gnorm 0.486 | train_wall 1442 | gb_free 13.3 | wall 10256
+epoch 007 | loss 0.038 | ppl 1.03 | wps 6567.8 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 33964 | lr 0.000171589 | gnorm 0.486 | train_wall 1442 | gb_free 13.3 | wall 10256
+Start iterating over samples
+begin validation on "valid" subset
+epoch 008 | valid on 'valid' subset | loss 0.085 | ppl 1.06 | wps 21767.2 | wpb 1954.1 | bsz 93.8 | num_updates 38816 | best_loss 0.085
+epoch 008 | valid on 'valid' subset | loss 0.085 | ppl 1.06 | wps 21767.2 | wpb 1954.1 | bsz 93.8 | num_updates 38816 | best_loss 0.085
+epoch 008 | valid on 'valid' subset | loss 0.085 | ppl 1.06 | wps 21767.2 | wpb 1954.1 | bsz 93.8 | num_updates 38816 | best_loss 0.085
+epoch 008 | valid on 'valid' subset | loss 0.085 | ppl 1.06 | wps 21767.2 | wpb 1954.1 | bsz 93.8 | num_updates 38816 | best_loss 0.085
+epoch 008 | valid on 'valid' subset | loss 0.085 | ppl 1.06 | wps 21767.2 | wpb 1954.1 | bsz 93.8 | num_updates 38816 | best_loss 0.085
+epoch 008 | valid on 'valid' subset | loss 0.085 | ppl 1.06 | wps 21767.2 | wpb 1954.1 | bsz 93.8 | num_updates 38816 | best_loss 0.085
+epoch 008 | valid on 'valid' subset | loss 0.085 | ppl 1.06 | wps 21767.2 | wpb 1954.1 | bsz 93.8 | num_updates 38816 | best_loss 0.085
+epoch 008 | valid on 'valid' subset | loss 0.085 | ppl 1.06 | wps 21767.2 | wpb 1954.1 | bsz 93.8 | num_updates 38816 | best_loss 0.085
+end of epoch 8 (average epoch stats below)
+epoch 008 | loss 0.028 | ppl 1.02 | wps 6567.8 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 38816 | lr 0.000160507 | gnorm 0.44 | train_wall 1443 | gb_free 12.8 | wall 11722
+epoch 008 | loss 0.028 | ppl 1.02 | wps 6567.8 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 38816 | lr 0.000160507 | gnorm 0.44 | train_wall 1443 | gb_free 12.8 | wall 11722
+epoch 008 | loss 0.028 | ppl 1.02 | wps 6567.8 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 38816 | lr 0.000160507 | gnorm 0.44 | train_wall 1443 | gb_free 12.8 | wall 11722
+epoch 008 | loss 0.028 | ppl 1.02 | wps 6567.8 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 38816 | lr 0.000160507 | gnorm 0.44 | train_wall 1443 | gb_free 12.8 | wall 11722
+epoch 008 | loss 0.028 | ppl 1.02 | wps 6567.8 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 38816 | lr 0.000160507 | gnorm 0.44 | train_wall 1443 | gb_free 12.8 | wall 11722
+epoch 008 | loss 0.028 | ppl 1.02 | wps 6567.8 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 38816 | lr 0.000160507 | gnorm 0.44 | train_wall 1443 | gb_free 12.8 | wall 11722
+epoch 008 | loss 0.028 | ppl 1.02 | wps 6567.8 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 38816 | lr 0.000160507 | gnorm 0.44 | train_wall 1443 | gb_free 12.8 | wall 11722
+epoch 008 | loss 0.028 | ppl 1.02 | wps 6567.8 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 38816 | lr 0.000160507 | gnorm 0.44 | train_wall 1443 | gb_free 12.8 | wall 11722
+Start iterating over samples
+begin validation on "valid" subset
+epoch 009 | valid on 'valid' subset | loss 0.083 | ppl 1.06 | wps 21736.2 | wpb 1954.1 | bsz 93.8 | num_updates 43668 | best_loss 0.083
+epoch 009 | valid on 'valid' subset | loss 0.083 | ppl 1.06 | wps 21736.2 | wpb 1954.1 | bsz 93.8 | num_updates 43668 | best_loss 0.083
+epoch 009 | valid on 'valid' subset | loss 0.083 | ppl 1.06 | wps 21736.2 | wpb 1954.1 | bsz 93.8 | num_updates 43668 | best_loss 0.083
+epoch 009 | valid on 'valid' subset | loss 0.083 | ppl 1.06 | wps 21736.2 | wpb 1954.1 | bsz 93.8 | num_updates 43668 | best_loss 0.083
+epoch 009 | valid on 'valid' subset | loss 0.083 | ppl 1.06 | wps 21736.2 | wpb 1954.1 | bsz 93.8 | num_updates 43668 | best_loss 0.083
+epoch 009 | valid on 'valid' subset | loss 0.083 | ppl 1.06 | wps 21736.2 | wpb 1954.1 | bsz 93.8 | num_updates 43668 | best_loss 0.083
+epoch 009 | valid on 'valid' subset | loss 0.083 | ppl 1.06 | wps 21736.2 | wpb 1954.1 | bsz 93.8 | num_updates 43668 | best_loss 0.083
+epoch 009 | valid on 'valid' subset | loss 0.083 | ppl 1.06 | wps 21736.2 | wpb 1954.1 | bsz 93.8 | num_updates 43668 | best_loss 0.083
+epoch 009 | valid on 'valid' subset | loss 0.083 | ppl 1.06 | wps 21736.2 | wpb 1954.1 | bsz 93.8 | num_updates 43668 | best_loss 0.083
+end of epoch 9 (average epoch stats below)
+epoch 009 | loss 0.022 | ppl 1.02 | wps 6564.3 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 43668 | lr 0.000151328 | gnorm 0.401 | train_wall 1443 | gb_free 12.9 | wall 13189
+epoch 009 | loss 0.022 | ppl 1.02 | wps 6564.3 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 43668 | lr 0.000151328 | gnorm 0.401 | train_wall 1443 | gb_free 12.9 | wall 13189
+epoch 009 | loss 0.022 | ppl 1.02 | wps 6564.3 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 43668 | lr 0.000151328 | gnorm 0.401 | train_wall 1443 | gb_free 12.9 | wall 13189
+epoch 009 | loss 0.022 | ppl 1.02 | wps 6564.3 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 43668 | lr 0.000151328 | gnorm 0.401 | train_wall 1443 | gb_free 12.9 | wall 13189
+epoch 009 | loss 0.022 | ppl 1.02 | wps 6564.3 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 43668 | lr 0.000151328 | gnorm 0.401 | train_wall 1443 | gb_free 12.9 | wall 13189
+epoch 009 | loss 0.022 | ppl 1.02 | wps 6564.3 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 43668 | lr 0.000151328 | gnorm 0.401 | train_wall 1443 | gb_free 12.9 | wall 13189
+epoch 009 | loss 0.022 | ppl 1.02 | wps 6564.3 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 43668 | lr 0.000151328 | gnorm 0.401 | train_wall 1443 | gb_free 12.9 | wall 13189
+epoch 009 | loss 0.022 | ppl 1.02 | wps 6564.3 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 43668 | lr 0.000151328 | gnorm 0.401 | train_wall 1443 | gb_free 12.9 | wall 13189
+epoch 009 | loss 0.022 | ppl 1.02 | wps 6564.3 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 43668 | lr 0.000151328 | gnorm 0.401 | train_wall 1443 | gb_free 12.9 | wall 13189
+Start iterating over samples
+begin validation on "valid" subset
+epoch 010 | valid on 'valid' subset | loss 0.084 | ppl 1.06 | wps 21769.7 | wpb 1954.1 | bsz 93.8 | num_updates 48520 | best_loss 0.083
+epoch 010 | valid on 'valid' subset | loss 0.084 | ppl 1.06 | wps 21769.7 | wpb 1954.1 | bsz 93.8 | num_updates 48520 | best_loss 0.083
+epoch 010 | valid on 'valid' subset | loss 0.084 | ppl 1.06 | wps 21769.7 | wpb 1954.1 | bsz 93.8 | num_updates 48520 | best_loss 0.083
+epoch 010 | valid on 'valid' subset | loss 0.084 | ppl 1.06 | wps 21769.7 | wpb 1954.1 | bsz 93.8 | num_updates 48520 | best_loss 0.083
+epoch 010 | valid on 'valid' subset | loss 0.084 | ppl 1.06 | wps 21769.7 | wpb 1954.1 | bsz 93.8 | num_updates 48520 | best_loss 0.083
+epoch 010 | valid on 'valid' subset | loss 0.084 | ppl 1.06 | wps 21769.7 | wpb 1954.1 | bsz 93.8 | num_updates 48520 | best_loss 0.083
+epoch 010 | valid on 'valid' subset | loss 0.084 | ppl 1.06 | wps 21769.7 | wpb 1954.1 | bsz 93.8 | num_updates 48520 | best_loss 0.083
+epoch 010 | valid on 'valid' subset | loss 0.084 | ppl 1.06 | wps 21769.7 | wpb 1954.1 | bsz 93.8 | num_updates 48520 | best_loss 0.083
+epoch 010 | valid on 'valid' subset | loss 0.084 | ppl 1.06 | wps 21769.7 | wpb 1954.1 | bsz 93.8 | num_updates 48520 | best_loss 0.083
+epoch 010 | valid on 'valid' subset | loss 0.084 | ppl 1.06 | wps 21769.7 | wpb 1954.1 | bsz 93.8 | num_updates 48520 | best_loss 0.083
+end of epoch 10 (average epoch stats below)
+epoch 010 | loss 0.018 | ppl 1.01 | wps 6566 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 48520 | lr 0.000143562 | gnorm 0.377 | train_wall 1443 | gb_free 12.8 | wall 14655
+epoch 010 | loss 0.018 | ppl 1.01 | wps 6566 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 48520 | lr 0.000143562 | gnorm 0.377 | train_wall 1443 | gb_free 12.8 | wall 14655
+epoch 010 | loss 0.018 | ppl 1.01 | wps 6566 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 48520 | lr 0.000143562 | gnorm 0.377 | train_wall 1443 | gb_free 12.8 | wall 14655
+epoch 010 | loss 0.018 | ppl 1.01 | wps 6566 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 48520 | lr 0.000143562 | gnorm 0.377 | train_wall 1443 | gb_free 12.8 | wall 14655
+epoch 010 | loss 0.018 | ppl 1.01 | wps 6566 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 48520 | lr 0.000143562 | gnorm 0.377 | train_wall 1443 | gb_free 12.8 | wall 14655
+epoch 010 | loss 0.018 | ppl 1.01 | wps 6566 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 48520 | lr 0.000143562 | gnorm 0.377 | train_wall 1443 | gb_free 12.8 | wall 14655
+epoch 010 | loss 0.018 | ppl 1.01 | wps 6566 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 48520 | lr 0.000143562 | gnorm 0.377 | train_wall 1443 | gb_free 12.8 | wall 14655
+epoch 010 | loss 0.018 | ppl 1.01 | wps 6566 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 48520 | lr 0.000143562 | gnorm 0.377 | train_wall 1443 | gb_free 12.8 | wall 14655
+epoch 010 | loss 0.018 | ppl 1.01 | wps 6566 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 48520 | lr 0.000143562 | gnorm 0.377 | train_wall 1443 | gb_free 12.8 | wall 14655
+epoch 010 | loss 0.018 | ppl 1.01 | wps 6566 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 48520 | lr 0.000143562 | gnorm 0.377 | train_wall 1443 | gb_free 12.8 | wall 14655
+Start iterating over samples
+begin validation on "valid" subset
+epoch 011 | valid on 'valid' subset | loss 0.078 | ppl 1.06 | wps 21426.4 | wpb 1954.1 | bsz 93.8 | num_updates 53372 | best_loss 0.078
+epoch 011 | valid on 'valid' subset | loss 0.078 | ppl 1.06 | wps 21426.4 | wpb 1954.1 | bsz 93.8 | num_updates 53372 | best_loss 0.078
+epoch 011 | valid on 'valid' subset | loss 0.078 | ppl 1.06 | wps 21426.4 | wpb 1954.1 | bsz 93.8 | num_updates 53372 | best_loss 0.078
+epoch 011 | valid on 'valid' subset | loss 0.078 | ppl 1.06 | wps 21426.4 | wpb 1954.1 | bsz 93.8 | num_updates 53372 | best_loss 0.078
+epoch 011 | valid on 'valid' subset | loss 0.078 | ppl 1.06 | wps 21426.4 | wpb 1954.1 | bsz 93.8 | num_updates 53372 | best_loss 0.078
+epoch 011 | valid on 'valid' subset | loss 0.078 | ppl 1.06 | wps 21426.4 | wpb 1954.1 | bsz 93.8 | num_updates 53372 | best_loss 0.078
+epoch 011 | valid on 'valid' subset | loss 0.078 | ppl 1.06 | wps 21426.4 | wpb 1954.1 | bsz 93.8 | num_updates 53372 | best_loss 0.078
+epoch 011 | valid on 'valid' subset | loss 0.078 | ppl 1.06 | wps 21426.4 | wpb 1954.1 | bsz 93.8 | num_updates 53372 | best_loss 0.078
+epoch 011 | valid on 'valid' subset | loss 0.078 | ppl 1.06 | wps 21426.4 | wpb 1954.1 | bsz 93.8 | num_updates 53372 | best_loss 0.078
+epoch 011 | valid on 'valid' subset | loss 0.078 | ppl 1.06 | wps 21426.4 | wpb 1954.1 | bsz 93.8 | num_updates 53372 | best_loss 0.078
+epoch 011 | valid on 'valid' subset | loss 0.078 | ppl 1.06 | wps 21426.4 | wpb 1954.1 | bsz 93.8 | num_updates 53372 | best_loss 0.078
+end of epoch 11 (average epoch stats below)
+epoch 011 | loss 0.015 | ppl 1.01 | wps 6564.3 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 53372 | lr 0.000136881 | gnorm 0.347 | train_wall 1443 | gb_free 12.8 | wall 16122
+epoch 011 | loss 0.015 | ppl 1.01 | wps 6564.3 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 53372 | lr 0.000136881 | gnorm 0.347 | train_wall 1443 | gb_free 12.8 | wall 16122
+epoch 011 | loss 0.015 | ppl 1.01 | wps 6564.3 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 53372 | lr 0.000136881 | gnorm 0.347 | train_wall 1443 | gb_free 12.8 | wall 16122
+epoch 011 | loss 0.015 | ppl 1.01 | wps 6564.3 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 53372 | lr 0.000136881 | gnorm 0.347 | train_wall 1443 | gb_free 12.8 | wall 16122
+epoch 011 | loss 0.015 | ppl 1.01 | wps 6564.3 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 53372 | lr 0.000136881 | gnorm 0.347 | train_wall 1443 | gb_free 12.8 | wall 16122
+epoch 011 | loss 0.015 | ppl 1.01 | wps 6564.3 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 53372 | lr 0.000136881 | gnorm 0.347 | train_wall 1443 | gb_free 12.8 | wall 16122
+epoch 011 | loss 0.015 | ppl 1.01 | wps 6564.3 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 53372 | lr 0.000136881 | gnorm 0.347 | train_wall 1443 | gb_free 12.8 | wall 16122
+epoch 011 | loss 0.015 | ppl 1.01 | wps 6564.3 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 53372 | lr 0.000136881 | gnorm 0.347 | train_wall 1443 | gb_free 12.8 | wall 16122
+epoch 011 | loss 0.015 | ppl 1.01 | wps 6564.3 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 53372 | lr 0.000136881 | gnorm 0.347 | train_wall 1443 | gb_free 12.8 | wall 16122
+epoch 011 | loss 0.015 | ppl 1.01 | wps 6564.3 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 53372 | lr 0.000136881 | gnorm 0.347 | train_wall 1443 | gb_free 12.8 | wall 16122
+epoch 011 | loss 0.015 | ppl 1.01 | wps 6564.3 | ups 3.31 | wpb 1984.3 | bsz 95.9 | num_updates 53372 | lr 0.000136881 | gnorm 0.347 | train_wall 1443 | gb_free 12.8 | wall 16122
+Start iterating over samples
+begin validation on "valid" subset
+epoch 012 | valid on 'valid' subset | loss 0.072 | ppl 1.05 | wps 21430.6 | wpb 1954.1 | bsz 93.8 | num_updates 58224 | best_loss 0.072
+epoch 012 | valid on 'valid' subset | loss 0.072 | ppl 1.05 | wps 21430.6 | wpb 1954.1 | bsz 93.8 | num_updates 58224 | best_loss 0.072
+epoch 012 | valid on 'valid' subset | loss 0.072 | ppl 1.05 | wps 21430.6 | wpb 1954.1 | bsz 93.8 | num_updates 58224 | best_loss 0.072
+epoch 012 | valid on 'valid' subset | loss 0.072 | ppl 1.05 | wps 21430.6 | wpb 1954.1 | bsz 93.8 | num_updates 58224 | best_loss 0.072
+epoch 012 | valid on 'valid' subset | loss 0.072 | ppl 1.05 | wps 21430.6 | wpb 1954.1 | bsz 93.8 | num_updates 58224 | best_loss 0.072
+epoch 012 | valid on 'valid' subset | loss 0.072 | ppl 1.05 | wps 21430.6 | wpb 1954.1 | bsz 93.8 | num_updates 58224 | best_loss 0.072
+epoch 012 | valid on 'valid' subset | loss 0.072 | ppl 1.05 | wps 21430.6 | wpb 1954.1 | bsz 93.8 | num_updates 58224 | best_loss 0.072
+epoch 012 | valid on 'valid' subset | loss 0.072 | ppl 1.05 | wps 21430.6 | wpb 1954.1 | bsz 93.8 | num_updates 58224 | best_loss 0.072
+epoch 012 | valid on 'valid' subset | loss 0.072 | ppl 1.05 | wps 21430.6 | wpb 1954.1 | bsz 93.8 | num_updates 58224 | best_loss 0.072
+epoch 012 | valid on 'valid' subset | loss 0.072 | ppl 1.05 | wps 21430.6 | wpb 1954.1 | bsz 93.8 | num_updates 58224 | best_loss 0.072
+epoch 012 | valid on 'valid' subset | loss 0.072 | ppl 1.05 | wps 21430.6 | wpb 1954.1 | bsz 93.8 | num_updates 58224 | best_loss 0.072
+epoch 012 | valid on 'valid' subset | loss 0.072 | ppl 1.05 | wps 21430.6 | wpb 1954.1 | bsz 93.8 | num_updates 58224 | best_loss 0.072
+end of epoch 12 (average epoch stats below)
+epoch 012 | loss 0.012 | ppl 1.01 | wps 6490.7 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 58224 | lr 0.000131054 | gnorm 0.316 | train_wall 1457 | gb_free 12.7 | wall 17605
+epoch 012 | loss 0.012 | ppl 1.01 | wps 6490.7 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 58224 | lr 0.000131054 | gnorm 0.316 | train_wall 1457 | gb_free 12.7 | wall 17605
+epoch 012 | loss 0.012 | ppl 1.01 | wps 6490.7 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 58224 | lr 0.000131054 | gnorm 0.316 | train_wall 1457 | gb_free 12.7 | wall 17605
+epoch 012 | loss 0.012 | ppl 1.01 | wps 6490.7 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 58224 | lr 0.000131054 | gnorm 0.316 | train_wall 1457 | gb_free 12.7 | wall 17605
+epoch 012 | loss 0.012 | ppl 1.01 | wps 6490.7 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 58224 | lr 0.000131054 | gnorm 0.316 | train_wall 1457 | gb_free 12.7 | wall 17605
+epoch 012 | loss 0.012 | ppl 1.01 | wps 6490.7 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 58224 | lr 0.000131054 | gnorm 0.316 | train_wall 1457 | gb_free 12.7 | wall 17605
+epoch 012 | loss 0.012 | ppl 1.01 | wps 6490.7 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 58224 | lr 0.000131054 | gnorm 0.316 | train_wall 1457 | gb_free 12.7 | wall 17605
+epoch 012 | loss 0.012 | ppl 1.01 | wps 6490.7 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 58224 | lr 0.000131054 | gnorm 0.316 | train_wall 1457 | gb_free 12.7 | wall 17605
+epoch 012 | loss 0.012 | ppl 1.01 | wps 6490.7 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 58224 | lr 0.000131054 | gnorm 0.316 | train_wall 1457 | gb_free 12.7 | wall 17605
+epoch 012 | loss 0.012 | ppl 1.01 | wps 6490.7 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 58224 | lr 0.000131054 | gnorm 0.316 | train_wall 1457 | gb_free 12.7 | wall 17605
+epoch 012 | loss 0.012 | ppl 1.01 | wps 6490.7 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 58224 | lr 0.000131054 | gnorm 0.316 | train_wall 1457 | gb_free 12.7 | wall 17605
+epoch 012 | loss 0.012 | ppl 1.01 | wps 6490.7 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 58224 | lr 0.000131054 | gnorm 0.316 | train_wall 1457 | gb_free 12.7 | wall 17605
+Start iterating over samples
+begin validation on "valid" subset
+epoch 013 | valid on 'valid' subset | loss 0.076 | ppl 1.05 | wps 21465.9 | wpb 1954.1 | bsz 93.8 | num_updates 63076 | best_loss 0.072
+epoch 013 | valid on 'valid' subset | loss 0.076 | ppl 1.05 | wps 21465.9 | wpb 1954.1 | bsz 93.8 | num_updates 63076 | best_loss 0.072
+epoch 013 | valid on 'valid' subset | loss 0.076 | ppl 1.05 | wps 21465.9 | wpb 1954.1 | bsz 93.8 | num_updates 63076 | best_loss 0.072
+epoch 013 | valid on 'valid' subset | loss 0.076 | ppl 1.05 | wps 21465.9 | wpb 1954.1 | bsz 93.8 | num_updates 63076 | best_loss 0.072
+epoch 013 | valid on 'valid' subset | loss 0.076 | ppl 1.05 | wps 21465.9 | wpb 1954.1 | bsz 93.8 | num_updates 63076 | best_loss 0.072
+epoch 013 | valid on 'valid' subset | loss 0.076 | ppl 1.05 | wps 21465.9 | wpb 1954.1 | bsz 93.8 | num_updates 63076 | best_loss 0.072
+epoch 013 | valid on 'valid' subset | loss 0.076 | ppl 1.05 | wps 21465.9 | wpb 1954.1 | bsz 93.8 | num_updates 63076 | best_loss 0.072
+epoch 013 | valid on 'valid' subset | loss 0.076 | ppl 1.05 | wps 21465.9 | wpb 1954.1 | bsz 93.8 | num_updates 63076 | best_loss 0.072
+epoch 013 | valid on 'valid' subset | loss 0.076 | ppl 1.05 | wps 21465.9 | wpb 1954.1 | bsz 93.8 | num_updates 63076 | best_loss 0.072
+epoch 013 | valid on 'valid' subset | loss 0.076 | ppl 1.05 | wps 21465.9 | wpb 1954.1 | bsz 93.8 | num_updates 63076 | best_loss 0.072
+epoch 013 | valid on 'valid' subset | loss 0.076 | ppl 1.05 | wps 21465.9 | wpb 1954.1 | bsz 93.8 | num_updates 63076 | best_loss 0.072
+epoch 013 | valid on 'valid' subset | loss 0.076 | ppl 1.05 | wps 21465.9 | wpb 1954.1 | bsz 93.8 | num_updates 63076 | best_loss 0.072
+epoch 013 | valid on 'valid' subset | loss 0.076 | ppl 1.05 | wps 21465.9 | wpb 1954.1 | bsz 93.8 | num_updates 63076 | best_loss 0.072
+end of epoch 13 (average epoch stats below)
+epoch 013 | loss 0.011 | ppl 1.01 | wps 6491.7 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 63076 | lr 0.000125912 | gnorm 0.297 | train_wall 1457 | gb_free 12.9 | wall 19088
+epoch 013 | loss 0.011 | ppl 1.01 | wps 6491.7 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 63076 | lr 0.000125912 | gnorm 0.297 | train_wall 1457 | gb_free 12.9 | wall 19088
+epoch 013 | loss 0.011 | ppl 1.01 | wps 6491.7 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 63076 | lr 0.000125912 | gnorm 0.297 | train_wall 1457 | gb_free 12.9 | wall 19088
+epoch 013 | loss 0.011 | ppl 1.01 | wps 6491.7 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 63076 | lr 0.000125912 | gnorm 0.297 | train_wall 1457 | gb_free 12.9 | wall 19088
+epoch 013 | loss 0.011 | ppl 1.01 | wps 6491.7 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 63076 | lr 0.000125912 | gnorm 0.297 | train_wall 1457 | gb_free 12.9 | wall 19088
+epoch 013 | loss 0.011 | ppl 1.01 | wps 6491.7 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 63076 | lr 0.000125912 | gnorm 0.297 | train_wall 1457 | gb_free 12.9 | wall 19088
+epoch 013 | loss 0.011 | ppl 1.01 | wps 6491.7 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 63076 | lr 0.000125912 | gnorm 0.297 | train_wall 1457 | gb_free 12.9 | wall 19088
+epoch 013 | loss 0.011 | ppl 1.01 | wps 6491.7 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 63076 | lr 0.000125912 | gnorm 0.297 | train_wall 1457 | gb_free 12.9 | wall 19088
+epoch 013 | loss 0.011 | ppl 1.01 | wps 6491.7 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 63076 | lr 0.000125912 | gnorm 0.297 | train_wall 1457 | gb_free 12.9 | wall 19088
+epoch 013 | loss 0.011 | ppl 1.01 | wps 6491.7 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 63076 | lr 0.000125912 | gnorm 0.297 | train_wall 1457 | gb_free 12.9 | wall 19088
+epoch 013 | loss 0.011 | ppl 1.01 | wps 6491.7 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 63076 | lr 0.000125912 | gnorm 0.297 | train_wall 1457 | gb_free 12.9 | wall 19088
+epoch 013 | loss 0.011 | ppl 1.01 | wps 6491.7 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 63076 | lr 0.000125912 | gnorm 0.297 | train_wall 1457 | gb_free 12.9 | wall 19088
+epoch 013 | loss 0.011 | ppl 1.01 | wps 6491.7 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 63076 | lr 0.000125912 | gnorm 0.297 | train_wall 1457 | gb_free 12.9 | wall 19088
+Start iterating over samples
+begin validation on "valid" subset
+epoch 014 | valid on 'valid' subset | loss 0.074 | ppl 1.05 | wps 21415.4 | wpb 1954.1 | bsz 93.8 | num_updates 67928 | best_loss 0.072
+epoch 014 | valid on 'valid' subset | loss 0.074 | ppl 1.05 | wps 21415.4 | wpb 1954.1 | bsz 93.8 | num_updates 67928 | best_loss 0.072
+epoch 014 | valid on 'valid' subset | loss 0.074 | ppl 1.05 | wps 21415.4 | wpb 1954.1 | bsz 93.8 | num_updates 67928 | best_loss 0.072
+epoch 014 | valid on 'valid' subset | loss 0.074 | ppl 1.05 | wps 21415.4 | wpb 1954.1 | bsz 93.8 | num_updates 67928 | best_loss 0.072
+epoch 014 | valid on 'valid' subset | loss 0.074 | ppl 1.05 | wps 21415.4 | wpb 1954.1 | bsz 93.8 | num_updates 67928 | best_loss 0.072
+epoch 014 | valid on 'valid' subset | loss 0.074 | ppl 1.05 | wps 21415.4 | wpb 1954.1 | bsz 93.8 | num_updates 67928 | best_loss 0.072
+epoch 014 | valid on 'valid' subset | loss 0.074 | ppl 1.05 | wps 21415.4 | wpb 1954.1 | bsz 93.8 | num_updates 67928 | best_loss 0.072
+epoch 014 | valid on 'valid' subset | loss 0.074 | ppl 1.05 | wps 21415.4 | wpb 1954.1 | bsz 93.8 | num_updates 67928 | best_loss 0.072
+epoch 014 | valid on 'valid' subset | loss 0.074 | ppl 1.05 | wps 21415.4 | wpb 1954.1 | bsz 93.8 | num_updates 67928 | best_loss 0.072
+epoch 014 | valid on 'valid' subset | loss 0.074 | ppl 1.05 | wps 21415.4 | wpb 1954.1 | bsz 93.8 | num_updates 67928 | best_loss 0.072
+epoch 014 | valid on 'valid' subset | loss 0.074 | ppl 1.05 | wps 21415.4 | wpb 1954.1 | bsz 93.8 | num_updates 67928 | best_loss 0.072
+epoch 014 | valid on 'valid' subset | loss 0.074 | ppl 1.05 | wps 21415.4 | wpb 1954.1 | bsz 93.8 | num_updates 67928 | best_loss 0.072
+epoch 014 | valid on 'valid' subset | loss 0.074 | ppl 1.05 | wps 21415.4 | wpb 1954.1 | bsz 93.8 | num_updates 67928 | best_loss 0.072
+epoch 014 | valid on 'valid' subset | loss 0.074 | ppl 1.05 | wps 21415.4 | wpb 1954.1 | bsz 93.8 | num_updates 67928 | best_loss 0.072
+end of epoch 14 (average epoch stats below)
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6482.8 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 67928 | lr 0.000121332 | gnorm 0.282 | train_wall 1458 | gb_free 13.1 | wall 20573
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6482.8 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 67928 | lr 0.000121332 | gnorm 0.282 | train_wall 1458 | gb_free 13.1 | wall 20573
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6482.8 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 67928 | lr 0.000121332 | gnorm 0.282 | train_wall 1458 | gb_free 13.1 | wall 20573
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6482.8 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 67928 | lr 0.000121332 | gnorm 0.282 | train_wall 1458 | gb_free 13.1 | wall 20573
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6482.8 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 67928 | lr 0.000121332 | gnorm 0.282 | train_wall 1458 | gb_free 13.1 | wall 20573
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6482.8 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 67928 | lr 0.000121332 | gnorm 0.282 | train_wall 1458 | gb_free 13.1 | wall 20573
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6482.8 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 67928 | lr 0.000121332 | gnorm 0.282 | train_wall 1458 | gb_free 13.1 | wall 20573
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6482.8 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 67928 | lr 0.000121332 | gnorm 0.282 | train_wall 1458 | gb_free 13.1 | wall 20573
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6482.8 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 67928 | lr 0.000121332 | gnorm 0.282 | train_wall 1458 | gb_free 13.1 | wall 20573
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6482.8 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 67928 | lr 0.000121332 | gnorm 0.282 | train_wall 1458 | gb_free 13.1 | wall 20573
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6482.8 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 67928 | lr 0.000121332 | gnorm 0.282 | train_wall 1458 | gb_free 13.1 | wall 20573
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6482.8 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 67928 | lr 0.000121332 | gnorm 0.282 | train_wall 1458 | gb_free 13.1 | wall 20573
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6482.8 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 67928 | lr 0.000121332 | gnorm 0.282 | train_wall 1458 | gb_free 13.1 | wall 20573
+epoch 014 | loss 0.01 | ppl 1.01 | wps 6482.8 | ups 3.27 | wpb 1984.3 | bsz 95.9 | num_updates 67928 | lr 0.000121332 | gnorm 0.282 | train_wall 1458 | gb_free 13.1 | wall 20573
+Start iterating over samples
+begin validation on "valid" subset
+epoch 015 | valid on 'valid' subset | loss 0.075 | ppl 1.05 | wps 21711.6 | wpb 1954.1 | bsz 93.8 | num_updates 72780 | best_loss 0.072
+epoch 015 | valid on 'valid' subset | loss 0.075 | ppl 1.05 | wps 21711.6 | wpb 1954.1 | bsz 93.8 | num_updates 72780 | best_loss 0.072
+epoch 015 | valid on 'valid' subset | loss 0.075 | ppl 1.05 | wps 21711.6 | wpb 1954.1 | bsz 93.8 | num_updates 72780 | best_loss 0.072
+epoch 015 | valid on 'valid' subset | loss 0.075 | ppl 1.05 | wps 21711.6 | wpb 1954.1 | bsz 93.8 | num_updates 72780 | best_loss 0.072
+epoch 015 | valid on 'valid' subset | loss 0.075 | ppl 1.05 | wps 21711.6 | wpb 1954.1 | bsz 93.8 | num_updates 72780 | best_loss 0.072
+epoch 015 | valid on 'valid' subset | loss 0.075 | ppl 1.05 | wps 21711.6 | wpb 1954.1 | bsz 93.8 | num_updates 72780 | best_loss 0.072
+epoch 015 | valid on 'valid' subset | loss 0.075 | ppl 1.05 | wps 21711.6 | wpb 1954.1 | bsz 93.8 | num_updates 72780 | best_loss 0.072
+epoch 015 | valid on 'valid' subset | loss 0.075 | ppl 1.05 | wps 21711.6 | wpb 1954.1 | bsz 93.8 | num_updates 72780 | best_loss 0.072
+epoch 015 | valid on 'valid' subset | loss 0.075 | ppl 1.05 | wps 21711.6 | wpb 1954.1 | bsz 93.8 | num_updates 72780 | best_loss 0.072
+epoch 015 | valid on 'valid' subset | loss 0.075 | ppl 1.05 | wps 21711.6 | wpb 1954.1 | bsz 93.8 | num_updates 72780 | best_loss 0.072
+epoch 015 | valid on 'valid' subset | loss 0.075 | ppl 1.05 | wps 21711.6 | wpb 1954.1 | bsz 93.8 | num_updates 72780 | best_loss 0.072
+epoch 015 | valid on 'valid' subset | loss 0.075 | ppl 1.05 | wps 21711.6 | wpb 1954.1 | bsz 93.8 | num_updates 72780 | best_loss 0.072
+epoch 015 | valid on 'valid' subset | loss 0.075 | ppl 1.05 | wps 21711.6 | wpb 1954.1 | bsz 93.8 | num_updates 72780 | best_loss 0.072
+epoch 015 | valid on 'valid' subset | loss 0.075 | ppl 1.05 | wps 21711.6 | wpb 1954.1 | bsz 93.8 | num_updates 72780 | best_loss 0.072
+epoch 015 | valid on 'valid' subset | loss 0.075 | ppl 1.05 | wps 21711.6 | wpb 1954.1 | bsz 93.8 | num_updates 72780 | best_loss 0.072
+end of epoch 15 (average epoch stats below)
+epoch 015 | loss 0.008 | ppl 1.01 | wps 6543.9 | ups 3.3 | wpb 1984.3 | bsz 95.9 | num_updates 72780 | lr 0.000117218 | gnorm 0.27 | train_wall 1448 | gb_free 12.8 | wall 22044
+epoch 015 | loss 0.008 | ppl 1.01 | wps 6543.9 | ups 3.3 | wpb 1984.3 | bsz 95.9 | num_updates 72780 | lr 0.000117218 | gnorm 0.27 | train_wall 1448 | gb_free 12.8 | wall 22044
+epoch 015 | loss 0.008 | ppl 1.01 | wps 6543.9 | ups 3.3 | wpb 1984.3 | bsz 95.9 | num_updates 72780 | lr 0.000117218 | gnorm 0.27 | train_wall 1448 | gb_free 12.8 | wall 22044
+epoch 015 | loss 0.008 | ppl 1.01 | wps 6543.9 | ups 3.3 | wpb 1984.3 | bsz 95.9 | num_updates 72780 | lr 0.000117218 | gnorm 0.27 | train_wall 1448 | gb_free 12.8 | wall 22044
+epoch 015 | loss 0.008 | ppl 1.01 | wps 6543.9 | ups 3.3 | wpb 1984.3 | bsz 95.9 | num_updates 72780 | lr 0.000117218 | gnorm 0.27 | train_wall 1448 | gb_free 12.8 | wall 22044
+epoch 015 | loss 0.008 | ppl 1.01 | wps 6543.9 | ups 3.3 | wpb 1984.3 | bsz 95.9 | num_updates 72780 | lr 0.000117218 | gnorm 0.27 | train_wall 1448 | gb_free 12.8 | wall 22044
+epoch 015 | loss 0.008 | ppl 1.01 | wps 6543.9 | ups 3.3 | wpb 1984.3 | bsz 95.9 | num_updates 72780 | lr 0.000117218 | gnorm 0.27 | train_wall 1448 | gb_free 12.8 | wall 22044
+epoch 015 | loss 0.008 | ppl 1.01 | wps 6543.9 | ups 3.3 | wpb 1984.3 | bsz 95.9 | num_updates 72780 | lr 0.000117218 | gnorm 0.27 | train_wall 1448 | gb_free 12.8 | wall 22044
+epoch 015 | loss 0.008 | ppl 1.01 | wps 6543.9 | ups 3.3 | wpb 1984.3 | bsz 95.9 | num_updates 72780 | lr 0.000117218 | gnorm 0.27 | train_wall 1448 | gb_free 12.8 | wall 22044
+epoch 015 | loss 0.008 | ppl 1.01 | wps 6543.9 | ups 3.3 | wpb 1984.3 | bsz 95.9 | num_updates 72780 | lr 0.000117218 | gnorm 0.27 | train_wall 1448 | gb_free 12.8 | wall 22044
+epoch 015 | loss 0.008 | ppl 1.01 | wps 6543.9 | ups 3.3 | wpb 1984.3 | bsz 95.9 | num_updates 72780 | lr 0.000117218 | gnorm 0.27 | train_wall 1448 | gb_free 12.8 | wall 22044
+epoch 015 | loss 0.008 | ppl 1.01 | wps 6543.9 | ups 3.3 | wpb 1984.3 | bsz 95.9 | num_updates 72780 | lr 0.000117218 | gnorm 0.27 | train_wall 1448 | gb_free 12.8 | wall 22044
+epoch 015 | loss 0.008 | ppl 1.01 | wps 6543.9 | ups 3.3 | wpb 1984.3 | bsz 95.9 | num_updates 72780 | lr 0.000117218 | gnorm 0.27 | train_wall 1448 | gb_free 12.8 | wall 22044
+epoch 015 | loss 0.008 | ppl 1.01 | wps 6543.9 | ups 3.3 | wpb 1984.3 | bsz 95.9 | num_updates 72780 | lr 0.000117218 | gnorm 0.27 | train_wall 1448 | gb_free 12.8 | wall 22044
+epoch 015 | loss 0.008 | ppl 1.01 | wps 6543.9 | ups 3.3 | wpb 1984.3 | bsz 95.9 | num_updates 72780 | lr 0.000117218 | gnorm 0.27 | train_wall 1448 | gb_free 12.8 | wall 22044
+Start iterating over samples
+begin validation on "valid" subset
+epoch 016 | valid on 'valid' subset | loss 0.079 | ppl 1.06 | wps 21661.2 | wpb 1954.1 | bsz 93.8 | num_updates 77632 | best_loss 0.072
+epoch 016 | valid on 'valid' subset | loss 0.079 | ppl 1.06 | wps 21661.2 | wpb 1954.1 | bsz 93.8 | num_updates 77632 | best_loss 0.072
+epoch 016 | valid on 'valid' subset | loss 0.079 | ppl 1.06 | wps 21661.2 | wpb 1954.1 | bsz 93.8 | num_updates 77632 | best_loss 0.072
+epoch 016 | valid on 'valid' subset | loss 0.079 | ppl 1.06 | wps 21661.2 | wpb 1954.1 | bsz 93.8 | num_updates 77632 | best_loss 0.072
+epoch 016 | valid on 'valid' subset | loss 0.079 | ppl 1.06 | wps 21661.2 | wpb 1954.1 | bsz 93.8 | num_updates 77632 | best_loss 0.072
+epoch 016 | valid on 'valid' subset | loss 0.079 | ppl 1.06 | wps 21661.2 | wpb 1954.1 | bsz 93.8 | num_updates 77632 | best_loss 0.072
+epoch 016 | valid on 'valid' subset | loss 0.079 | ppl 1.06 | wps 21661.2 | wpb 1954.1 | bsz 93.8 | num_updates 77632 | best_loss 0.072
+epoch 016 | valid on 'valid' subset | loss 0.079 | ppl 1.06 | wps 21661.2 | wpb 1954.1 | bsz 93.8 | num_updates 77632 | best_loss 0.072
+epoch 016 | valid on 'valid' subset | loss 0.079 | ppl 1.06 | wps 21661.2 | wpb 1954.1 | bsz 93.8 | num_updates 77632 | best_loss 0.072
+epoch 016 | valid on 'valid' subset | loss 0.079 | ppl 1.06 | wps 21661.2 | wpb 1954.1 | bsz 93.8 | num_updates 77632 | best_loss 0.072
+epoch 016 | valid on 'valid' subset | loss 0.079 | ppl 1.06 | wps 21661.2 | wpb 1954.1 | bsz 93.8 | num_updates 77632 | best_loss 0.072
+epoch 016 | valid on 'valid' subset | loss 0.079 | ppl 1.06 | wps 21661.2 | wpb 1954.1 | bsz 93.8 | num_updates 77632 | best_loss 0.072
+epoch 016 | valid on 'valid' subset | loss 0.079 | ppl 1.06 | wps 21661.2 | wpb 1954.1 | bsz 93.8 | num_updates 77632 | best_loss 0.072
+epoch 016 | valid on 'valid' subset | loss 0.079 | ppl 1.06 | wps 21661.2 | wpb 1954.1 | bsz 93.8 | num_updates 77632 | best_loss 0.072
+epoch 016 | valid on 'valid' subset | loss 0.079 | ppl 1.06 | wps 21661.2 | wpb 1954.1 | bsz 93.8 | num_updates 77632 | best_loss 0.072
+epoch 016 | valid on 'valid' subset | loss 0.079 | ppl 1.06 | wps 21661.2 | wpb 1954.1 | bsz 93.8 | num_updates 77632 | best_loss 0.072
+early stop since valid performance hasn't improved for last 4 runs
+end of epoch 16 (average epoch stats below)
+epoch 016 | loss 0.007 | ppl 1.01 | wps 6535.8 | ups 3.29 | wpb 1984.3 | bsz 95.9 | num_updates 77632 | lr 0.000113496 | gnorm 0.252 | train_wall 1450 | gb_free 13 | wall 23517
+epoch 016 | loss 0.007 | ppl 1.01 | wps 6535.8 | ups 3.29 | wpb 1984.3 | bsz 95.9 | num_updates 77632 | lr 0.000113496 | gnorm 0.252 | train_wall 1450 | gb_free 13 | wall 23517
+epoch 016 | loss 0.007 | ppl 1.01 | wps 6535.8 | ups 3.29 | wpb 1984.3 | bsz 95.9 | num_updates 77632 | lr 0.000113496 | gnorm 0.252 | train_wall 1450 | gb_free 13 | wall 23517
+epoch 016 | loss 0.007 | ppl 1.01 | wps 6535.8 | ups 3.29 | wpb 1984.3 | bsz 95.9 | num_updates 77632 | lr 0.000113496 | gnorm 0.252 | train_wall 1450 | gb_free 13 | wall 23517
+epoch 016 | loss 0.007 | ppl 1.01 | wps 6535.8 | ups 3.29 | wpb 1984.3 | bsz 95.9 | num_updates 77632 | lr 0.000113496 | gnorm 0.252 | train_wall 1450 | gb_free 13 | wall 23517
+epoch 016 | loss 0.007 | ppl 1.01 | wps 6535.8 | ups 3.29 | wpb 1984.3 | bsz 95.9 | num_updates 77632 | lr 0.000113496 | gnorm 0.252 | train_wall 1450 | gb_free 13 | wall 23517
+epoch 016 | loss 0.007 | ppl 1.01 | wps 6535.8 | ups 3.29 | wpb 1984.3 | bsz 95.9 | num_updates 77632 | lr 0.000113496 | gnorm 0.252 | train_wall 1450 | gb_free 13 | wall 23517
+epoch 016 | loss 0.007 | ppl 1.01 | wps 6535.8 | ups 3.29 | wpb 1984.3 | bsz 95.9 | num_updates 77632 | lr 0.000113496 | gnorm 0.252 | train_wall 1450 | gb_free 13 | wall 23517
+epoch 016 | loss 0.007 | ppl 1.01 | wps 6535.8 | ups 3.29 | wpb 1984.3 | bsz 95.9 | num_updates 77632 | lr 0.000113496 | gnorm 0.252 | train_wall 1450 | gb_free 13 | wall 23517
+epoch 016 | loss 0.007 | ppl 1.01 | wps 6535.8 | ups 3.29 | wpb 1984.3 | bsz 95.9 | num_updates 77632 | lr 0.000113496 | gnorm 0.252 | train_wall 1450 | gb_free 13 | wall 23517
+epoch 016 | loss 0.007 | ppl 1.01 | wps 6535.8 | ups 3.29 | wpb 1984.3 | bsz 95.9 | num_updates 77632 | lr 0.000113496 | gnorm 0.252 | train_wall 1450 | gb_free 13 | wall 23517
+epoch 016 | loss 0.007 | ppl 1.01 | wps 6535.8 | ups 3.29 | wpb 1984.3 | bsz 95.9 | num_updates 77632 | lr 0.000113496 | gnorm 0.252 | train_wall 1450 | gb_free 13 | wall 23517
+epoch 016 | loss 0.007 | ppl 1.01 | wps 6535.8 | ups 3.29 | wpb 1984.3 | bsz 95.9 | num_updates 77632 | lr 0.000113496 | gnorm 0.252 | train_wall 1450 | gb_free 13 | wall 23517
+epoch 016 | loss 0.007 | ppl 1.01 | wps 6535.8 | ups 3.29 | wpb 1984.3 | bsz 95.9 | num_updates 77632 | lr 0.000113496 | gnorm 0.252 | train_wall 1450 | gb_free 13 | wall 23517
+epoch 016 | loss 0.007 | ppl 1.01 | wps 6535.8 | ups 3.29 | wpb 1984.3 | bsz 95.9 | num_updates 77632 | lr 0.000113496 | gnorm 0.252 | train_wall 1450 | gb_free 13 | wall 23517
+epoch 016 | loss 0.007 | ppl 1.01 | wps 6535.8 | ups 3.29 | wpb 1984.3 | bsz 95.9 | num_updates 77632 | lr 0.000113496 | gnorm 0.252 | train_wall 1450 | gb_free 13 | wall 23517
+done training in 23517.0 seconds