MIyamoto Keisuke commited on
Commit
ea0713b
·
verified ·
1 Parent(s): 116fa4e

Upload lambda-160m-midtrained pretrained model

Browse files
Files changed (2) hide show
  1. model.pth +2 -2
  2. model_config.json +1 -1
model.pth CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:f6a0e578e1cdcb1e4c84cda070f39150862a762df33915b62cd5fbe7e39d530b
3
- size 658115811
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:90f1cf7b59a93d3cc60b0ec367b40dbdb968f0105d0c64695c363c9844b7ffa9
3
+ size 655045351
model_config.json CHANGED
@@ -1 +1 @@
1
- {"max_len": 1024, "d_model": 768, "num_layers": 16, "num_heads": 12, "d_ff": 3072, "learning_rate": 0.0001, "batch_size": 72, "gradient_accumulation_steps": 2, "effective_batch_size": 144, "lr_schedule": "fixed", "loss_chunk_size": 32, "pad_token_id": 0, "bos_token_id": 2, "eos_token_id": 3, "corpus_signature": "ba5e40d4dae8dceb", "dataset_case": {"name": "synthetic-textbook-jp", "genre": "textbook", "language": "ja", "dataset_path": "MK0727/SyntheticTextbook-jp", "config_name": "default", "split": "train", "text_column": "rewrite"}, "val_split_modulo": 100, "val_split_index": 0, "validation_cache_path": "models/lambda-160m-midtrained/validation-cache-ba5e40d4dae8dceb-bucket-packing-v1-len1024-samples4608-split100-0.pt", "validation_sample_count": 4608, "packing_version": "bucket-packing-v1", "trained_steps": 10240, "midtraining_source_model": "models/lambda-160m", "midtraining_max_steps": 10240, "shuffle_buffer_size": 10000, "shuffle_seed": 17}
 
1
+ {"max_len": 1024, "d_model": 768, "num_layers": 16, "num_heads": 12, "d_ff": 2048, "learning_rate": 0.0001, "batch_size": 16, "gradient_accumulation_steps": 2, "devices": "auto", "device_count": 4, "global_batch_size": 64, "effective_batch_size": 32, "global_effective_batch_size": 128, "lr_schedule": "warmup_cosine", "lr_warmup_steps": 1000, "min_learning_rate": 2e-05, "min_learning_rate_ratio": 0.2, "loss_chunk_size": 32, "pad_token_id": 0, "bos_token_id": 2, "eos_token_id": 3, "corpus_signature": "ba5e40d4dae8dceb", "dataset_case": {"name": "synthetic-textbook-jp", "genre": "textbook", "language": "ja", "dataset_path": "MK0727/SyntheticTextbook-jp", "config_name": "default", "split": "train", "text_column": "rewrite"}, "val_split_modulo": 100, "val_split_index": 0, "validation_cache_path": "models/lambda-160m-midtrained/validation-cache-ba5e40d4dae8dceb-bucket-packing-v1-len1024-samples4096-split100-0.pt", "validation_sample_count": 4096, "packing_version": "bucket-packing-v1", "shuffle_buffer_size": 10000, "base_shuffle_seed": 17, "shuffle_seed": 17, "trained_steps": 20480, "midtraining_source_model": "models/lambda-160m", "midtraining_max_steps": 20480}