model_config.json

{"max_len": 1024, "d_model": 768, "num_layers": 16, "num_heads": 12, "d_ff": 3072, "learning_rate": 0.0002, "batch_size": 96, "gradient_accumulation_steps": 4, "effective_batch_size": 384, "lr_schedule": "warmup_cosine", "lr_warmup_steps": 2000, "min_learning_rate": 0.0001, "min_learning_rate_ratio": 0.5, "loss_chunk_size": 32, "pad_token_id": 0, "bos_token_id": 2, "eos_token_id": 3, "corpus_signature": "fb2af8e50eb90fad", "dataset_case": {"name": "cleaned-fineweb2-edu-jp", "genre": "web", "language": "ja", "dataset_path": "MK0727/CleanedFineWeb2Edu-jp", "config_name": "default", "split": "train", "text_column": "text"}, "val_split_modulo": 100, "val_split_index": 0, "validation_cache_path": "models/lambda-160m/validation-cache-fb2af8e50eb90fad-bucket-packing-v1-len1024-samples6144-split100-0.pt", "validation_sample_count": 6144, "packing_version": "bucket-packing-v1", "trained_steps": 10240}