| {"max_len": 1024, "d_model": 960, "num_layers": 32, "num_heads": 15, "num_kv_heads": 5, "d_ff": 4096, "learning_rate": 5e-05, "pad_token_id": 0, "bos_token_id": 2, "eos_token_id": 3, "training_max_len": 1024, "end_of_turn_token_id": 12, "base_model_id": "KeisukeMiyamoto/lambda-1-360m-mid", "batch_size": 48, "gradient_accumulation_steps": 2, "devices": "auto", "device_count": 2, "global_batch_size": 96, "effective_batch_size": 96, "global_effective_batch_size": 192, "lr_schedule": "warmup_cosine", "lr_warmup_steps": 200, "min_learning_rate": 2.5e-05, "min_learning_rate_ratio": 0.5, "trainable_layers": "all", "chat_template_version": 1, "posttraining_datasets": ["KeisukeMiyamoto/SyntheticTalk-jp:train"], "validation_dataset": "KeisukeMiyamoto/SyntheticTalk-jp:validation", "validation_cache_path": "models/lambda-1-360m-it/validation-cache-SyntheticTalk-jp-bucket-packing-v1-len1024-samples768.pt", "validation_sample_count": 768, "posttraining_steps": 5000, "training_precision": "fp8", "float8_recipe": "tensorwise"} |