Invalid JSON:
Unexpected token 'I', ..."val_bpb": Infinity,
"... is not valid JSON
| { | |
| "step": 1000, | |
| "val_bpb": 0.0, | |
| "model_config": { | |
| "max_seq_len": 2048, | |
| "vocab_size": 65536, | |
| "n_layer": 6, | |
| "n_head": 4, | |
| "n_embd": 512, | |
| "n_hidden": 32768 | |
| }, | |
| "user_config": { | |
| "run": "test_run3", | |
| "device_type": "", | |
| "depth": 6, | |
| "max_seq_len": 2048, | |
| "num_iterations": 2000, | |
| "target_flops": -1.0, | |
| "target_param_data_ratio": 20, | |
| "device_batch_size": 4, | |
| "total_batch_size": 524288, | |
| "embedding_lr": 0.002, | |
| "unembedding_lr": 0.004, | |
| "weight_decay": 0.1, | |
| "matrix_lr": 0.02, | |
| "grad_clip": 1.0, | |
| "warmup_ratio": 0.0, | |
| "warmdown_ratio": 0.6, | |
| "final_lr_frac": 0.1, | |
| "resume_from_step": -1, | |
| "eval_every": 10000, | |
| "eval_tokens": 10485760, | |
| "core_metric_every": 10000, | |
| "core_metric_max_per_task": 500, | |
| "sample_every": 100, | |
| "save_every": 1000, | |
| "model_tag": "d3" | |
| }, | |
| "device_batch_size": 4, | |
| "max_seq_len": 2048, | |
| "dataloader_state_dict": { | |
| "batches_yielded": 8000 | |
| }, | |
| "loop_state": { | |
| "min_val_bpb": Infinity, | |
| "smooth_train_loss": 4.2489351594465, | |
| "total_training_time": 4557.3429136276245 | |
| } | |
| } |