| { |
| "training_config": { |
| "training_data_paths": "['/data/horse/ws/juha972b-BERT/Data/training_mix.arrow', '/data/horse/ws/juha972b-BERT/Data/kernelsynth.arrow']", |
| "probability": "[0.9, 0.1]", |
| "drop_prob": 0.2, |
| "context_length": 512, |
| "prediction_length": 64, |
| "min_past": 60, |
| "max_steps": 200000, |
| "save_steps": 10000, |
| "log_steps": 500, |
| "per_device_train_batch_size": 32, |
| "learning_rate": 0.0001, |
| "optim": "adamw_torch_fused", |
| "shuffle_buffer_length": 10000, |
| "gradient_accumulation_steps": 1, |
| "model_id": "prajjwal1/bert-small", |
| "model_type": "mlm", |
| "random_init": true, |
| "tie_embeddings": false, |
| "output_dir": "/data/horse/ws/juha972b-AION-BERT-Chronos/BERTi/BertModel/BertSmall", |
| "tf32": true, |
| "torch_compile": true, |
| "tokenizer_class": "MeanScaleUniformBins", |
| "tokenizer_kwargs": "{'low_limit': -15.0, 'high_limit': 15.0}", |
| "n_tokens": 4096, |
| "n_special_tokens": 4, |
| "pad_token_id": 0, |
| "bos_token_id": 1, |
| "eos_token_id": 2, |
| "mask_token_id": 3, |
| "use_eos_token": true, |
| "lr_scheduler_type": "linear", |
| "warmup_ratio": 0.01, |
| "dataloader_num_workers": 1, |
| "max_missing_prop": 0.9, |
| "num_samples": 20, |
| "temperature": 1.0, |
| "top_k": 50, |
| "top_p": 1.0, |
| "seed": 270923275, |
| "fp16": false, |
| "bolt": false, |
| "patch_size": 16, |
| "patch_stride": 16, |
| "use_reg_token": true, |
| "limit_model_size": true, |
| "is_gated_act": "False", |
| "config_overrides": "None", |
| "hidden_size": 768, |
| "num_hidden_layers": 12, |
| "num_attention_heads": 12, |
| "intermediate_size": 3072, |
| "hidden_act": "gelu", |
| "hidden_dropout_prob": 0.1, |
| "attention_probs_dropout_prob": 0.1, |
| "layer_norm_eps": 1e-12, |
| "span_masking": false, |
| "mean_span_length": 3, |
| "masking_prob": 0.15, |
| "individual_dropout": 0.05, |
| "unchanged_patch_prob": 0.05, |
| "patch_perturbation_prob": 0.1, |
| "patch_masking_prob": 0.8, |
| "patch_perturbation_scale_strength": 0.1, |
| "patch_perturbation_noise_strength": 0.1, |
| "debug_patching": false, |
| "debug_batch_idx": 0, |
| "debug_max_patches": 3, |
| "task": "mlm", |
| "num_labels": 6, |
| "loss_type": "mae", |
| "TrainInnerModel": true, |
| "dropout_head": 0.1, |
| "adam_beta1": 0.9, |
| "adam_beta2": 0.98, |
| "adam_epsilon": 1e-06, |
| "weight_decay": 0.01, |
| "max_grad_norm": 0.0, |
| "warmup_steps": 0, |
| "num_train_epochs": 1 |
| }, |
| "job_info": { |
| "cuda_available": true, |
| "device_count": 1, |
| "device_names": { |
| "0": "NVIDIA H100" |
| }, |
| "mem_info": { |
| "0": [ |
| 95520423936, |
| 99949674496 |
| ] |
| }, |
| "torchelastic_launched": false, |
| "python_version": "3.12.3 (main, Dec 3 2025, 08:59:03) [GCC 13.3.0]", |
| "torch_version": "2.6.0+cu126", |
| "numpy_version": "1.26.4", |
| "gluonts_version": "0.16.2", |
| "transformers_version": "4.49.0", |
| "accelerate_version": "0.34.2" |
| } |
| } |
