| { | |
| "_hub_mixin_config": { | |
| "val_split": null, | |
| "image_size": 96, | |
| "patch_size": 4, | |
| "in_channels": 3, | |
| "hidden_size": 384, | |
| "num_hidden_layers": 12, | |
| "num_attention_heads": 3, | |
| "qkv_bias": true, | |
| "intermediate_size": 768, | |
| "dropout_hidden": 0.0, | |
| "dropout_attention": 0.0, | |
| "dropout_path": 0.05, | |
| "dino_out_dim": 4096, | |
| "dino_use_bn": true, | |
| "dino_norm_last_layer": true, | |
| "dino_num_layers": 3, | |
| "dino_hidden_dim": 1024, | |
| "dino_bottleneck_dim": 256, | |
| "dino_base_teacher_temp": 0.04, | |
| "dino_final_teacher_temp": 0.04, | |
| "dino_warmup_epochs": 0, | |
| "num_local_crops": 6, | |
| "local_crop_size": 48, | |
| "global_crops_scale": [ | |
| 0.4, | |
| 1.0 | |
| ], | |
| "local_crops_scale": [ | |
| 0.05, | |
| 0.4 | |
| ], | |
| "solarization": 0.2, | |
| "gaussian": [ | |
| 1.0, | |
| 0.5 | |
| ], | |
| "checkpoint": null, | |
| "batch_size": 64, | |
| "num_epochs": 100, | |
| "learning_rate": 0.000125, | |
| "optimizer_class": "adamw", | |
| "base_wd": 0.04, | |
| "final_wd": 0.4, | |
| "base_momentum": 0.996, | |
| "final_momentum": 1.0, | |
| "lr_scheduler_class": "cosine", | |
| "warmup_ratio": 0.1, | |
| "log_interval_steps": 15, | |
| "save_interval_steps": 315, | |
| "save_dir": "./saved_modelcls/vit-t4-chatgpt-hyperparams", | |
| "save_latest": true, | |
| "save_best": true, | |
| "loss_metric_for_best_model": "train", | |
| "use_wandb": true, | |
| "wandb_entity": "image-ssl", | |
| "wandb_project": "pretraining", | |
| "wandb_name": "vit-t4-chatgpt-hyperparams", | |
| "upload_model_to_hub": true, | |
| "repo_id": "image-ssl/vit-t4-chatgpt-hyperparams", | |
| "device": "cuda:0", | |
| "seed": 42, | |
| "total_steps": 781200 | |
| }, | |
| "hf_api": "<huggingface_hub.hf_api.HfApi object at 0x14e5868dd3a0>", | |
| "wandb_writer": "<wandb.sdk.wandb_run.Run object at 0x14e57f3612e0>", | |
| "wandb_table": null, | |
| "optimizer": "AdamW (\nParameter Group 0\n amsgrad: False\n betas: (0.9, 0.999)\n capturable: False\n decoupled_weight_decay: True\n differentiable: False\n eps: 1e-08\n foreach: None\n fused: None\n initial_lr: 0.000125\n lr: 3.7449596774193655e-06\n maximize: False\n weight_decay: 0.040003606000574454\n)", | |
| "lr_scheduler": "<torch.optim.lr_scheduler.SequentialLR object at 0x14e586a1dca0>", | |
| "wd_scheduler": "<trainers.schedulers.weight_decay.WeightDecayScheduler object at 0x14e57e241820>", | |
| "momentum_scheduler": "<trainers.schedulers.momentum.MomentumScheduler object at 0x14e57e241850>", | |
| "optimizer_class": "adamw", | |
| "lr_scheduler_class": "cosine", | |
| "student_model": "VisionTransformerWithPretrainingHeads(\n (encoder): VisionTransformer(\n (patch_embed): PatchEmbedding(\n (proj): Conv2d(3, 384, kernel_size=(4, 4), stride=(4, 4))\n )\n (pos_drop): Dropout(p=0.0, inplace=False)\n (blocks): ModuleList(\n (0): TransformerBlock(\n (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n (attn): Attention(\n (qkv): Linear(in_features=384, out_features=1152, bias=True)\n (proj): Linear(in_features=384, out_features=384, bias=True)\n (proj_drop): Dropout(p=0.0, inplace=False)\n )\n (drop_path_attn): Identity()\n (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n (mlp): MLP(\n (fc1): Linear(in_features=384, out_features=768, bias=True)\n (act): GELU(approximate='none')\n (fc2): Linear(in_features=768, out_features=384, bias=True)\n (drop): Dropout(p=0.0, inplace=False)\n )\n (drop_path_mlp): Identity()\n )\n (1-11): 11 x TransformerBlock(\n (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n (attn): Attention(\n (qkv): Linear(in_features=384, out_features=1152, bias=True)\n (proj): Linear(in_features=384, out_features=384, bias=True)\n (proj_drop): Dropout(p=0.0, inplace=False)\n )\n (drop_path_attn): DropPath()\n (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n (mlp): MLP(\n (fc1): Linear(in_features=384, out_features=768, bias=True)\n (act): GELU(approximate='none')\n (fc2): Linear(in_features=768, out_features=384, bias=True)\n (drop): Dropout(p=0.0, inplace=False)\n )\n (drop_path_mlp): DropPath()\n )\n )\n (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n )\n (heads): ModuleDict(\n (dino): DINOHead(\n (mlp): Sequential(\n (0): Linear(in_features=384, out_features=1024, bias=True)\n (1): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (2): GELU(approximate='none')\n (3): Linear(in_features=1024, out_features=1024, bias=True)\n (4): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (5): GELU(approximate='none')\n (6): Linear(in_features=1024, out_features=256, bias=True)\n )\n (last_layer): Linear(in_features=256, out_features=4096, bias=False)\n )\n )\n)", | |
| "teacher_model": "VisionTransformerWithPretrainingHeads(\n (encoder): VisionTransformer(\n (patch_embed): PatchEmbedding(\n (proj): Conv2d(3, 384, kernel_size=(4, 4), stride=(4, 4))\n )\n (pos_drop): Dropout(p=0.0, inplace=False)\n (blocks): ModuleList(\n (0): TransformerBlock(\n (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n (attn): Attention(\n (qkv): Linear(in_features=384, out_features=1152, bias=True)\n (proj): Linear(in_features=384, out_features=384, bias=True)\n (proj_drop): Dropout(p=0.0, inplace=False)\n )\n (drop_path_attn): Identity()\n (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n (mlp): MLP(\n (fc1): Linear(in_features=384, out_features=768, bias=True)\n (act): GELU(approximate='none')\n (fc2): Linear(in_features=768, out_features=384, bias=True)\n (drop): Dropout(p=0.0, inplace=False)\n )\n (drop_path_mlp): Identity()\n )\n (1-11): 11 x TransformerBlock(\n (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n (attn): Attention(\n (qkv): Linear(in_features=384, out_features=1152, bias=True)\n (proj): Linear(in_features=384, out_features=384, bias=True)\n (proj_drop): Dropout(p=0.0, inplace=False)\n )\n (drop_path_attn): DropPath()\n (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n (mlp): MLP(\n (fc1): Linear(in_features=384, out_features=768, bias=True)\n (act): GELU(approximate='none')\n (fc2): Linear(in_features=768, out_features=384, bias=True)\n (drop): Dropout(p=0.0, inplace=False)\n )\n (drop_path_mlp): DropPath()\n )\n )\n (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n )\n (heads): ModuleDict(\n (dino): DINOHead(\n (mlp): Sequential(\n (0): Linear(in_features=384, out_features=1024, bias=True)\n (1): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (2): GELU(approximate='none')\n (3): Linear(in_features=1024, out_features=1024, bias=True)\n (4): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (5): GELU(approximate='none')\n (6): Linear(in_features=1024, out_features=256, bias=True)\n )\n (last_layer): Linear(in_features=256, out_features=4096, bias=False)\n )\n )\n)", | |
| "learning_rate": 0.000125, | |
| "_dino_loss": "DINOLoss()" | |
| } |