| { | |
| "_hub_mixin_config": { | |
| "val_split": null, | |
| "image_size": 96, | |
| "patch_size": 8, | |
| "in_channels": 3, | |
| "hidden_size": 384, | |
| "num_hidden_layers": 12, | |
| "num_attention_heads": 6, | |
| "qkv_bias": true, | |
| "intermediate_size": 1536, | |
| "dropout_hidden": 0.0, | |
| "dropout_attention": 0.0, | |
| "dropout_path": 0.0, | |
| "dino_out_dim": 16384, | |
| "dino_use_bn": true, | |
| "dino_norm_last_layer": true, | |
| "dino_num_layers": 3, | |
| "dino_hidden_dim": 2048, | |
| "dino_bottleneck_dim": 256, | |
| "dino_base_teacher_temp": 0.04, | |
| "dino_final_teacher_temp": 0.04, | |
| "dino_warmup_epochs": 0, | |
| "num_local_crops": 4, | |
| "local_crop_size": 48, | |
| "global_crops_scale": [ | |
| 0.7, | |
| 1.0 | |
| ], | |
| "local_crops_scale": [ | |
| 0.3, | |
| 0.7 | |
| ], | |
| "checkpoint": null, | |
| "batch_size": 256, | |
| "num_epochs": 100, | |
| "learning_rate": 0.0002, | |
| "optimizer_class": "adamw", | |
| "base_wd": 0.04, | |
| "final_wd": 0.4, | |
| "base_momentum": 0.996, | |
| "final_momentum": 1.0, | |
| "lr_scheduler_class": "cosine", | |
| "warmup_ratio": 0.1, | |
| "log_interval_steps": 15, | |
| "save_interval_steps": 315, | |
| "save_dir": "./saved_models/vit-s8-highOutDim", | |
| "save_latest": true, | |
| "save_best": true, | |
| "loss_metric_for_best_model": "train", | |
| "use_wandb": true, | |
| "wandb_entity": "image-ssl", | |
| "wandb_project": "pretraining", | |
| "wandb_name": "vit-s8-highOutDim", | |
| "upload_model_to_hub": true, | |
| "repo_id": "image-ssl/vit-s8-highOutDim", | |
| "device": "cuda:0", | |
| "seed": 42, | |
| "total_steps": 195300 | |
| }, | |
| "hf_api": "<huggingface_hub.hf_api.HfApi object at 0x14e641e1e600>", | |
| "wandb_writer": "<wandb.sdk.wandb_run.Run object at 0x14e642ade690>", | |
| "wandb_table": null, | |
| "optimizer": "AdamW (\nParameter Group 0\n amsgrad: False\n betas: (0.9, 0.999)\n capturable: False\n decoupled_weight_decay: True\n differentiable: False\n eps: 1e-08\n foreach: None\n fused: None\n initial_lr: 0.0002\n lr: 2.179999999999991e-05\n maximize: False\n weight_decay: 0.0400887282083115\n)", | |
| "lr_scheduler": "<torch.optim.lr_scheduler.SequentialLR object at 0x14e642129e50>", | |
| "wd_scheduler": "<trainers.schedulers.weight_decay.WeightDecayScheduler object at 0x14e642129f40>", | |
| "momentum_scheduler": "<trainers.schedulers.momentum.MomentumScheduler object at 0x14e642129e80>", | |
| "optimizer_class": "adamw", | |
| "lr_scheduler_class": "cosine", | |
| "student_model": "VisionTransformerWithPretrainingHeads(\n (encoder): VisionTransformer(\n (patch_embed): PatchEmbedding(\n (proj): Conv2d(3, 384, kernel_size=(8, 8), stride=(8, 8))\n )\n (pos_drop): Dropout(p=0.0, inplace=False)\n (blocks): ModuleList(\n (0-11): 12 x TransformerBlock(\n (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n (attn): Attention(\n (qkv): Linear(in_features=384, out_features=1152, bias=True)\n (proj): Linear(in_features=384, out_features=384, bias=True)\n (proj_drop): Dropout(p=0.0, inplace=False)\n )\n (drop_path_attn): Identity()\n (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n (mlp): MLP(\n (fc1): Linear(in_features=384, out_features=1536, bias=True)\n (act): GELU(approximate='none')\n (fc2): Linear(in_features=1536, out_features=384, bias=True)\n (drop): Dropout(p=0.0, inplace=False)\n )\n (drop_path_mlp): Identity()\n )\n )\n (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n )\n (heads): ModuleDict(\n (dino): DINOHead(\n (mlp): Sequential(\n (0): Linear(in_features=384, out_features=2048, bias=True)\n (1): BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (2): GELU(approximate='none')\n (3): Linear(in_features=2048, out_features=2048, bias=True)\n (4): BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (5): GELU(approximate='none')\n (6): Linear(in_features=2048, out_features=256, bias=True)\n )\n (last_layer): Linear(in_features=256, out_features=16384, bias=False)\n )\n )\n)", | |
| "teacher_model": "VisionTransformerWithPretrainingHeads(\n (encoder): VisionTransformer(\n (patch_embed): PatchEmbedding(\n (proj): Conv2d(3, 384, kernel_size=(8, 8), stride=(8, 8))\n )\n (pos_drop): Dropout(p=0.0, inplace=False)\n (blocks): ModuleList(\n (0-11): 12 x TransformerBlock(\n (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n (attn): Attention(\n (qkv): Linear(in_features=384, out_features=1152, bias=True)\n (proj): Linear(in_features=384, out_features=384, bias=True)\n (proj_drop): Dropout(p=0.0, inplace=False)\n )\n (drop_path_attn): Identity()\n (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n (mlp): MLP(\n (fc1): Linear(in_features=384, out_features=1536, bias=True)\n (act): GELU(approximate='none')\n (fc2): Linear(in_features=1536, out_features=384, bias=True)\n (drop): Dropout(p=0.0, inplace=False)\n )\n (drop_path_mlp): Identity()\n )\n )\n (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n )\n (heads): ModuleDict(\n (dino): DINOHead(\n (mlp): Sequential(\n (0): Linear(in_features=384, out_features=2048, bias=True)\n (1): BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (2): GELU(approximate='none')\n (3): Linear(in_features=2048, out_features=2048, bias=True)\n (4): BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (5): GELU(approximate='none')\n (6): Linear(in_features=2048, out_features=256, bias=True)\n )\n (last_layer): Linear(in_features=256, out_features=16384, bias=False)\n )\n )\n)", | |
| "learning_rate": 0.0002, | |
| "_dino_loss": "DINOLoss()" | |
| } |