{
  "_hub_mixin_config": {
    "val_split": null,
    "image_size": 96,
    "patch_size": 4,
    "in_channels": 3,
    "hidden_size": 384,
    "num_hidden_layers": 12,
    "num_attention_heads": 3,
    "qkv_bias": true,
    "intermediate_size": 768,
    "dropout_hidden": 0.0,
    "dropout_attention": 0.0,
    "dropout_path": 0.05,
    "dino_out_dim": 4096,
    "dino_use_bn": true,
    "dino_norm_last_layer": true,
    "dino_num_layers": 3,
    "dino_hidden_dim": 1024,
    "dino_bottleneck_dim": 256,
    "dino_base_teacher_temp": 0.04,
    "dino_final_teacher_temp": 0.04,
    "dino_warmup_epochs": 0,
    "num_local_crops": 6,
    "local_crop_size": 48,
    "global_crops_scale": [
      0.4,
      1.0
    ],
    "local_crops_scale": [
      0.05,
      0.4
    ],
    "solarization": 0.2,
    "gaussian": [
      1.0,
      0.5
    ],
    "checkpoint": null,
    "batch_size": 64,
    "num_epochs": 100,
    "learning_rate": 0.000125,
    "optimizer_class": "adamw",
    "base_wd": 0.04,
    "final_wd": 0.4,
    "base_momentum": 0.996,
    "final_momentum": 1.0,
    "lr_scheduler_class": "cosine",
    "warmup_ratio": 0.1,
    "log_interval_steps": 15,
    "save_interval_steps": 315,
    "save_dir": "./saved_modelcls/vit-t4-chatgpt-hyperparams",
    "save_latest": true,
    "save_best": true,
    "loss_metric_for_best_model": "train",
    "use_wandb": true,
    "wandb_entity": "image-ssl",
    "wandb_project": "pretraining",
    "wandb_name": "vit-t4-chatgpt-hyperparams",
    "upload_model_to_hub": true,
    "repo_id": "image-ssl/vit-t4-chatgpt-hyperparams",
    "device": "cuda:0",
    "seed": 42,
    "total_steps": 781200
  },
  "hf_api": "<huggingface_hub.hf_api.HfApi object at 0x14e5868dd3a0>",
  "wandb_writer": "<wandb.sdk.wandb_run.Run object at 0x14e57f3612e0>",
  "wandb_table": null,
  "optimizer": "AdamW (\nParameter Group 0\n    amsgrad: False\n    betas: (0.9, 0.999)\n    capturable: False\n    decoupled_weight_decay: True\n    differentiable: False\n    eps: 1e-08\n    foreach: None\n    fused: None\n    initial_lr: 0.000125\n    lr: 3.7449596774193655e-06\n    maximize: False\n    weight_decay: 0.040003606000574454\n)",
  "lr_scheduler": "<torch.optim.lr_scheduler.SequentialLR object at 0x14e586a1dca0>",
  "wd_scheduler": "<trainers.schedulers.weight_decay.WeightDecayScheduler object at 0x14e57e241820>",
  "momentum_scheduler": "<trainers.schedulers.momentum.MomentumScheduler object at 0x14e57e241850>",
  "optimizer_class": "adamw",
  "lr_scheduler_class": "cosine",
  "student_model": "VisionTransformerWithPretrainingHeads(\n  (encoder): VisionTransformer(\n    (patch_embed): PatchEmbedding(\n      (proj): Conv2d(3, 384, kernel_size=(4, 4), stride=(4, 4))\n    )\n    (pos_drop): Dropout(p=0.0, inplace=False)\n    (blocks): ModuleList(\n      (0): TransformerBlock(\n        (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (qkv): Linear(in_features=384, out_features=1152, bias=True)\n          (proj): Linear(in_features=384, out_features=384, bias=True)\n          (proj_drop): Dropout(p=0.0, inplace=False)\n        )\n        (drop_path_attn): Identity()\n        (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (fc1): Linear(in_features=384, out_features=768, bias=True)\n          (act): GELU(approximate='none')\n          (fc2): Linear(in_features=768, out_features=384, bias=True)\n          (drop): Dropout(p=0.0, inplace=False)\n        )\n        (drop_path_mlp): Identity()\n      )\n      (1-11): 11 x TransformerBlock(\n        (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (qkv): Linear(in_features=384, out_features=1152, bias=True)\n          (proj): Linear(in_features=384, out_features=384, bias=True)\n          (proj_drop): Dropout(p=0.0, inplace=False)\n        )\n        (drop_path_attn): DropPath()\n        (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (fc1): Linear(in_features=384, out_features=768, bias=True)\n          (act): GELU(approximate='none')\n          (fc2): Linear(in_features=768, out_features=384, bias=True)\n          (drop): Dropout(p=0.0, inplace=False)\n        )\n        (drop_path_mlp): DropPath()\n      )\n    )\n    (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n  )\n  (heads): ModuleDict(\n    (dino): DINOHead(\n      (mlp): Sequential(\n        (0): Linear(in_features=384, out_features=1024, bias=True)\n        (1): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (2): GELU(approximate='none')\n        (3): Linear(in_features=1024, out_features=1024, bias=True)\n        (4): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (5): GELU(approximate='none')\n        (6): Linear(in_features=1024, out_features=256, bias=True)\n      )\n      (last_layer): Linear(in_features=256, out_features=4096, bias=False)\n    )\n  )\n)",
  "teacher_model": "VisionTransformerWithPretrainingHeads(\n  (encoder): VisionTransformer(\n    (patch_embed): PatchEmbedding(\n      (proj): Conv2d(3, 384, kernel_size=(4, 4), stride=(4, 4))\n    )\n    (pos_drop): Dropout(p=0.0, inplace=False)\n    (blocks): ModuleList(\n      (0): TransformerBlock(\n        (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (qkv): Linear(in_features=384, out_features=1152, bias=True)\n          (proj): Linear(in_features=384, out_features=384, bias=True)\n          (proj_drop): Dropout(p=0.0, inplace=False)\n        )\n        (drop_path_attn): Identity()\n        (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (fc1): Linear(in_features=384, out_features=768, bias=True)\n          (act): GELU(approximate='none')\n          (fc2): Linear(in_features=768, out_features=384, bias=True)\n          (drop): Dropout(p=0.0, inplace=False)\n        )\n        (drop_path_mlp): Identity()\n      )\n      (1-11): 11 x TransformerBlock(\n        (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (qkv): Linear(in_features=384, out_features=1152, bias=True)\n          (proj): Linear(in_features=384, out_features=384, bias=True)\n          (proj_drop): Dropout(p=0.0, inplace=False)\n        )\n        (drop_path_attn): DropPath()\n        (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (fc1): Linear(in_features=384, out_features=768, bias=True)\n          (act): GELU(approximate='none')\n          (fc2): Linear(in_features=768, out_features=384, bias=True)\n          (drop): Dropout(p=0.0, inplace=False)\n        )\n        (drop_path_mlp): DropPath()\n      )\n    )\n    (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n  )\n  (heads): ModuleDict(\n    (dino): DINOHead(\n      (mlp): Sequential(\n        (0): Linear(in_features=384, out_features=1024, bias=True)\n        (1): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (2): GELU(approximate='none')\n        (3): Linear(in_features=1024, out_features=1024, bias=True)\n        (4): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (5): GELU(approximate='none')\n        (6): Linear(in_features=1024, out_features=256, bias=True)\n      )\n      (last_layer): Linear(in_features=256, out_features=4096, bias=False)\n    )\n  )\n)",
  "learning_rate": 0.000125,
  "_dino_loss": "DINOLoss()"
}