Training Step 315

README.md

@@ -0,0 +1,10 @@
+---
+tags:
+- model_hub_mixin
+- pytorch_model_hub_mixin
+---
+
+This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration:
+- Code: [More Information Needed]
+- Paper: [More Information Needed]
+- Docs: [More Information Needed]

config.json

@@ -0,0 +1,19 @@
+{
+  "dino_bottleneck_dim": 256,
+  "dino_hidden_dim": 1024,
+  "dino_norm_last_layer": true,
+  "dino_num_layers": 3,
+  "dino_out_dim": 4096,
+  "dino_use_bn": true,
+  "dropout_attention": 0.0,
+  "dropout_hidden": 0.0,
+  "dropout_path": 0.1,
+  "hidden_size": 384,
+  "image_size": 96,
+  "in_channels": 3,
+  "intermediate_size": 1536,
+  "num_attention_heads": 6,
+  "num_hidden_layers": 12,
+  "patch_size": 4,
+  "qkv_bias": true
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b1971c8d31feccbde6c5331ced230b570141a15916042d0088457db6afbe33b
+size 97225416

trainer_state.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31bde77b8ea3078d292b0cd3956e666b050a10c86f2ad0f88349e9581b8f567a
+size 186100579

training_config.json

@@ -0,0 +1,79 @@
+{
+  "_hub_mixin_config": {
+    "val_split": null,
+    "image_size": 96,
+    "patch_size": 4,
+    "in_channels": 3,
+    "hidden_size": 384,
+    "num_hidden_layers": 12,
+    "num_attention_heads": 6,
+    "qkv_bias": true,
+    "intermediate_size": 1536,
+    "dropout_hidden": 0.0,
+    "dropout_attention": 0.0,
+    "dropout_path": 0.1,
+    "dino_out_dim": 4096,
+    "dino_use_bn": true,
+    "dino_norm_last_layer": true,
+    "dino_num_layers": 3,
+    "dino_hidden_dim": 1024,
+    "dino_bottleneck_dim": 256,
+    "dino_base_teacher_temp": 0.04,
+    "dino_final_teacher_temp": 0.04,
+    "dino_warmup_epochs": 0,
+    "num_local_crops": 8,
+    "local_crop_size": 48,
+    "global_crops_scale": [
+      0.4,
+      1.0
+    ],
+    "local_crops_scale": [
+      0.05,
+      0.4
+    ],
+    "solarization": 0.2,
+    "gaussian": [
+      1.0,
+      0.5
+    ],
+    "checkpoint": null,
+    "batch_size": 32,
+    "num_epochs": 100,
+    "learning_rate": 6.25e-05,
+    "optimizer_class": "adamw",
+    "base_wd": 0.04,
+    "final_wd": 0.4,
+    "base_momentum": 0.996,
+    "final_momentum": 1.0,
+    "lr_scheduler_class": "cosine",
+    "warmup_ratio": 0.1,
+    "log_interval_steps": 15,
+    "save_interval_steps": 315,
+    "save_dir": "./saved_modelcls/vit-s4-chatgpt-hyperparams",
+    "save_latest": true,
+    "save_best": true,
+    "loss_metric_for_best_model": "train",
+    "use_wandb": true,
+    "wandb_entity": "image-ssl",
+    "wandb_project": "pretraining",
+    "wandb_name": "vit-s4-chatgpt-hyperparams",
+    "upload_model_to_hub": true,
+    "repo_id": "image-ssl/vit-s4-chatgpt-hyperparams",
+    "device": "cuda:0",
+    "seed": 42,
+    "total_steps": 1562500
+  },
+  "hf_api": "<huggingface_hub.hf_api.HfApi object at 0x14d8adf96f00>",
+  "wandb_writer": "<wandb.sdk.wandb_run.Run object at 0x14d8ae47d280>",
+  "wandb_table": null,
+  "optimizer": "AdamW (\nParameter Group 0\n    amsgrad: False\n    betas: (0.9, 0.999)\n    capturable: False\n    decoupled_weight_decay: True\n    differentiable: False\n    eps: 1e-08\n    foreach: None\n    fused: None\n    initial_lr: 6.25e-05\n    lr: 7.497399999999969e-07\n    maximize: False\n    weight_decay: 0.040000035872486794\n)",
+  "lr_scheduler": "<torch.optim.lr_scheduler.SequentialLR object at 0x14d8ad275340>",
+  "wd_scheduler": "<trainers.schedulers.weight_decay.WeightDecayScheduler object at 0x14d8ae8119a0>",
+  "momentum_scheduler": "<trainers.schedulers.momentum.MomentumScheduler object at 0x14d8ae811940>",
+  "optimizer_class": "adamw",
+  "lr_scheduler_class": "cosine",
+  "student_model": "VisionTransformerWithPretrainingHeads(\n  (encoder): VisionTransformer(\n    (patch_embed): PatchEmbedding(\n      (proj): Conv2d(3, 384, kernel_size=(4, 4), stride=(4, 4))\n    )\n    (pos_drop): Dropout(p=0.0, inplace=False)\n    (blocks): ModuleList(\n      (0): TransformerBlock(\n        (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (qkv): Linear(in_features=384, out_features=1152, bias=True)\n          (proj): Linear(in_features=384, out_features=384, bias=True)\n          (proj_drop): Dropout(p=0.0, inplace=False)\n        )\n        (drop_path_attn): Identity()\n        (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (fc1): Linear(in_features=384, out_features=1536, bias=True)\n          (act): GELU(approximate='none')\n          (fc2): Linear(in_features=1536, out_features=384, bias=True)\n          (drop): Dropout(p=0.0, inplace=False)\n        )\n        (drop_path_mlp): Identity()\n      )\n      (1-11): 11 x TransformerBlock(\n        (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (qkv): Linear(in_features=384, out_features=1152, bias=True)\n          (proj): Linear(in_features=384, out_features=384, bias=True)\n          (proj_drop): Dropout(p=0.0, inplace=False)\n        )\n        (drop_path_attn): DropPath()\n        (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (fc1): Linear(in_features=384, out_features=1536, bias=True)\n          (act): GELU(approximate='none')\n          (fc2): Linear(in_features=1536, out_features=384, bias=True)\n          (drop): Dropout(p=0.0, inplace=False)\n        )\n        (drop_path_mlp): DropPath()\n      )\n    )\n    (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n  )\n  (heads): ModuleDict(\n    (dino): DINOHead(\n      (mlp): Sequential(\n        (0): Linear(in_features=384, out_features=1024, bias=True)\n        (1): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (2): GELU(approximate='none')\n        (3): Linear(in_features=1024, out_features=1024, bias=True)\n        (4): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (5): GELU(approximate='none')\n        (6): Linear(in_features=1024, out_features=256, bias=True)\n      )\n      (last_layer): Linear(in_features=256, out_features=4096, bias=False)\n    )\n  )\n)",
+  "teacher_model": "VisionTransformerWithPretrainingHeads(\n  (encoder): VisionTransformer(\n    (patch_embed): PatchEmbedding(\n      (proj): Conv2d(3, 384, kernel_size=(4, 4), stride=(4, 4))\n    )\n    (pos_drop): Dropout(p=0.0, inplace=False)\n    (blocks): ModuleList(\n      (0): TransformerBlock(\n        (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (qkv): Linear(in_features=384, out_features=1152, bias=True)\n          (proj): Linear(in_features=384, out_features=384, bias=True)\n          (proj_drop): Dropout(p=0.0, inplace=False)\n        )\n        (drop_path_attn): Identity()\n        (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (fc1): Linear(in_features=384, out_features=1536, bias=True)\n          (act): GELU(approximate='none')\n          (fc2): Linear(in_features=1536, out_features=384, bias=True)\n          (drop): Dropout(p=0.0, inplace=False)\n        )\n        (drop_path_mlp): Identity()\n      )\n      (1-11): 11 x TransformerBlock(\n        (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (qkv): Linear(in_features=384, out_features=1152, bias=True)\n          (proj): Linear(in_features=384, out_features=384, bias=True)\n          (proj_drop): Dropout(p=0.0, inplace=False)\n        )\n        (drop_path_attn): DropPath()\n        (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (fc1): Linear(in_features=384, out_features=1536, bias=True)\n          (act): GELU(approximate='none')\n          (fc2): Linear(in_features=1536, out_features=384, bias=True)\n          (drop): Dropout(p=0.0, inplace=False)\n        )\n        (drop_path_mlp): DropPath()\n      )\n    )\n    (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n  )\n  (heads): ModuleDict(\n    (dino): DINOHead(\n      (mlp): Sequential(\n        (0): Linear(in_features=384, out_features=1024, bias=True)\n        (1): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (2): GELU(approximate='none')\n        (3): Linear(in_features=1024, out_features=1024, bias=True)\n        (4): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (5): GELU(approximate='none')\n        (6): Linear(in_features=1024, out_features=256, bias=True)\n      )\n      (last_layer): Linear(in_features=256, out_features=4096, bias=False)\n    )\n  )\n)",
+  "learning_rate": 6.25e-05,
+  "_dino_loss": "DINOLoss()"
+}