Training Step 315

README.md

@@ -0,0 +1,10 @@
+---
+tags:
+- model_hub_mixin
+- pytorch_model_hub_mixin
+---
+
+This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration:
+- Code: [More Information Needed]
+- Paper: [More Information Needed]
+- Docs: [More Information Needed]

config.json

@@ -0,0 +1,19 @@
+{
+  "dino_bottleneck_dim": 256,
+  "dino_hidden_dim": 2048,
+  "dino_norm_last_layer": true,
+  "dino_num_layers": 3,
+  "dino_out_dim": 16384,
+  "dino_use_bn": true,
+  "dropout_attention": 0.0,
+  "dropout_hidden": 0.0,
+  "dropout_path": 0.0,
+  "hidden_size": 384,
+  "image_size": 96,
+  "in_channels": 3,
+  "intermediate_size": 1536,
+  "num_attention_heads": 6,
+  "num_hidden_layers": 12,
+  "patch_size": 8,
+  "qkv_bias": true
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09bed1e0d89889aae2f289010b766c1cadb8fccf4ab76f69bbc98e8ddec971bc
+size 124660456

trainer_state.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2aa1061a97cc424d0b35921ca39679bf6a74dd5c88045643ef8da14f835cdb0d
+size 215673699

training_config.json

@@ -0,0 +1,74 @@
+{
+  "_hub_mixin_config": {
+    "val_split": null,
+    "image_size": 96,
+    "patch_size": 8,
+    "in_channels": 3,
+    "hidden_size": 384,
+    "num_hidden_layers": 12,
+    "num_attention_heads": 6,
+    "qkv_bias": true,
+    "intermediate_size": 1536,
+    "dropout_hidden": 0.0,
+    "dropout_attention": 0.0,
+    "dropout_path": 0.0,
+    "dino_out_dim": 16384,
+    "dino_use_bn": true,
+    "dino_norm_last_layer": true,
+    "dino_num_layers": 3,
+    "dino_hidden_dim": 2048,
+    "dino_bottleneck_dim": 256,
+    "dino_base_teacher_temp": 0.04,
+    "dino_final_teacher_temp": 0.04,
+    "dino_warmup_epochs": 0,
+    "num_local_crops": 4,
+    "local_crop_size": 48,
+    "global_crops_scale": [
+      0.7,
+      1.0
+    ],
+    "local_crops_scale": [
+      0.3,
+      0.7
+    ],
+    "checkpoint": null,
+    "batch_size": 256,
+    "num_epochs": 100,
+    "learning_rate": 0.0002,
+    "optimizer_class": "adamw",
+    "base_wd": 0.04,
+    "final_wd": 0.4,
+    "base_momentum": 0.996,
+    "final_momentum": 1.0,
+    "lr_scheduler_class": "cosine",
+    "warmup_ratio": 0.1,
+    "log_interval_steps": 15,
+    "save_interval_steps": 315,
+    "save_dir": "./saved_models/vit-s8-highOutDim",
+    "save_latest": true,
+    "save_best": true,
+    "loss_metric_for_best_model": "train",
+    "use_wandb": true,
+    "wandb_entity": "image-ssl",
+    "wandb_project": "pretraining",
+    "wandb_name": "vit-s8-highOutDim",
+    "upload_model_to_hub": true,
+    "repo_id": "image-ssl/vit-s8-highOutDim",
+    "device": "cuda:0",
+    "seed": 42,
+    "total_steps": 195300
+  },
+  "hf_api": "<huggingface_hub.hf_api.HfApi object at 0x14e641e1e600>",
+  "wandb_writer": "<wandb.sdk.wandb_run.Run object at 0x14e642ade690>",
+  "wandb_table": null,
+  "optimizer": "AdamW (\nParameter Group 0\n    amsgrad: False\n    betas: (0.9, 0.999)\n    capturable: False\n    decoupled_weight_decay: True\n    differentiable: False\n    eps: 1e-08\n    foreach: None\n    fused: None\n    initial_lr: 0.0002\n    lr: 5.1935483870967665e-06\n    maximize: False\n    weight_decay: 0.040002296128245685\n)",
+  "lr_scheduler": "<torch.optim.lr_scheduler.SequentialLR object at 0x14e642129e50>",
+  "wd_scheduler": "<trainers.schedulers.weight_decay.WeightDecayScheduler object at 0x14e642129f40>",
+  "momentum_scheduler": "<trainers.schedulers.momentum.MomentumScheduler object at 0x14e642129e80>",
+  "optimizer_class": "adamw",
+  "lr_scheduler_class": "cosine",
+  "student_model": "VisionTransformerWithPretrainingHeads(\n  (encoder): VisionTransformer(\n    (patch_embed): PatchEmbedding(\n      (proj): Conv2d(3, 384, kernel_size=(8, 8), stride=(8, 8))\n    )\n    (pos_drop): Dropout(p=0.0, inplace=False)\n    (blocks): ModuleList(\n      (0-11): 12 x TransformerBlock(\n        (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (qkv): Linear(in_features=384, out_features=1152, bias=True)\n          (proj): Linear(in_features=384, out_features=384, bias=True)\n          (proj_drop): Dropout(p=0.0, inplace=False)\n        )\n        (drop_path_attn): Identity()\n        (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (fc1): Linear(in_features=384, out_features=1536, bias=True)\n          (act): GELU(approximate='none')\n          (fc2): Linear(in_features=1536, out_features=384, bias=True)\n          (drop): Dropout(p=0.0, inplace=False)\n        )\n        (drop_path_mlp): Identity()\n      )\n    )\n    (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n  )\n  (heads): ModuleDict(\n    (dino): DINOHead(\n      (mlp): Sequential(\n        (0): Linear(in_features=384, out_features=2048, bias=True)\n        (1): BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (2): GELU(approximate='none')\n        (3): Linear(in_features=2048, out_features=2048, bias=True)\n        (4): BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (5): GELU(approximate='none')\n        (6): Linear(in_features=2048, out_features=256, bias=True)\n      )\n      (last_layer): Linear(in_features=256, out_features=16384, bias=False)\n    )\n  )\n)",
+  "teacher_model": "VisionTransformerWithPretrainingHeads(\n  (encoder): VisionTransformer(\n    (patch_embed): PatchEmbedding(\n      (proj): Conv2d(3, 384, kernel_size=(8, 8), stride=(8, 8))\n    )\n    (pos_drop): Dropout(p=0.0, inplace=False)\n    (blocks): ModuleList(\n      (0-11): 12 x TransformerBlock(\n        (norm1): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n        (attn): Attention(\n          (qkv): Linear(in_features=384, out_features=1152, bias=True)\n          (proj): Linear(in_features=384, out_features=384, bias=True)\n          (proj_drop): Dropout(p=0.0, inplace=False)\n        )\n        (drop_path_attn): Identity()\n        (norm2): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n        (mlp): MLP(\n          (fc1): Linear(in_features=384, out_features=1536, bias=True)\n          (act): GELU(approximate='none')\n          (fc2): Linear(in_features=1536, out_features=384, bias=True)\n          (drop): Dropout(p=0.0, inplace=False)\n        )\n        (drop_path_mlp): Identity()\n      )\n    )\n    (norm): LayerNorm((384,), eps=1e-05, elementwise_affine=True)\n  )\n  (heads): ModuleDict(\n    (dino): DINOHead(\n      (mlp): Sequential(\n        (0): Linear(in_features=384, out_features=2048, bias=True)\n        (1): BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (2): GELU(approximate='none')\n        (3): Linear(in_features=2048, out_features=2048, bias=True)\n        (4): BatchNorm1d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (5): GELU(approximate='none')\n        (6): Linear(in_features=2048, out_features=256, bias=True)\n      )\n      (last_layer): Linear(in_features=256, out_features=16384, bias=False)\n    )\n  )\n)",
+  "learning_rate": 0.0002,
+  "_dino_loss": "DINOLoss()"
+}