Training in progress, epoch 5, checkpoint

checkpoint-535/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b39311a457b4000680a589d4e2309f7aa7625ff5748bc83e74188fd774033f3c
+oid sha256:8215f087a9f5bf98702b4ac5cdeaeeacbe8bd6a88141a191a846ece3c23017dd
 size 174079796

checkpoint-535/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:54430a5b39632bf4eaa8cc2f1e8fc8311713d856a22e1b18b3d46d31bcec682c
+oid sha256:7c347fdff918d0c20191a4a836bdace3d5278f65e88bcdd07258a29726626454
 size 345687577

checkpoint-535/trainer_state.json

@@ -1,5 +1,5 @@
 {
-  "best_metric": 0.0177,
+  "best_metric": 0.0041,
   "best_model_checkpoint": "test_detr_finetuned_cppe5/checkpoint-535",
   "epoch": 5.0,
   "eval_steps": 500,
@@ -11,308 +11,308 @@
     {
       "epoch": 1.0,
       "step": 107,
-      "train_loss": 3.0001344680786133,
-      "train_map": 0.0047,
-      "train_map_50": 0.0176,
-      "train_map_75": 0.0014,
-      "train_map_Coverall": 0.0232,
+      "train_loss": 3.2706406116485596,
+      "train_map": 0.0002,
+      "train_map_50": 0.0006,
+      "train_map_75": 0.0001,
+      "train_map_Coverall": 0.0007,
       "train_map_Face_Shield": 0.0,
       "train_map_Gloves": 0.0001,
       "train_map_Goggles": 0.0,
-      "train_map_Mask": 0.0004,
-      "train_map_large": 0.0049,
-      "train_map_medium": 0.0002,
-      "train_map_small": 0.0016,
-      "train_mar_1": 0.0108,
-      "train_mar_10": 0.0458,
-      "train_mar_100": 0.0837,
-      "train_mar_100_Coverall": 0.3473,
-      "train_mar_100_Face_Shield": 0.0,
-      "train_mar_100_Gloves": 0.0145,
-      "train_mar_100_Goggles": 0.0,
-      "train_mar_100_Mask": 0.0567,
-      "train_mar_large": 0.0731,
-      "train_mar_medium": 0.0246,
-      "train_mar_small": 0.0013,
-      "train_runtime": 83.8338,
-      "train_samples_per_second": 10.139,
-      "train_steps_per_second": 1.276
+      "train_map_Mask": 0.0,
+      "train_map_large": 0.0003,
+      "train_map_medium": 0.0,
+      "train_map_small": 0.0,
+      "train_mar_1": 0.003,
+      "train_mar_10": 0.0087,
+      "train_mar_100": 0.0266,
+      "train_mar_100_Coverall": 0.0956,
+      "train_mar_100_Face_Shield": 0.0006,
+      "train_mar_100_Gloves": 0.034,
+      "train_mar_100_Goggles": 0.0013,
+      "train_mar_100_Mask": 0.0016,
+      "train_mar_large": 0.0244,
+      "train_mar_medium": 0.0105,
+      "train_mar_small": 0.002,
+      "train_runtime": 75.6338,
+      "train_samples_per_second": 11.238,
+      "train_steps_per_second": 1.415
     },
     {
       "epoch": 1.0,
-      "eval_loss": 3.0698611736297607,
-      "eval_map": 0.0038,
-      "eval_map_50": 0.0126,
-      "eval_map_75": 0.0008,
-      "eval_map_Coverall": 0.0183,
+      "eval_loss": 3.281048059463501,
+      "eval_map": 0.0001,
+      "eval_map_50": 0.0005,
+      "eval_map_75": 0.0,
+      "eval_map_Coverall": 0.0005,
       "eval_map_Face_Shield": 0.0,
-      "eval_map_Gloves": 0.0002,
+      "eval_map_Gloves": 0.0001,
       "eval_map_Goggles": 0.0,
-      "eval_map_Mask": 0.0004,
-      "eval_map_large": 0.0044,
-      "eval_map_medium": 0.0002,
-      "eval_map_small": 0.0022,
-      "eval_mar_1": 0.0063,
-      "eval_mar_10": 0.0308,
-      "eval_mar_100": 0.0561,
-      "eval_mar_100_Coverall": 0.2185,
+      "eval_map_Mask": 0.0,
+      "eval_map_large": 0.0002,
+      "eval_map_medium": 0.0001,
+      "eval_map_small": 0.0001,
+      "eval_mar_1": 0.002,
+      "eval_mar_10": 0.0057,
+      "eval_mar_100": 0.0185,
+      "eval_mar_100_Coverall": 0.0676,
       "eval_mar_100_Face_Shield": 0.0,
-      "eval_mar_100_Gloves": 0.0156,
+      "eval_mar_100_Gloves": 0.0246,
       "eval_mar_100_Goggles": 0.0,
-      "eval_mar_100_Mask": 0.0462,
-      "eval_mar_large": 0.05,
-      "eval_mar_medium": 0.0266,
-      "eval_mar_small": 0.0022,
-      "eval_runtime": 43.5223,
-      "eval_samples_per_second": 3.447,
-      "eval_steps_per_second": 0.437,
+      "eval_mar_100_Mask": 0.0004,
+      "eval_mar_large": 0.0155,
+      "eval_mar_medium": 0.0078,
+      "eval_mar_small": 0.0027,
+      "eval_runtime": 41.9051,
+      "eval_samples_per_second": 3.58,
+      "eval_steps_per_second": 0.453,
       "step": 107
     },
     {
       "epoch": 2.0,
       "step": 214,
-      "train_loss": 2.964989423751831,
-      "train_map": 0.0073,
-      "train_map_50": 0.0272,
-      "train_map_75": 0.0018,
-      "train_map_Coverall": 0.0358,
+      "train_loss": 3.312178373336792,
+      "train_map": 0.0021,
+      "train_map_50": 0.0092,
+      "train_map_75": 0.0002,
+      "train_map_Coverall": 0.0106,
       "train_map_Face_Shield": 0.0,
-      "train_map_Gloves": 0.0002,
+      "train_map_Gloves": 0.0,
       "train_map_Goggles": 0.0,
-      "train_map_Mask": 0.0005,
-      "train_map_large": 0.0075,
-      "train_map_medium": 0.0002,
-      "train_map_small": 0.0018,
-      "train_mar_1": 0.0107,
-      "train_mar_10": 0.045,
-      "train_mar_100": 0.0775,
-      "train_mar_100_Coverall": 0.2974,
+      "train_map_Mask": 0.0001,
+      "train_map_large": 0.0023,
+      "train_map_medium": 0.0001,
+      "train_map_small": 0.0001,
+      "train_mar_1": 0.0084,
+      "train_mar_10": 0.0299,
+      "train_mar_100": 0.0442,
+      "train_mar_100_Coverall": 0.2058,
       "train_mar_100_Face_Shield": 0.0,
-      "train_mar_100_Gloves": 0.0302,
+      "train_mar_100_Gloves": 0.015,
       "train_mar_100_Goggles": 0.0,
-      "train_mar_100_Mask": 0.06,
-      "train_mar_large": 0.0627,
-      "train_mar_medium": 0.0311,
-      "train_mar_small": 0.0012,
-      "train_runtime": 86.7545,
-      "train_samples_per_second": 9.798,
-      "train_steps_per_second": 1.233
+      "train_mar_100_Mask": 0.0001,
+      "train_mar_large": 0.0452,
+      "train_mar_medium": 0.0064,
+      "train_mar_small": 0.0003,
+      "train_runtime": 71.4629,
+      "train_samples_per_second": 11.894,
+      "train_steps_per_second": 1.497
     },
     {
       "epoch": 2.0,
-      "eval_loss": 3.0518693923950195,
-      "eval_map": 0.0042,
-      "eval_map_50": 0.0172,
-      "eval_map_75": 0.0009,
-      "eval_map_Coverall": 0.0203,
+      "eval_loss": 3.3038530349731445,
+      "eval_map": 0.002,
+      "eval_map_50": 0.0085,
+      "eval_map_75": 0.0002,
+      "eval_map_Coverall": 0.0098,
       "eval_map_Face_Shield": 0.0,
-      "eval_map_Gloves": 0.0003,
+      "eval_map_Gloves": 0.0,
       "eval_map_Goggles": 0.0,
-      "eval_map_Mask": 0.0004,
-      "eval_map_large": 0.0046,
-      "eval_map_medium": 0.0003,
-      "eval_map_small": 0.0005,
-      "eval_mar_1": 0.0064,
-      "eval_mar_10": 0.0308,
-      "eval_mar_100": 0.0561,
-      "eval_mar_100_Coverall": 0.2045,
+      "eval_map_Mask": 0.0,
+      "eval_map_large": 0.0023,
+      "eval_map_medium": 0.0,
+      "eval_map_small": 0.0002,
+      "eval_mar_1": 0.0058,
+      "eval_mar_10": 0.0224,
+      "eval_mar_100": 0.0351,
+      "eval_mar_100_Coverall": 0.1644,
       "eval_mar_100_Face_Shield": 0.0,
-      "eval_mar_100_Gloves": 0.0339,
+      "eval_mar_100_Gloves": 0.0112,
       "eval_mar_100_Goggles": 0.0,
-      "eval_mar_100_Mask": 0.0422,
-      "eval_mar_large": 0.0468,
-      "eval_mar_medium": 0.0333,
-      "eval_mar_small": 0.0004,
-      "eval_runtime": 41.1994,
-      "eval_samples_per_second": 3.641,
-      "eval_steps_per_second": 0.461,
+      "eval_mar_100_Mask": 0.0,
+      "eval_mar_large": 0.0373,
+      "eval_mar_medium": 0.006,
+      "eval_mar_small": 0.0005,
+      "eval_runtime": 36.9767,
+      "eval_samples_per_second": 4.057,
+      "eval_steps_per_second": 0.514,
       "step": 214
     },
     {
       "epoch": 3.0,
       "step": 321,
-      "train_loss": 2.9912493228912354,
-      "train_map": 0.0034,
-      "train_map_50": 0.0129,
-      "train_map_75": 0.0014,
-      "train_map_Coverall": 0.0166,
-      "train_map_Face_Shield": 0.0003,
+      "train_loss": 3.66046404838562,
+      "train_map": 0.0004,
+      "train_map_50": 0.0022,
+      "train_map_75": 0.0,
+      "train_map_Coverall": 0.0018,
+      "train_map_Face_Shield": 0.0,
       "train_map_Gloves": 0.0,
       "train_map_Goggles": 0.0,
-      "train_map_Mask": 0.0003,
-      "train_map_large": 0.0039,
-      "train_map_medium": 0.0001,
-      "train_map_small": 0.0028,
-      "train_mar_1": 0.0112,
-      "train_mar_10": 0.0458,
-      "train_mar_100": 0.0675,
-      "train_mar_100_Coverall": 0.2772,
-      "train_mar_100_Face_Shield": 0.004,
-      "train_mar_100_Gloves": 0.014,
+      "train_map_Mask": 0.0,
+      "train_map_large": 0.0004,
+      "train_map_medium": 0.0,
+      "train_map_small": 0.0002,
+      "train_mar_1": 0.0025,
+      "train_mar_10": 0.0094,
+      "train_mar_100": 0.0171,
+      "train_mar_100_Coverall": 0.0597,
+      "train_mar_100_Face_Shield": 0.0,
+      "train_mar_100_Gloves": 0.0,
       "train_mar_100_Goggles": 0.0,
-      "train_mar_100_Mask": 0.0425,
-      "train_mar_large": 0.0589,
-      "train_mar_medium": 0.02,
-      "train_mar_small": 0.0021,
-      "train_runtime": 80.1813,
-      "train_samples_per_second": 10.601,
-      "train_steps_per_second": 1.334
+      "train_mar_100_Mask": 0.0259,
+      "train_mar_large": 0.0126,
+      "train_mar_medium": 0.0095,
+      "train_mar_small": 0.0002,
+      "train_runtime": 71.7786,
+      "train_samples_per_second": 11.842,
+      "train_steps_per_second": 1.491
     },
     {
       "epoch": 3.0,
-      "eval_loss": 3.0276060104370117,
-      "eval_map": 0.0029,
-      "eval_map_50": 0.0113,
-      "eval_map_75": 0.0007,
-      "eval_map_Coverall": 0.0137,
-      "eval_map_Face_Shield": 0.0003,
+      "eval_loss": 3.6040291786193848,
+      "eval_map": 0.0007,
+      "eval_map_50": 0.0032,
+      "eval_map_75": 0.0,
+      "eval_map_Coverall": 0.0035,
+      "eval_map_Face_Shield": 0.0,
       "eval_map_Gloves": 0.0,
       "eval_map_Goggles": 0.0,
-      "eval_map_Mask": 0.0003,
-      "eval_map_large": 0.0031,
-      "eval_map_medium": 0.0002,
-      "eval_map_small": 0.001,
-      "eval_mar_1": 0.0057,
-      "eval_mar_10": 0.0326,
-      "eval_mar_100": 0.0506,
-      "eval_mar_100_Coverall": 0.2108,
-      "eval_mar_100_Face_Shield": 0.0063,
-      "eval_mar_100_Gloves": 0.0058,
+      "eval_map_Mask": 0.0,
+      "eval_map_large": 0.0007,
+      "eval_map_medium": 0.0,
+      "eval_map_small": 0.0,
+      "eval_mar_1": 0.0017,
+      "eval_mar_10": 0.0077,
+      "eval_mar_100": 0.0126,
+      "eval_mar_100_Coverall": 0.0486,
+      "eval_mar_100_Face_Shield": 0.0,
+      "eval_mar_100_Gloves": 0.0,
       "eval_mar_100_Goggles": 0.0,
-      "eval_mar_100_Mask": 0.0302,
-      "eval_mar_large": 0.0482,
-      "eval_mar_medium": 0.0184,
-      "eval_mar_small": 0.0009,
-      "eval_runtime": 49.8337,
-      "eval_samples_per_second": 3.01,
-      "eval_steps_per_second": 0.381,
+      "eval_mar_100_Mask": 0.0142,
+      "eval_mar_large": 0.0111,
+      "eval_mar_medium": 0.007,
+      "eval_mar_small": 0.0,
+      "eval_runtime": 36.2619,
+      "eval_samples_per_second": 4.137,
+      "eval_steps_per_second": 0.524,
       "step": 321
     },
     {
       "epoch": 4.0,
       "step": 428,
-      "train_loss": 2.774980068206787,
-      "train_map": 0.0215,
-      "train_map_50": 0.0526,
-      "train_map_75": 0.0145,
-      "train_map_Coverall": 0.1068,
-      "train_map_Face_Shield": 0.0003,
-      "train_map_Gloves": 0.0001,
+      "train_loss": 3.2305893898010254,
+      "train_map": 0.001,
+      "train_map_50": 0.0041,
+      "train_map_75": 0.0001,
+      "train_map_Coverall": 0.0052,
+      "train_map_Face_Shield": 0.0,
+      "train_map_Gloves": 0.0,
       "train_map_Goggles": 0.0,
-      "train_map_Mask": 0.0003,
-      "train_map_large": 0.0226,
-      "train_map_medium": 0.0001,
-      "train_map_small": 0.0025,
-      "train_mar_1": 0.0295,
-      "train_mar_10": 0.08,
-      "train_mar_100": 0.106,
-      "train_mar_100_Coverall": 0.431,
-      "train_mar_100_Face_Shield": 0.0397,
-      "train_mar_100_Gloves": 0.0193,
+      "train_map_Mask": 0.0,
+      "train_map_large": 0.0011,
+      "train_map_medium": 0.0,
+      "train_map_small": 0.0004,
+      "train_mar_1": 0.0027,
+      "train_mar_10": 0.0234,
+      "train_mar_100": 0.054,
+      "train_mar_100_Coverall": 0.2667,
+      "train_mar_100_Face_Shield": 0.0,
+      "train_mar_100_Gloves": 0.0025,
       "train_mar_100_Goggles": 0.0,
-      "train_mar_100_Mask": 0.0401,
-      "train_mar_large": 0.1015,
-      "train_mar_medium": 0.0255,
-      "train_mar_small": 0.005,
-      "train_runtime": 75.8168,
-      "train_samples_per_second": 11.211,
-      "train_steps_per_second": 1.411
+      "train_mar_100_Mask": 0.0008,
+      "train_mar_large": 0.0563,
+      "train_mar_medium": 0.001,
+      "train_mar_small": 0.0001,
+      "train_runtime": 68.5556,
+      "train_samples_per_second": 12.399,
+      "train_steps_per_second": 1.561
     },
     {
       "epoch": 4.0,
-      "eval_loss": 2.858452558517456,
-      "eval_map": 0.0149,
-      "eval_map_50": 0.0389,
-      "eval_map_75": 0.0106,
-      "eval_map_Coverall": 0.0739,
-      "eval_map_Face_Shield": 0.0001,
+      "eval_loss": 3.286728858947754,
+      "eval_map": 0.0006,
+      "eval_map_50": 0.0028,
+      "eval_map_75": 0.0001,
+      "eval_map_Coverall": 0.0032,
+      "eval_map_Face_Shield": 0.0,
       "eval_map_Gloves": 0.0,
       "eval_map_Goggles": 0.0,
-      "eval_map_Mask": 0.0005,
-      "eval_map_large": 0.0169,
-      "eval_map_medium": 0.0001,
-      "eval_map_small": 0.003,
-      "eval_mar_1": 0.018,
-      "eval_mar_10": 0.051,
-      "eval_mar_100": 0.0751,
-      "eval_mar_100_Coverall": 0.3243,
-      "eval_mar_100_Face_Shield": 0.0063,
-      "eval_mar_100_Gloves": 0.0098,
+      "eval_map_Mask": 0.0,
+      "eval_map_large": 0.0007,
+      "eval_map_medium": 0.0,
+      "eval_map_small": 0.0,
+      "eval_mar_1": 0.0016,
+      "eval_mar_10": 0.0152,
+      "eval_mar_100": 0.0368,
+      "eval_mar_100_Coverall": 0.1833,
+      "eval_mar_100_Face_Shield": 0.0,
+      "eval_mar_100_Gloves": 0.0004,
       "eval_mar_100_Goggles": 0.0,
-      "eval_mar_100_Mask": 0.0351,
-      "eval_mar_large": 0.0756,
-      "eval_mar_medium": 0.0173,
-      "eval_mar_small": 0.0055,
-      "eval_runtime": 39.8886,
-      "eval_samples_per_second": 3.76,
-      "eval_steps_per_second": 0.476,
+      "eval_mar_100_Mask": 0.0004,
+      "eval_mar_large": 0.042,
+      "eval_mar_medium": 0.0004,
+      "eval_mar_small": 0.0,
+      "eval_runtime": 35.94,
+      "eval_samples_per_second": 4.174,
+      "eval_steps_per_second": 0.529,
       "step": 428
     },
     {
       "epoch": 4.672897196261682,
-      "grad_norm": 42.34880065917969,
+      "grad_norm": 96.62178802490234,
       "learning_rate": 4.97310930008156e-05,
-      "loss": 2.8744,
+      "loss": 5.244,
       "step": 500
     },
     {
       "epoch": 5.0,
       "step": 535,
-      "train_loss": 2.7138683795928955,
-      "train_map": 0.0264,
-      "train_map_50": 0.0577,
-      "train_map_75": 0.0211,
-      "train_map_Coverall": 0.1315,
-      "train_map_Face_Shield": 0.0001,
-      "train_map_Gloves": 0.0001,
+      "train_loss": 3.113837957382202,
+      "train_map": 0.0047,
+      "train_map_50": 0.0141,
+      "train_map_75": 0.0021,
+      "train_map_Coverall": 0.0236,
+      "train_map_Face_Shield": 0.0,
+      "train_map_Gloves": 0.0,
       "train_map_Goggles": 0.0,
-      "train_map_Mask": 0.0004,
-      "train_map_large": 0.0278,
-      "train_map_medium": 0.0002,
-      "train_map_small": 0.0006,
-      "train_mar_1": 0.0329,
-      "train_mar_10": 0.0861,
-      "train_mar_100": 0.1129,
-      "train_mar_100_Coverall": 0.4616,
-      "train_mar_100_Face_Shield": 0.0175,
-      "train_mar_100_Gloves": 0.0374,
+      "train_map_Mask": 0.0,
+      "train_map_large": 0.005,
+      "train_map_medium": 0.0,
+      "train_map_small": 0.0,
+      "train_mar_1": 0.0147,
+      "train_mar_10": 0.0356,
+      "train_mar_100": 0.0795,
+      "train_mar_100_Coverall": 0.3905,
+      "train_mar_100_Face_Shield": 0.0,
+      "train_mar_100_Gloves": 0.0061,
       "train_mar_100_Goggles": 0.0,
-      "train_mar_100_Mask": 0.048,
-      "train_mar_large": 0.1028,
-      "train_mar_medium": 0.033,
-      "train_mar_small": 0.0004,
-      "train_runtime": 83.2574,
-      "train_samples_per_second": 10.209,
-      "train_steps_per_second": 1.285
+      "train_mar_100_Mask": 0.0007,
+      "train_mar_large": 0.0824,
+      "train_mar_medium": 0.0023,
+      "train_mar_small": 0.0,
+      "train_runtime": 69.5355,
+      "train_samples_per_second": 12.224,
+      "train_steps_per_second": 1.539
     },
     {
       "epoch": 5.0,
-      "eval_loss": 2.8448832035064697,
-      "eval_map": 0.0177,
-      "eval_map_50": 0.0404,
-      "eval_map_75": 0.0141,
-      "eval_map_Coverall": 0.0879,
-      "eval_map_Face_Shield": 0.0001,
-      "eval_map_Gloves": 0.0001,
+      "eval_loss": 3.183504581451416,
+      "eval_map": 0.0041,
+      "eval_map_50": 0.0123,
+      "eval_map_75": 0.0031,
+      "eval_map_Coverall": 0.0203,
+      "eval_map_Face_Shield": 0.0,
+      "eval_map_Gloves": 0.0,
       "eval_map_Goggles": 0.0,
-      "eval_map_Mask": 0.0004,
-      "eval_map_large": 0.02,
-      "eval_map_medium": 0.0002,
+      "eval_map_Mask": 0.0,
+      "eval_map_large": 0.0046,
+      "eval_map_medium": 0.0,
       "eval_map_small": 0.0,
-      "eval_mar_1": 0.0218,
-      "eval_mar_10": 0.0578,
-      "eval_mar_100": 0.0829,
-      "eval_mar_100_Coverall": 0.3347,
-      "eval_mar_100_Face_Shield": 0.0076,
-      "eval_mar_100_Gloves": 0.0263,
+      "eval_mar_1": 0.0092,
+      "eval_mar_10": 0.0232,
+      "eval_mar_100": 0.0572,
+      "eval_mar_100_Coverall": 0.2829,
+      "eval_mar_100_Face_Shield": 0.0,
+      "eval_mar_100_Gloves": 0.0031,
       "eval_mar_100_Goggles": 0.0,
-      "eval_mar_100_Mask": 0.0458,
-      "eval_mar_large": 0.0793,
-      "eval_mar_medium": 0.0343,
+      "eval_mar_100_Mask": 0.0,
+      "eval_mar_large": 0.0647,
+      "eval_mar_medium": 0.0012,
       "eval_mar_small": 0.0,
-      "eval_runtime": 40.7733,
-      "eval_samples_per_second": 3.679,
-      "eval_steps_per_second": 0.466,
+      "eval_runtime": 36.8768,
+      "eval_samples_per_second": 4.068,
+      "eval_steps_per_second": 0.515,
       "step": 535
     }
   ],

checkpoint-535/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dfb5b1b016fe6c9e411ad0aa21f5650b9f5bdfae6d91c31b103f2e514cbab6ff
+oid sha256:70ab6612c795f2330a136ddf97d7c6e0ba7798175cc9cae994cc3f067700c650
 size 5176