Upload metrics.json with huggingface_hub

metrics.json

@@ -1,18 +1,70 @@
 {
-  "flags": {
-    "fallback_recommended": false,
-    "min_macro_f1_for_push": 0.3
+  "direct_report": {
+    "final_dir": "/content/yandex_disk_cache/ml/subgroup_distillation/group_e/direct/alexyalunin-rubiobert/final",
+    "model_name": "alexyalunin/RuBioBERT",
+    "run_slug": "direct_hard_training",
+    "selection_metric": "hit@3",
+    "selection_metric_value": 0.835,
+    "test_metrics": {
+      "hit@1": 0.628140703517588,
+      "hit@3": 0.8291457286432161,
+      "macro_f1": 0.5157875868327818,
+      "micro_f1": 0.48293963254593175,
+      "mrr": 0.744193071244231,
+      "recall@3": 0.8197654941373534,
+      "subset_accuracy": 0.1708542713567839,
+      "weighted_f1": 0.5126833520048437
+    },
+    "train_duration_sec": 100.4,
+    "train_metrics": {
+      "epoch": 9.0,
+      "total_flos": 2177031722148864.0,
+      "train_loss": 0.47315873063128927,
+      "train_runtime": 100.0873,
+      "train_samples_per_second": 110.184,
+      "train_steps_per_second": 13.788
+    },
+    "val_metrics": {
+      "hit@1": 0.645,
+      "hit@3": 0.835,
+      "macro_f1": 0.5239739935027187,
+      "micro_f1": 0.48812664907651715,
+      "mrr": 0.750362070235822,
+      "recall@3": 0.8273333333333333,
+      "subset_accuracy": 0.175,
+      "weighted_f1": 0.5181260346642617
+    }
+  },
+  "direct_test": {
+    "hit@1": 0.628140703517588,
+    "hit@3": 0.8291457286432161,
+    "macro_f1": 0.5157875868327818,
+    "micro_f1": 0.48293963254593175,
+    "mrr": 0.744193071244231,
+    "recall@3": 0.8197654941373534,
+    "subset_accuracy": 0.1708542713567839,
+    "weighted_f1": 0.5126833520048437
+  },
+  "direct_val": {
+    "hit@1": 0.645,
+    "hit@3": 0.835,
+    "macro_f1": 0.5239739935027187,
+    "micro_f1": 0.48812664907651715,
+    "mrr": 0.750362070235822,
+    "recall@3": 0.8273333333333333,
+    "subset_accuracy": 0.175,
+    "weighted_f1": 0.5181260346642617
   },
   "per_label": [
     {
-      "f1": 0.24390243902439024,
-      "fn": 5,
-      "fp": 26,
+      "f1": 0.3076923076923077,
+      "fn": 2,
+      "fp": 34,
       "label": "E01",
-      "precision": 0.16129032258064516,
-      "recall": 0.5,
+      "precision": 0.19047619047619047,
+      "recall": 0.8,
       "support": 10,
-      "tp": 5
+      "tp": 8
     },
     {
       "f1": 0.0,
@@ -25,44 +77,44 @@
       "tp": 0
     },
     {
-      "f1": 0.16,
+      "f1": 0.15384615384615385,
       "fn": 6,
-      "fp": 15,
+      "fp": 16,
       "label": "E03",
-      "precision": 0.11764705882352941,
+      "precision": 0.1111111111111111,
       "recall": 0.25,
       "support": 8,
       "tp": 2
     },
     {
-      "f1": 0.425531914893617,
-      "fn": 4,
-      "fp": 23,
+      "f1": 0.5,
+      "fn": 3,
+      "fp": 19,
       "label": "E04",
-      "precision": 0.30303030303030304,
-      "recall": 0.7142857142857143,
+      "precision": 0.36666666666666664,
+      "recall": 0.7857142857142857,
       "support": 14,
-      "tp": 10
+      "tp": 11
     },
     {
-      "f1": 0.38461538461538464,
-      "fn": 6,
+      "f1": 0.4444444444444444,
+      "fn": 5,
       "fp": 10,
       "label": "E05",
-      "precision": 0.3333333333333333,
-      "recall": 0.45454545454545453,
+      "precision": 0.375,
+      "recall": 0.5454545454545454,
       "support": 11,
-      "tp": 5
+      "tp": 6
     },
     {
-      "f1": 0.559322033898305,
-      "fn": 3,
-      "fp": 49,
+      "f1": 0.5528455284552846,
+      "fn": 2,
+      "fp": 53,
       "label": "E06",
-      "precision": 0.4024390243902439,
-      "recall": 0.9166666666666666,
+      "precision": 0.39080459770114945,
+      "recall": 0.9444444444444444,
       "support": 36,
-      "tp": 33
+      "tp": 34
     },
     {
       "f1": 0.5,
@@ -75,31 +127,31 @@
       "tp": 1
     },
     {
-      "f1": 0.6666666666666666,
+      "f1": 0.7058823529411765,
       "fn": 1,
-      "fp": 5,
+      "fp": 4,
       "label": "E10",
-      "precision": 0.5454545454545454,
+      "precision": 0.6,
       "recall": 0.8571428571428571,
       "support": 7,
       "tp": 6
     },
     {
-      "f1": 0.6521739130434783,
-      "fn": 1,
-      "fp": 15,
+      "f1": 0.6956521739130435,
+      "fn": 0,
+      "fp": 14,
       "label": "E11",
-      "precision": 0.5,
-      "recall": 0.9375,
+      "precision": 0.5333333333333333,
+      "recall": 1.0,
       "support": 16,
-      "tp": 15
+      "tp": 16
     },
     {
-      "f1": 0.8,
+      "f1": 1.0,
       "fn": 0,
-      "fp": 1,
+      "fp": 0,
       "label": "E12",
-      "precision": 0.6666666666666666,
+      "precision": 1.0,
       "recall": 1.0,
       "support": 2,
       "tp": 2
@@ -145,34 +197,34 @@
       "tp": 1
     },
     {
-      "f1": 0.75,
-      "fn": 1,
-      "fp": 1,
+      "f1": 0.7272727272727273,
+      "fn": 0,
+      "fp": 3,
       "label": "E21",
-      "precision": 0.75,
-      "recall": 0.75,
+      "precision": 0.5714285714285714,
+      "recall": 1.0,
       "support": 4,
-      "tp": 3
+      "tp": 4
     },
     {
-      "f1": 0.4,
-      "fn": 3,
-      "fp": 3,
+      "f1": 0.38095238095238093,
+      "fn": 1,
+      "fp": 12,
       "label": "E22",
-      "precision": 0.4,
-      "recall": 0.4,
+      "precision": 0.25,
+      "recall": 0.8,
       "support": 5,
-      "tp": 2
+      "tp": 4
     },
     {
-      "f1": 0.0,
-      "fn": 2,
-      "fp": 0,
+      "f1": 0.5,
+      "fn": 1,
+      "fp": 1,
       "label": "E23",
-      "precision": 0.0,
-      "recall": 0.0,
+      "precision": 0.5,
+      "recall": 0.5,
       "support": 2,
-      "tp": 0
+      "tp": 1
     },
     {
       "f1": 0.0,
@@ -195,21 +247,21 @@
       "tp": 1
     },
     {
-      "f1": 0.9333333333333333,
+      "f1": 0.4827586206896552,
       "fn": 0,
-      "fp": 1,
+      "fp": 15,
       "label": "E28",
-      "precision": 0.875,
+      "precision": 0.3181818181818182,
       "recall": 1.0,
       "support": 7,
       "tp": 7
     },
     {
-      "f1": 0.6666666666666666,
+      "f1": 0.5,
       "fn": 1,
-      "fp": 0,
+      "fp": 1,
       "label": "E29",
-      "precision": 1.0,
+      "precision": 0.5,
       "recall": 0.5,
       "support": 2,
       "tp": 1
@@ -295,44 +347,44 @@
       "tp": 0
     },
     {
-      "f1": 0.4444444444444444,
+      "f1": 0.6666666666666666,
       "fn": 2,
-      "fp": 3,
+      "fp": 0,
       "label": "E61",
-      "precision": 0.4,
+      "precision": 1.0,
       "recall": 0.5,
       "support": 4,
       "tp": 2
     },
     {
-      "f1": 0.6176470588235294,
-      "fn": 2,
-      "fp": 24,
+      "f1": 0.5806451612903226,
+      "fn": 5,
+      "fp": 21,
       "label": "E66",
-      "precision": 0.4666666666666667,
-      "recall": 0.9130434782608695,
+      "precision": 0.46153846153846156,
+      "recall": 0.782608695652174,
       "support": 23,
-      "tp": 21
+      "tp": 18
     },
     {
-      "f1": 1.0,
+      "f1": 0.6666666666666666,
       "fn": 0,
-      "fp": 0,
+      "fp": 1,
       "label": "E67",
-      "precision": 1.0,
+      "precision": 0.5,
       "recall": 1.0,
       "support": 1,
       "tp": 1
     },
     {
-      "f1": 0.0,
-      "fn": 2,
-      "fp": 0,
+      "f1": 0.5,
+      "fn": 1,
+      "fp": 1,
       "label": "E70",
-      "precision": 0.0,
-      "recall": 0.0,
+      "precision": 0.5,
+      "recall": 0.5,
       "support": 2,
-      "tp": 0
+      "tp": 1
     },
     {
       "f1": 0.6666666666666666,
@@ -355,31 +407,31 @@
       "tp": 2
     },
     {
-      "f1": 0.3333333333333333,
-      "fn": 3,
-      "fp": 5,
+      "f1": 0.5,
+      "fn": 2,
+      "fp": 4,
       "label": "E74",
-      "precision": 0.2857142857142857,
-      "recall": 0.4,
+      "precision": 0.42857142857142855,
+      "recall": 0.6,
       "support": 5,
-      "tp": 2
+      "tp": 3
     },
     {
-      "f1": 0.8333333333333334,
+      "f1": 0.9090909090909091,
       "fn": 0,
-      "fp": 2,
+      "fp": 1,
       "label": "E75",
-      "precision": 0.7142857142857143,
+      "precision": 0.8333333333333334,
       "recall": 1.0,
       "support": 5,
       "tp": 5
     },
     {
-      "f1": 0.7058823529411765,
+      "f1": 0.8,
       "fn": 0,
-      "fp": 5,
+      "fp": 3,
       "label": "E76",
-      "precision": 0.5454545454545454,
+      "precision": 0.6666666666666666,
       "recall": 1.0,
       "support": 6,
       "tp": 6
@@ -395,11 +447,11 @@
       "tp": 0
     },
     {
-      "f1": 1.0,
+      "f1": 0.8571428571428571,
       "fn": 0,
-      "fp": 0,
+      "fp": 1,
       "label": "E79",
-      "precision": 1.0,
+      "precision": 0.75,
       "recall": 1.0,
       "support": 3,
       "tp": 3
@@ -445,44 +497,175 @@
       "tp": 1
     },
     {
-      "f1": 0.6086956521739131,
+      "f1": 0.5833333333333334,
       "fn": 1,
-      "fp": 8,
+      "fp": 9,
       "label": "E89",
-      "precision": 0.4666666666666667,
+      "precision": 0.4375,
       "recall": 0.875,
       "support": 8,
       "tp": 7
     }
   ],
+  "quality_gate": {
+    "direct_hit3_threshold": 0.9,
+    "force_push_weak": true,
+    "hit3_drop_tolerance": 0.02,
+    "hit3_floor": 0.8242211055276382,
+    "macro_f1_drop_tolerance": 0.03,
+    "macro_floor": 0.65047534381331,
+    "min_macro_f1_for_push": 0.3,
+    "passed": false,
+    "push_allowed": true,
+    "push_policy": "always_push_when_push_to_hub_enabled"
+  },
+  "teacher_errors": {},
+  "teacher_reports": [
+    {
+      "final_dir": "/content/yandex_disk_cache/ml/subgroup_distillation/group_e/teachers/alexyalunin-rubioroberta/final",
+      "model_name": "alexyalunin/RuBioRoBERTa",
+      "pushed_to_hub": false,
+      "run_slug": "alexyalunin-rubioroberta",
+      "test_metrics": {
+        "hit@1": 0.6482412060301508,
+        "hit@3": 0.8592964824120602,
+        "macro_f1": 0.6865864313407604,
+        "micro_f1": 0.6485436893203883,
+        "mrr": 0.7635155763702912,
+        "recall@3": 0.8532663316582914,
+        "subset_accuracy": 0.507537688442211,
+        "weighted_f1": 0.6514279861261107
+      },
+      "train_duration_sec": 314.8,
+      "train_metrics": {
+        "epoch": 12.0,
+        "total_flos": 1.0278832248336384e+16,
+        "train_loss": 0.31829168511473616,
+        "train_runtime": 299.4665,
+        "train_samples_per_second": 36.825,
+        "train_steps_per_second": 4.608
+      },
+      "val_metrics": {
+        "hit@1": 0.665,
+        "hit@3": 0.83,
+        "macro_f1": 0.6816071208348493,
+        "micro_f1": 0.6377358490566037,
+        "mrr": 0.7651676992824679,
+        "recall@3": 0.8214999999999999,
+        "subset_accuracy": 0.48,
+        "weighted_f1": 0.654032532287305
+      }
+    },
+    {
+      "final_dir": "/content/yandex_disk_cache/ml/subgroup_distillation/group_e/teachers/ai-forever-rubert-base/final",
+      "model_name": "ai-forever/ruBert-base",
+      "pushed_to_hub": false,
+      "run_slug": "ai-forever-rubert-base",
+      "test_metrics": {
+        "hit@1": 0.6633165829145728,
+        "hit@3": 0.8241206030150754,
+        "macro_f1": 0.567726321569097,
+        "micro_f1": 0.5497630331753555,
+        "mrr": 0.7552527410980668,
+        "recall@3": 0.8164154103852594,
+        "subset_accuracy": 0.3015075376884422,
+        "weighted_f1": 0.5636139940062055
+      },
+      "train_duration_sec": 144.6,
+      "train_metrics": {
+        "epoch": 12.0,
+        "total_flos": 2902708962865152.0,
+        "train_loss": 0.4238858779271444,
+        "train_runtime": 133.7006,
+        "train_samples_per_second": 82.483,
+        "train_steps_per_second": 10.322
+      },
+      "val_metrics": {
+        "hit@1": 0.65,
+        "hit@3": 0.82,
+        "macro_f1": 0.5512155779574014,
+        "micro_f1": 0.5529953917050692,
+        "mrr": 0.7548125116338592,
+        "recall@3": 0.814,
+        "subset_accuracy": 0.315,
+        "weighted_f1": 0.5726346327425935
+      }
+    },
+    {
+      "final_dir": "/content/yandex_disk_cache/ml/subgroup_distillation/group_e/teachers/deeppavlov-rubert-base-cased/final",
+      "model_name": "DeepPavlov/rubert-base-cased",
+      "pushed_to_hub": false,
+      "run_slug": "deeppavlov-rubert-base-cased",
+      "test_metrics": {
+        "hit@1": 0.6130653266331658,
+        "hit@3": 0.7939698492462312,
+        "macro_f1": 0.5623096108520506,
+        "micro_f1": 0.5097451274362819,
+        "mrr": 0.7264908372260924,
+        "recall@3": 0.7879396984924623,
+        "subset_accuracy": 0.2613065326633166,
+        "weighted_f1": 0.5247350484683884
+      },
+      "train_duration_sec": 145.8,
+      "train_metrics": {
+        "epoch": 12.0,
+        "total_flos": 2902708962865152.0,
+        "train_loss": 0.45112168028734734,
+        "train_runtime": 136.666,
+        "train_samples_per_second": 80.693,
+        "train_steps_per_second": 10.098
+      },
+      "val_metrics": {
+        "hit@1": 0.66,
+        "hit@3": 0.81,
+        "macro_f1": 0.5291229985764946,
+        "micro_f1": 0.5123010130246021,
+        "mrr": 0.755827678749699,
+        "recall@3": 0.8014999999999999,
+        "subset_accuracy": 0.23,
+        "weighted_f1": 0.5291843956354477
+      }
+    }
+  ],
+  "teacher_test": {
+    "hit@1": 0.678391959798995,
+    "hit@3": 0.8442211055276382,
+    "macro_f1": 0.68047534381331,
+    "micro_f1": 0.6541353383458647,
+    "mrr": 0.7735248354904848,
+    "recall@3": 0.8381909547738693,
+    "subset_accuracy": 0.4824120603015075,
+    "weighted_f1": 0.6548964986426676
+  },
+  "teacher_val": {
+    "hit@1": 0.675,
+    "hit@3": 0.84,
+    "macro_f1": 0.6592846608011086,
+    "micro_f1": 0.6483516483516484,
+    "mrr": 0.7742923534798536,
+    "recall@3": 0.834,
+    "subset_accuracy": 0.48,
+    "weighted_f1": 0.6615922586454719
+  },
   "test": {
-    "test_hit@1": 0.6432160804020101,
-    "test_hit@3": 0.8140703517587939,
-    "test_loss": 0.36334311962127686,
-    "test_macro_f1": 0.5937529302338868,
-    "test_micro_f1": 0.5618729096989966,
-    "test_mrr": 0.7459331824475639,
-    "test_recall@3": 0.8046901172529313,
-    "test_runtime": 0.7448,
-    "test_samples_per_second": 267.2,
-    "test_steps_per_second": 17.455,
-    "test_subset_accuracy": 0.35678391959798994,
-    "test_weighted_f1": 0.5713189296891098
+    "hit@1": 0.6482412060301508,
+    "hit@3": 0.8241206030150754,
+    "macro_f1": 0.6121827915051391,
+    "micro_f1": 0.5606299212598426,
+    "mrr": 0.7515661740428731,
+    "recall@3": 0.8147403685092127,
+    "subset_accuracy": 0.2964824120603015,
+    "weighted_f1": 0.5830428669342762
   },
   "threshold": 0.5,
   "val_best": {
-    "epoch": 12.0,
-    "val_hit@1": 0.675,
-    "val_hit@3": 0.835,
-    "val_loss": 0.3367632329463959,
-    "val_macro_f1": 0.6232852428734424,
-    "val_micro_f1": 0.5727848101265823,
-    "val_mrr": 0.769334612787883,
-    "val_recall@3": 0.8273333333333333,
-    "val_runtime": 0.6576,
-    "val_samples_per_second": 304.123,
-    "val_steps_per_second": 19.768,
-    "val_subset_accuracy": 0.32,
-    "val_weighted_f1": 0.5951135806524868
+    "hit@1": 0.635,
+    "hit@3": 0.8,
+    "macro_f1": 0.6257847034784294,
+    "micro_f1": 0.5578446909667195,
+    "mrr": 0.7417874066293184,
+    "recall@3": 0.7939999999999999,
+    "subset_accuracy": 0.3,
+    "weighted_f1": 0.581422089009702
   }
 }