Training in progress, step 4

README.md

@@ -21,7 +21,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [DedalusHealthCare/tinybert-mlm-en](https://huggingface.co/DedalusHealthCare/tinybert-mlm-en) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6019
+- Loss: 0.5703
 - Accuracy: 0.9816
 - F1: 0.0
 - Precision: 0.0
@@ -60,15 +60,15 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Accuracy | F1     | Precision | Recall |
 |:-------------:|:------:|:----:|:---------------:|:--------:|:------:|:---------:|:------:|
-| 0.7327        | 0.2857 | 1    | 0.7513          | 0.0341   | 0.0366 | 0.0187    | 1.0    |
-| 0.7327        | 0.5714 | 2    | 0.7464          | 0.0525   | 0.0373 | 0.0190    | 1.0    |
-| 0.7327        | 0.8571 | 3    | 0.7368          | 0.0892   | 0.0388 | 0.0198    | 1.0    |
-| 0.7327        | 1.1429 | 4    | 0.7225          | 0.1916   | 0.0375 | 0.0192    | 0.8571 |
-| 0.7327        | 1.4286 | 5    | 0.7037          | 0.3885   | 0.0251 | 0.0129    | 0.4286 |
-| 0.7327        | 1.7143 | 6    | 0.6806          | 0.6325   | 0.0    | 0.0       | 0.0    |
-| 0.7327        | 2.0    | 7    | 0.6538          | 0.8688   | 0.0    | 0.0       | 0.0    |
-| 0.7327        | 2.2857 | 8    | 0.6274          | 0.9738   | 0.0    | 0.0       | 0.0    |
-| 0.7327        | 2.5714 | 9    | 0.6019          | 0.9816   | 0.0    | 0.0       | 0.0    |
+| 0.703         | 0.2857 | 1    | 0.7166          | 0.2310   | 0.0456 | 0.0233    | 1.0    |
+| 0.703         | 0.5714 | 2    | 0.7117          | 0.2677   | 0.0412 | 0.0211    | 0.8571 |
+| 0.703         | 0.8571 | 3    | 0.7020          | 0.3701   | 0.04   | 0.0206    | 0.7143 |
+| 0.703         | 1.1429 | 4    | 0.6875          | 0.5774   | 0.0359 | 0.0187    | 0.4286 |
+| 0.703         | 1.4286 | 5    | 0.6686          | 0.7874   | 0.0    | 0.0       | 0.0    |
+| 0.703         | 1.7143 | 6    | 0.6458          | 0.9423   | 0.0    | 0.0       | 0.0    |
+| 0.703         | 2.0    | 7    | 0.6198          | 0.9816   | 0.0    | 0.0       | 0.0    |
+| 0.703         | 2.2857 | 8    | 0.5946          | 0.9816   | 0.0    | 0.0       | 0.0    |
+| 0.703         | 2.5714 | 9    | 0.5703          | 0.9816   | 0.0    | 0.0       | 0.0    |
 
 
 ### Framework versions

evaluation_results_en/classification_report_goldset.json

@@ -1,34 +1,34 @@
 {
     "NOT_FAMILY": {
-        "precision": 0.4482758620689655,
-        "recall": 0.8863636363636364,
-        "f1-score": 0.5954198473282443,
-        "support": 44.0
+        "precision": 1.0,
+        "recall": 0.13471502590673576,
+        "f1-score": 0.2374429223744292,
+        "support": 386.0
     },
     "FAMILY": {
-        "precision": 0.6153846153846154,
-        "recall": 0.14285714285714285,
-        "f1-score": 0.23188405797101452,
-        "support": 56.0
+        "precision": 0.014749262536873156,
+        "recall": 1.0,
+        "f1-score": 0.029069767441860465,
+        "support": 5.0
     },
-    "accuracy": 0.47,
+    "accuracy": 0.14578005115089515,
     "macro avg": {
-        "precision": 0.5318302387267905,
-        "recall": 0.5146103896103896,
-        "f1-score": 0.4136519526496294,
-        "support": 100.0
+        "precision": 0.5073746312684366,
+        "recall": 0.5673575129533679,
+        "f1-score": 0.13325634490814484,
+        "support": 391.0
     },
     "weighted avg": {
-        "precision": 0.5418567639257295,
-        "recall": 0.47,
-        "f1-score": 0.3918398052881956,
-        "support": 100.0
+        "precision": 0.9874008857101901,
+        "recall": 0.14578005115089515,
+        "f1-score": 0.23477830402490785,
+        "support": 391.0
     },
     "evaluation_metadata": {
         "num_classes": 2,
         "eval_type": "Binary",
-        "dataset_source_type": "goldset",
-        "dataset_path": "/data/scl_demo_en/evaluation/research_development/scl_demo_en_goldset.jsonl",
-        "is_fallback": false
+        "dataset_source_type": "huggingface_hub",
+        "dataset_path": "DedalusHealthCare/scl_demo_en",
+        "is_fallback": true
     }
 }

logs/events.out.tfevents.1760001356.1be56bd00f27.673.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c181b888f8284702c952fc62200d29d10f177b608ac2449f45d6c267ee1686fc
+size 7160

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8aacff2c15f2be28bc6a28dbe6b917ce1a64cb9b4ea7b535bffebc91aed3c342
+oid sha256:0a499650b10cae4e2bdde16b75ec4aa4a18717889acb124c1fad024b7e51c65a
 size 46634832

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dd6ab76cdb6290642263495ef9ccbac4bce2acbc40c68819c1dc3fde0b088978
-size 5240
+oid sha256:7a4cc4e9eff3450f9d5c31bd5bf3c299968002ed6b50bab6a7a9d4b76137fbc6
+size 5176

training_artifacts/training_history.csv

@@ -1,12 +1,12 @@
 loss;grad_norm;learning_rate;epoch;step;eval_loss;eval_accuracy;eval_f1;eval_precision;eval_recall;eval_runtime;eval_samples_per_second;eval_steps_per_second;train_runtime;train_samples_per_second;train_steps_per_second;total_flos;train_loss
-0.7327;2.7726080417633057;3.3333333333333333e-06;0.2857142857142857;1;;;;;;;;;;;;;
-;;;0.2857142857142857;1;0.7512667775154114;0.03412073490813648;0.03664921465968587;0.018666666666666668;1.0;5.3956;70.613;4.448;;;;;
-;;;0.5714285714285714;2;0.7464109659194946;0.05249343832020997;0.037333333333333336;0.019021739130434784;1.0;5.5933;68.117;4.291;;;;;
-;;;0.8571428571428571;3;0.7368069291114807;0.08923884514435695;0.038781163434903045;0.01977401129943503;1.0;5.5621;68.5;4.315;;;;;
-;;;1.1428571428571428;4;0.7224730849266052;0.19160104986876642;0.03749999999999999;0.019169329073482427;0.8571428571428571;5.6279;67.698;4.264;;;;;
-;;;1.4285714285714286;5;0.7036912441253662;0.3884514435695538;0.02510460251046025;0.01293103448275862;0.42857142857142855;5.703;66.807;4.208;;;;;
-;;;1.7142857142857144;6;0.6806334853172302;0.6325459317585301;0.0;0.0;0.0;5.773;65.997;4.157;;;;;
-;;;2.0;7;0.6538448929786682;0.868766404199475;0.0;0.0;0.0;5.6817;67.057;4.224;;;;;
-;;;2.2857142857142856;8;0.6274169683456421;0.973753280839895;0.0;0.0;0.0;5.8624;64.991;4.094;;;;;
-;;;2.571428571428571;9;0.6018883585929871;0.9816272965879265;0.0;0.0;0.0;5.644;67.505;4.252;;;;;
-;;;2.571428571428571;9;;;;;;;;;101.6832;41.698;0.59;2156353020864.0;0.6984957986407809
+0.703;2.2138702869415283;3.3333333333333333e-06;0.2857142857142857;1;;;;;;;;;;;;;
+;;;0.2857142857142857;1;0.7165892720222473;0.23097112860892388;0.04560260586319218;0.023333333333333334;1.0;0.7454;511.131;32.197;;;;;
+;;;0.5714285714285714;2;0.7117474675178528;0.2677165354330709;0.041237113402061855;0.02112676056338028;0.8571428571428571;1.5578;244.576;15.406;;;;;
+;;;0.8571428571428571;3;0.7019844055175781;0.3700787401574803;0.04;0.0205761316872428;0.7142857142857143;2.8847;132.075;8.32;;;;;
+;;;1.1428571428571428;4;0.6875269412994385;0.5774278215223098;0.03592814371257485;0.01875;0.42857142857142855;2.1358;178.386;11.237;;;;;
+;;;1.4285714285714286;5;0.6685800552368164;0.7874015748031497;0.0;0.0;0.0;0.7158;532.274;33.529;;;;;
+;;;1.7142857142857144;6;0.6458192467689514;0.9422572178477691;0.0;0.0;0.0;1.2139;313.874;19.772;;;;;
+;;;2.0;7;0.6197686195373535;0.9816272965879265;0.0;0.0;0.0;0.7728;493.017;31.056;;;;;
+;;;2.2857142857142856;8;0.5946394205093384;0.9816272965879265;0.0;0.0;0.0;0.7291;522.579;32.918;;;;;
+;;;2.571428571428571;9;0.5703445672988892;0.9816272965879265;0.0;0.0;0.0;0.9339;407.972;25.699;;;;;
+;;;2.571428571428571;9;;;;;;;;;22.1708;191.242;2.706;2172566201472.0;0.6836379369099935

training_artifacts/training_history.json

@@ -1,125 +1,125 @@
 [
   {
-    "loss": 0.7327,
-    "grad_norm": 2.7726080417633057,
+    "loss": 0.703,
+    "grad_norm": 2.2138702869415283,
     "learning_rate": 3.3333333333333333e-06,
     "epoch": 0.2857142857142857,
     "step": 1
   },
   {
-    "eval_loss": 0.7512667775154114,
-    "eval_accuracy": 0.03412073490813648,
-    "eval_f1": 0.03664921465968587,
-    "eval_precision": 0.018666666666666668,
+    "eval_loss": 0.7165892720222473,
+    "eval_accuracy": 0.23097112860892388,
+    "eval_f1": 0.04560260586319218,
+    "eval_precision": 0.023333333333333334,
     "eval_recall": 1.0,
-    "eval_runtime": 5.3956,
-    "eval_samples_per_second": 70.613,
-    "eval_steps_per_second": 4.448,
+    "eval_runtime": 0.7454,
+    "eval_samples_per_second": 511.131,
+    "eval_steps_per_second": 32.197,
     "epoch": 0.2857142857142857,
     "step": 1
   },
   {
-    "eval_loss": 0.7464109659194946,
-    "eval_accuracy": 0.05249343832020997,
-    "eval_f1": 0.037333333333333336,
-    "eval_precision": 0.019021739130434784,
-    "eval_recall": 1.0,
-    "eval_runtime": 5.5933,
-    "eval_samples_per_second": 68.117,
-    "eval_steps_per_second": 4.291,
+    "eval_loss": 0.7117474675178528,
+    "eval_accuracy": 0.2677165354330709,
+    "eval_f1": 0.041237113402061855,
+    "eval_precision": 0.02112676056338028,
+    "eval_recall": 0.8571428571428571,
+    "eval_runtime": 1.5578,
+    "eval_samples_per_second": 244.576,
+    "eval_steps_per_second": 15.406,
     "epoch": 0.5714285714285714,
     "step": 2
   },
   {
-    "eval_loss": 0.7368069291114807,
-    "eval_accuracy": 0.08923884514435695,
-    "eval_f1": 0.038781163434903045,
-    "eval_precision": 0.01977401129943503,
-    "eval_recall": 1.0,
-    "eval_runtime": 5.5621,
-    "eval_samples_per_second": 68.5,
-    "eval_steps_per_second": 4.315,
+    "eval_loss": 0.7019844055175781,
+    "eval_accuracy": 0.3700787401574803,
+    "eval_f1": 0.04,
+    "eval_precision": 0.0205761316872428,
+    "eval_recall": 0.7142857142857143,
+    "eval_runtime": 2.8847,
+    "eval_samples_per_second": 132.075,
+    "eval_steps_per_second": 8.32,
     "epoch": 0.8571428571428571,
     "step": 3
   },
   {
-    "eval_loss": 0.7224730849266052,
-    "eval_accuracy": 0.19160104986876642,
-    "eval_f1": 0.03749999999999999,
-    "eval_precision": 0.019169329073482427,
-    "eval_recall": 0.8571428571428571,
-    "eval_runtime": 5.6279,
-    "eval_samples_per_second": 67.698,
-    "eval_steps_per_second": 4.264,
+    "eval_loss": 0.6875269412994385,
+    "eval_accuracy": 0.5774278215223098,
+    "eval_f1": 0.03592814371257485,
+    "eval_precision": 0.01875,
+    "eval_recall": 0.42857142857142855,
+    "eval_runtime": 2.1358,
+    "eval_samples_per_second": 178.386,
+    "eval_steps_per_second": 11.237,
     "epoch": 1.1428571428571428,
     "step": 4
   },
   {
-    "eval_loss": 0.7036912441253662,
-    "eval_accuracy": 0.3884514435695538,
-    "eval_f1": 0.02510460251046025,
-    "eval_precision": 0.01293103448275862,
-    "eval_recall": 0.42857142857142855,
-    "eval_runtime": 5.703,
-    "eval_samples_per_second": 66.807,
-    "eval_steps_per_second": 4.208,
+    "eval_loss": 0.6685800552368164,
+    "eval_accuracy": 0.7874015748031497,
+    "eval_f1": 0.0,
+    "eval_precision": 0.0,
+    "eval_recall": 0.0,
+    "eval_runtime": 0.7158,
+    "eval_samples_per_second": 532.274,
+    "eval_steps_per_second": 33.529,
     "epoch": 1.4285714285714286,
     "step": 5
   },
   {
-    "eval_loss": 0.6806334853172302,
-    "eval_accuracy": 0.6325459317585301,
+    "eval_loss": 0.6458192467689514,
+    "eval_accuracy": 0.9422572178477691,
     "eval_f1": 0.0,
     "eval_precision": 0.0,
     "eval_recall": 0.0,
-    "eval_runtime": 5.773,
-    "eval_samples_per_second": 65.997,
-    "eval_steps_per_second": 4.157,
+    "eval_runtime": 1.2139,
+    "eval_samples_per_second": 313.874,
+    "eval_steps_per_second": 19.772,
     "epoch": 1.7142857142857144,
     "step": 6
   },
   {
-    "eval_loss": 0.6538448929786682,
-    "eval_accuracy": 0.868766404199475,
+    "eval_loss": 0.6197686195373535,
+    "eval_accuracy": 0.9816272965879265,
     "eval_f1": 0.0,
     "eval_precision": 0.0,
     "eval_recall": 0.0,
-    "eval_runtime": 5.6817,
-    "eval_samples_per_second": 67.057,
-    "eval_steps_per_second": 4.224,
+    "eval_runtime": 0.7728,
+    "eval_samples_per_second": 493.017,
+    "eval_steps_per_second": 31.056,
     "epoch": 2.0,
     "step": 7
   },
   {
-    "eval_loss": 0.6274169683456421,
-    "eval_accuracy": 0.973753280839895,
+    "eval_loss": 0.5946394205093384,
+    "eval_accuracy": 0.9816272965879265,
     "eval_f1": 0.0,
     "eval_precision": 0.0,
     "eval_recall": 0.0,
-    "eval_runtime": 5.8624,
-    "eval_samples_per_second": 64.991,
-    "eval_steps_per_second": 4.094,
+    "eval_runtime": 0.7291,
+    "eval_samples_per_second": 522.579,
+    "eval_steps_per_second": 32.918,
     "epoch": 2.2857142857142856,
     "step": 8
   },
   {
-    "eval_loss": 0.6018883585929871,
+    "eval_loss": 0.5703445672988892,
     "eval_accuracy": 0.9816272965879265,
     "eval_f1": 0.0,
     "eval_precision": 0.0,
     "eval_recall": 0.0,
-    "eval_runtime": 5.644,
-    "eval_samples_per_second": 67.505,
-    "eval_steps_per_second": 4.252,
+    "eval_runtime": 0.9339,
+    "eval_samples_per_second": 407.972,
+    "eval_steps_per_second": 25.699,
     "epoch": 2.571428571428571,
     "step": 9
   },
   {
-    "train_runtime": 101.6832,
-    "train_samples_per_second": 41.698,
-    "train_steps_per_second": 0.59,
-    "total_flos": 2156353020864.0,
-    "train_loss": 0.6984957986407809,
+    "train_runtime": 22.1708,
+    "train_samples_per_second": 191.242,
+    "train_steps_per_second": 2.706,
+    "total_flos": 2172566201472.0,
+    "train_loss": 0.6836379369099935,
     "epoch": 2.571428571428571,
     "step": 9
   }

training_artifacts/training_summary.json

@@ -1,7 +1,7 @@
 {
   "total_epochs": 2.571428571428571,
   "total_steps": "9",
-  "final_train_loss": 0.7327,
-  "final_eval_loss": 0.6018883585929871,
-  "best_eval_loss": 0.6018883585929871
+  "final_train_loss": 0.703,
+  "final_eval_loss": 0.5703445672988892,
+  "best_eval_loss": 0.5703445672988892
 }