Upload spellchecker evaluation

spellcheck_report.json

@@ -0,0 +1,219 @@
+{
+  "config": {
+    "model": "LiquidAI/LFM2.5-Encoder-350M-Spellchecker",
+    "output_dir": "/tmp/minima-spellcheck",
+    "output_repo": "ProCreations/minima-spellcheck",
+    "dataset": "HuggingFaceFW/fineweb-edu",
+    "dataset_config": "sample-10BT",
+    "sequence_length": 128,
+    "batch_size": 2,
+    "gradient_accumulation": 4,
+    "steps": 1000,
+    "learning_rate": 5e-05,
+    "group_size": 32,
+    "recovery_rank": 128,
+    "topk": 16,
+    "eval_batches": 8,
+    "seed": 77,
+    "log_every": 50
+  },
+  "duration_seconds": 394.26561307907104,
+  "trainable_parameters": 57020424,
+  "tagger_label_top1_candidate_agreement": 0.9951171875,
+  "detection_top1_agreement": 0.994140625,
+  "correction_exact_agreement": 0.7,
+  "correction_evaluation_examples": 20,
+  "example_exact_agreement": 1.0,
+  "examples": [
+    {
+      "input": "She go to school every day .",
+      "teacher": "She goes to school every day .",
+      "minima": "She goes to school every day ."
+    },
+    {
+      "input": "I has went to the stor yesterday .",
+      "teacher": "I went to the store yesterday .",
+      "minima": "I went to the store yesterday ."
+    },
+    {
+      "input": "Their are many reason to study hard .",
+      "teacher": "There are many reasons to study hard .",
+      "minima": "There are many reasons to study hard ."
+    },
+    {
+      "input": "That 's a fair point , let 's discuss it tomorrow .",
+      "teacher": "That 's a fair point , let 's discuss it tomorrow .",
+      "minima": "That 's a fair point , let 's discuss it tomorrow ."
+    }
+  ],
+  "history": [
+    {
+      "step": 1,
+      "elapsed_s": 9.728960990905762,
+      "loss": 2.4879839420318604,
+      "label": 1.3699460625648499,
+      "detect": 0.3631512150168419,
+      "hidden": 0.4682311415672302
+    },
+    {
+      "step": 51,
+      "elapsed_s": 47.39888334274292,
+      "loss": 0.5646287947893143,
+      "label": 0.21763740107417107,
+      "detect": 0.027677168138325214,
+      "hidden": 0.16657640412449837
+    },
+    {
+      "step": 101,
+      "elapsed_s": 64.47661709785461,
+      "loss": 0.433330237865448,
+      "label": 0.14443234354257584,
+      "detect": 0.012719596037641168,
+      "hidden": 0.1412690468132496
+    },
+    {
+      "step": 151,
+      "elapsed_s": 81.66340661048889,
+      "loss": 0.374489426612854,
+      "label": 0.12132998369634151,
+      "detect": 0.0135470035020262,
+      "hidden": 0.12319297343492508
+    },
+    {
+      "step": 201,
+      "elapsed_s": 98.94163203239441,
+      "loss": 0.38375477492809296,
+      "label": 0.12871305830776691,
+      "detect": 0.015309339622035623,
+      "hidden": 0.12369352579116821
+    },
+    {
+      "step": 251,
+      "elapsed_s": 116.117196559906,
+      "loss": 0.4129961282014847,
+      "label": 0.16921869292855263,
+      "detect": 0.018067472614347935,
+      "hidden": 0.11737184971570969
+    },
+    {
+      "step": 301,
+      "elapsed_s": 133.41278839111328,
+      "loss": 0.3421344459056854,
+      "label": 0.12530853413045406,
+      "detect": 0.008865324780344963,
+      "hidden": 0.10619662329554558
+    },
+    {
+      "step": 351,
+      "elapsed_s": 150.6108753681183,
+      "loss": 0.4015732929110527,
+      "label": 0.15202416479587555,
+      "detect": 0.021410188637673855,
+      "hidden": 0.11942201666533947
+    },
+    {
+      "step": 401,
+      "elapsed_s": 167.74786019325256,
+      "loss": 0.3250713273882866,
+      "label": 0.11101343296468258,
+      "detect": 0.014453576179221272,
+      "hidden": 0.10341555811464787
+    },
+    {
+      "step": 451,
+      "elapsed_s": 184.91804909706116,
+      "loss": 0.30887674540281296,
+      "label": 0.1138718631118536,
+      "detect": 0.009780762484297156,
+      "hidden": 0.09505725093185902
+    },
+    {
+      "step": 501,
+      "elapsed_s": 201.9468400478363,
+      "loss": 0.2911076098680496,
+      "label": 0.093521011993289,
+      "detect": 0.011887337197549641,
+      "hidden": 0.09582146629691124
+    },
+    {
+      "step": 551,
+      "elapsed_s": 218.95755100250244,
+      "loss": 0.3279508873820305,
+      "label": 0.12233076430857182,
+      "detect": 0.012748322915285826,
+      "hidden": 0.09962298162281513
+    },
+    {
+      "step": 601,
+      "elapsed_s": 235.95491790771484,
+      "loss": 0.2837226167321205,
+      "label": 0.08802595920860767,
+      "detect": 0.006528681959025562,
+      "hidden": 0.09621615894138813
+    },
+    {
+      "step": 651,
+      "elapsed_s": 252.9777970314026,
+      "loss": 0.29664454981684685,
+      "label": 0.1072889119386673,
+      "detect": 0.010023394948802888,
+      "hidden": 0.09217197075486183
+    },
+    {
+      "step": 701,
+      "elapsed_s": 270.00525283813477,
+      "loss": 0.31047341972589493,
+      "label": 0.11355133540928364,
+      "detect": 0.01577813387848437,
+      "hidden": 0.09451650828123093
+    },
+    {
+      "step": 751,
+      "elapsed_s": 287.11339259147644,
+      "loss": 0.2924303859472275,
+      "label": 0.09383961372077465,
+      "detect": 0.008933465695008636,
+      "hidden": 0.09706202149391174
+    },
+    {
+      "step": 801,
+      "elapsed_s": 304.38399958610535,
+      "loss": 0.26952020823955536,
+      "label": 0.08447664976119995,
+      "detect": 0.01343391730915755,
+      "hidden": 0.08916330337524414
+    },
+    {
+      "step": 851,
+      "elapsed_s": 321.6072781085968,
+      "loss": 0.27088215947151184,
+      "label": 0.08796967100352049,
+      "detect": 0.008289208286441863,
+      "hidden": 0.08938394114375114
+    },
+    {
+      "step": 901,
+      "elapsed_s": 338.75015807151794,
+      "loss": 0.2826335057616234,
+      "label": 0.09538542665541172,
+      "detect": 0.010135660180822015,
+      "hidden": 0.0910901241004467
+    },
+    {
+      "step": 951,
+      "elapsed_s": 355.75725746154785,
+      "loss": 0.2881084829568863,
+      "label": 0.09700636379420757,
+      "detect": 0.014985204674303532,
+      "hidden": 0.0918047558516264
+    },
+    {
+      "step": 1000,
+      "elapsed_s": 372.5106551647186,
+      "loss": 0.3113468959927559,
+      "label": 0.1258483398705721,
+      "detect": 0.012814029469154775,
+      "hidden": 0.08954576589167118
+    }
+  ]
+}