use best model

last-checkpoint/added_tokens.json

@@ -1,3 +0,0 @@
-{
-  "[MASK]": 128000
-}

last-checkpoint/config.json

@@ -1,71 +0,0 @@
-{
-  "architectures": [
-    "DebertaV2ForTokenClassification"
-  ],
-  "attention_probs_dropout_prob": 0.14,
-  "bos_token_id": 1,
-  "dtype": "float32",
-  "eos_token_id": 2,
-  "hidden_act": "gelu",
-  "hidden_dropout_prob": 0.14,
-  "hidden_size": 768,
-  "id2label": {
-    "0": "O",
-    "1": "B-CHA",
-    "2": "I-CHA",
-    "3": "B-LOC",
-    "4": "I-LOC",
-    "5": "B-FAC",
-    "6": "I-FAC",
-    "7": "B-OBJ",
-    "8": "I-OBJ",
-    "9": "B-EVT",
-    "10": "I-EVT",
-    "11": "B-ORG",
-    "12": "I-ORG",
-    "13": "B-MISC",
-    "14": "I-MISC"
-  },
-  "initializer_range": 0.02,
-  "intermediate_size": 3072,
-  "label2id": {
-    "B-CHA": 1,
-    "B-EVT": 9,
-    "B-FAC": 5,
-    "B-LOC": 3,
-    "B-MISC": 13,
-    "B-OBJ": 7,
-    "B-ORG": 11,
-    "I-CHA": 2,
-    "I-EVT": 10,
-    "I-FAC": 6,
-    "I-LOC": 4,
-    "I-MISC": 14,
-    "I-OBJ": 8,
-    "I-ORG": 12,
-    "O": 0
-  },
-  "layer_norm_eps": 1e-07,
-  "legacy": true,
-  "max_position_embeddings": 512,
-  "max_relative_positions": -1,
-  "model_type": "deberta-v2",
-  "norm_rel_ebd": "layer_norm",
-  "num_attention_heads": 12,
-  "num_hidden_layers": 12,
-  "pad_token_id": 0,
-  "pooler_dropout": 0,
-  "pooler_hidden_act": "gelu",
-  "pooler_hidden_size": 768,
-  "pos_att_type": [
-    "p2c",
-    "c2p"
-  ],
-  "position_biased_input": false,
-  "position_buckets": 256,
-  "relative_attention": true,
-  "share_att_key": true,
-  "transformers_version": "4.56.0",
-  "type_vocab_size": 0,
-  "vocab_size": 128100
-}

last-checkpoint/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c1afa714bdd56bfbbb1efbf628f4c15f0b6ae266654356a88e0048e0cc7982eb
-size 735396724

last-checkpoint/optimizer.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a5d4e5065ff82276872b5c3d5a9b5a2c4dac0bfedf089f07966a780f6764b8dd
-size 1470915147

last-checkpoint/rng_state_0.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8eaef9a54b77e4410eb73091cca1813561231aab2270b6ed20afa38e56d957f0
-size 16325

last-checkpoint/rng_state_1.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:961966abaa0cb4be309c2dbb6bdbb184ae4138fcea22f730fdf31fa9583dd8d9
-size 16325

last-checkpoint/rng_state_2.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:116bac6566cf0d4caaea0074b84a22a40b2bd48fc5625a8d91c71f896a73b639
-size 16325

last-checkpoint/rng_state_3.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:897cd381e5d0e4f14794bfbc22ff307365b08a5ee1d7a2f7ca8224735e21d7e4
-size 16325

last-checkpoint/rng_state_4.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:aa521eed41a9fa3a1caf2b5b93bb2d46a8d473401226946c74853145f0fa0bbc
-size 16325

last-checkpoint/rng_state_5.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1985ee7e4ad215de1b263bc720f678bd0365e11ad07a2b9a440683734aa5e894
-size 16325

last-checkpoint/rng_state_6.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:884f53d156a859a4328175084fdf5db1ceaedae01554df966e9595888d0f4139
-size 16325

last-checkpoint/rng_state_7.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:cfb6ded3154a8a3c9fead3a4146b442c6599a2709d9507537d87653ee9014dc0
-size 16325

last-checkpoint/scheduler.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f59d63c52755d6dcb913edf584d7b233e8a521a9d79564a1c979c864c86896da
-size 1465

last-checkpoint/special_tokens_map.json

@@ -1,15 +0,0 @@
-{
-  "bos_token": "[CLS]",
-  "cls_token": "[CLS]",
-  "eos_token": "[SEP]",
-  "mask_token": "[MASK]",
-  "pad_token": "[PAD]",
-  "sep_token": "[SEP]",
-  "unk_token": {
-    "content": "[UNK]",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  }
-}

last-checkpoint/spm.model

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c679fbf93643d19aab7ee10c0b99e460bdbc02fedf34b92b05af343b4af586fd
-size 2464616

last-checkpoint/tokenizer_config.json

@@ -1,59 +0,0 @@
-{
-  "added_tokens_decoder": {
-    "0": {
-      "content": "[PAD]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "1": {
-      "content": "[CLS]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "2": {
-      "content": "[SEP]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "3": {
-      "content": "[UNK]",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128000": {
-      "content": "[MASK]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    }
-  },
-  "bos_token": "[CLS]",
-  "clean_up_tokenization_spaces": false,
-  "cls_token": "[CLS]",
-  "do_lower_case": false,
-  "eos_token": "[SEP]",
-  "extra_special_tokens": {},
-  "mask_token": "[MASK]",
-  "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "[PAD]",
-  "sep_token": "[SEP]",
-  "sp_model_kwargs": {},
-  "split_by_punct": false,
-  "tokenizer_class": "DebertaV2Tokenizer",
-  "unk_token": "[UNK]",
-  "vocab_type": "spm"
-}

last-checkpoint/trainer_state.json

@@ -1,897 +0,0 @@
-{
-  "best_global_step": 4395,
-  "best_metric": 0.7449346267936984,
-  "best_model_checkpoint": "model/checkpoint-4395",
-  "epoch": 1.0,
-  "eval_steps": 1465,
-  "global_step": 5860,
-  "is_hyper_param_search": false,
-  "is_local_process_zero": true,
-  "is_world_process_zero": true,
-  "log_history": [
-    {
-      "epoch": 0.008532423208191127,
-      "grad_norm": 1.5641576051712036,
-      "learning_rate": 6.960227272727272e-06,
-      "loss": 1.6624,
-      "step": 50
-    },
-    {
-      "epoch": 0.017064846416382253,
-      "grad_norm": 0.4061637222766876,
-      "learning_rate": 1.4062500000000001e-05,
-      "loss": 0.3169,
-      "step": 100
-    },
-    {
-      "epoch": 0.025597269624573378,
-      "grad_norm": 0.4451664984226227,
-      "learning_rate": 2.116477272727273e-05,
-      "loss": 0.1786,
-      "step": 150
-    },
-    {
-      "epoch": 0.034129692832764506,
-      "grad_norm": 0.3457334637641907,
-      "learning_rate": 2.499898999982817e-05,
-      "loss": 0.1447,
-      "step": 200
-    },
-    {
-      "epoch": 0.042662116040955635,
-      "grad_norm": 0.45870351791381836,
-      "learning_rate": 2.4989826780227188e-05,
-      "loss": 0.1324,
-      "step": 250
-    },
-    {
-      "epoch": 0.051194539249146756,
-      "grad_norm": 0.3399343490600586,
-      "learning_rate": 2.4971125493142457e-05,
-      "loss": 0.1289,
-      "step": 300
-    },
-    {
-      "epoch": 0.059726962457337884,
-      "grad_norm": 0.47232744097709656,
-      "learning_rate": 2.4942900420128184e-05,
-      "loss": 0.1235,
-      "step": 350
-    },
-    {
-      "epoch": 0.06825938566552901,
-      "grad_norm": 0.31067341566085815,
-      "learning_rate": 2.49051731157388e-05,
-      "loss": 0.1194,
-      "step": 400
-    },
-    {
-      "epoch": 0.07679180887372014,
-      "grad_norm": 0.3713296353816986,
-      "learning_rate": 2.485797239106845e-05,
-      "loss": 0.121,
-      "step": 450
-    },
-    {
-      "epoch": 0.08532423208191127,
-      "grad_norm": 0.3094009459018707,
-      "learning_rate": 2.4801334291748917e-05,
-      "loss": 0.1173,
-      "step": 500
-    },
-    {
-      "epoch": 0.09385665529010238,
-      "grad_norm": 0.283159464597702,
-      "learning_rate": 2.473530207042278e-05,
-      "loss": 0.1208,
-      "step": 550
-    },
-    {
-      "epoch": 0.10238907849829351,
-      "grad_norm": 0.30564266443252563,
-      "learning_rate": 2.4659926153712765e-05,
-      "loss": 0.1116,
-      "step": 600
-    },
-    {
-      "epoch": 0.11092150170648464,
-      "grad_norm": 0.2683025002479553,
-      "learning_rate": 2.4575264103712642e-05,
-      "loss": 0.1104,
-      "step": 650
-    },
-    {
-      "epoch": 0.11945392491467577,
-      "grad_norm": 0.3415144085884094,
-      "learning_rate": 2.4481380574028934e-05,
-      "loss": 0.1119,
-      "step": 700
-    },
-    {
-      "epoch": 0.12798634812286688,
-      "grad_norm": 0.250636488199234,
-      "learning_rate": 2.437834726040711e-05,
-      "loss": 0.108,
-      "step": 750
-    },
-    {
-      "epoch": 0.13651877133105803,
-      "grad_norm": 0.2625179886817932,
-      "learning_rate": 2.4266242845979902e-05,
-      "loss": 0.107,
-      "step": 800
-    },
-    {
-      "epoch": 0.14505119453924914,
-      "grad_norm": 0.31926393508911133,
-      "learning_rate": 2.4145152941179615e-05,
-      "loss": 0.1075,
-      "step": 850
-    },
-    {
-      "epoch": 0.15358361774744028,
-      "grad_norm": 0.2925693988800049,
-      "learning_rate": 2.401517001836026e-05,
-      "loss": 0.1087,
-      "step": 900
-    },
-    {
-      "epoch": 0.1621160409556314,
-      "grad_norm": 0.2825545370578766,
-      "learning_rate": 2.3876393341179486e-05,
-      "loss": 0.1044,
-      "step": 950
-    },
-    {
-      "epoch": 0.17064846416382254,
-      "grad_norm": 0.33026885986328125,
-      "learning_rate": 2.3728928888794205e-05,
-      "loss": 0.1052,
-      "step": 1000
-    },
-    {
-      "epoch": 0.17918088737201365,
-      "grad_norm": 0.32080453634262085,
-      "learning_rate": 2.3572889274927805e-05,
-      "loss": 0.1072,
-      "step": 1050
-    },
-    {
-      "epoch": 0.18771331058020477,
-      "grad_norm": 0.24436074495315552,
-      "learning_rate": 2.3408393661870808e-05,
-      "loss": 0.1031,
-      "step": 1100
-    },
-    {
-      "epoch": 0.1962457337883959,
-      "grad_norm": 0.22855418920516968,
-      "learning_rate": 2.3235567669480528e-05,
-      "loss": 0.104,
-      "step": 1150
-    },
-    {
-      "epoch": 0.20477815699658702,
-      "grad_norm": 0.2983805239200592,
-      "learning_rate": 2.3054543279249373e-05,
-      "loss": 0.1048,
-      "step": 1200
-    },
-    {
-      "epoch": 0.21331058020477817,
-      "grad_norm": 0.22958669066429138,
-      "learning_rate": 2.286545873351494e-05,
-      "loss": 0.1042,
-      "step": 1250
-    },
-    {
-      "epoch": 0.22184300341296928,
-      "grad_norm": 0.2709587514400482,
-      "learning_rate": 2.2668458429888906e-05,
-      "loss": 0.1028,
-      "step": 1300
-    },
-    {
-      "epoch": 0.23037542662116042,
-      "grad_norm": 0.3249916136264801,
-      "learning_rate": 2.2463692810985354e-05,
-      "loss": 0.104,
-      "step": 1350
-    },
-    {
-      "epoch": 0.23890784982935154,
-      "grad_norm": 0.33006545901298523,
-      "learning_rate": 2.225131824953274e-05,
-      "loss": 0.1011,
-      "step": 1400
-    },
-    {
-      "epoch": 0.24744027303754265,
-      "grad_norm": 0.27488061785697937,
-      "learning_rate": 2.203149692895718e-05,
-      "loss": 0.1015,
-      "step": 1450
-    },
-    {
-      "epoch": 0.25,
-      "eval_entity_f1": 0.7333329297742454,
-      "eval_entity_precision": 0.6570234291058094,
-      "eval_entity_recall": 0.876211987659762,
-      "eval_loss": 0.09791089594364166,
-      "eval_runtime": 794.2111,
-      "eval_samples_per_second": 1510.933,
-      "eval_steps_per_second": 23.608,
-      "step": 1465
-    },
-    {
-      "epoch": 0.25597269624573377,
-      "grad_norm": 0.21379445493221283,
-      "learning_rate": 2.180439671952838e-05,
-      "loss": 0.1029,
-      "step": 1500
-    },
-    {
-      "epoch": 0.2645051194539249,
-      "grad_norm": 0.24124079942703247,
-      "learning_rate": 2.157019105016262e-05,
-      "loss": 0.1021,
-      "step": 1550
-    },
-    {
-      "epoch": 0.27303754266211605,
-      "grad_norm": 0.2577572464942932,
-      "learning_rate": 2.1329058775980853e-05,
-      "loss": 0.1009,
-      "step": 1600
-    },
-    {
-      "epoch": 0.2815699658703072,
-      "grad_norm": 0.25605666637420654,
-      "learning_rate": 2.1081184041722966e-05,
-      "loss": 0.0994,
-      "step": 1650
-    },
-    {
-      "epoch": 0.2901023890784983,
-      "grad_norm": 0.20153528451919556,
-      "learning_rate": 2.0826756141122535e-05,
-      "loss": 0.101,
-      "step": 1700
-    },
-    {
-      "epoch": 0.2986348122866894,
-      "grad_norm": 0.24819990992546082,
-      "learning_rate": 2.0565969372349447e-05,
-      "loss": 0.1005,
-      "step": 1750
-    },
-    {
-      "epoch": 0.30716723549488056,
-      "grad_norm": 0.282105416059494,
-      "learning_rate": 2.0299022889630834e-05,
-      "loss": 0.0989,
-      "step": 1800
-    },
-    {
-      "epoch": 0.31569965870307165,
-      "grad_norm": 0.24825932085514069,
-      "learning_rate": 2.0026120551163576e-05,
-      "loss": 0.1015,
-      "step": 1850
-    },
-    {
-      "epoch": 0.3242320819112628,
-      "grad_norm": 0.19880250096321106,
-      "learning_rate": 1.9747470763434527e-05,
-      "loss": 0.0981,
-      "step": 1900
-    },
-    {
-      "epoch": 0.33276450511945393,
-      "grad_norm": 0.25610214471817017,
-      "learning_rate": 1.9463286322067397e-05,
-      "loss": 0.0993,
-      "step": 1950
-    },
-    {
-      "epoch": 0.3412969283276451,
-      "grad_norm": 0.28560763597488403,
-      "learning_rate": 1.9173784249317774e-05,
-      "loss": 0.097,
-      "step": 2000
-    },
-    {
-      "epoch": 0.34982935153583616,
-      "grad_norm": 0.36660489439964294,
-      "learning_rate": 1.8879185628340366e-05,
-      "loss": 0.0965,
-      "step": 2050
-    },
-    {
-      "epoch": 0.3583617747440273,
-      "grad_norm": 0.26472488045692444,
-      "learning_rate": 1.8579715434355174e-05,
-      "loss": 0.0988,
-      "step": 2100
-    },
-    {
-      "epoch": 0.36689419795221845,
-      "grad_norm": 0.25718653202056885,
-      "learning_rate": 1.8275602362841312e-05,
-      "loss": 0.0989,
-      "step": 2150
-    },
-    {
-      "epoch": 0.37542662116040953,
-      "grad_norm": 0.2548709213733673,
-      "learning_rate": 1.7967078654889858e-05,
-      "loss": 0.0974,
-      "step": 2200
-    },
-    {
-      "epoch": 0.3839590443686007,
-      "grad_norm": 0.235497385263443,
-      "learning_rate": 1.7654379919849003e-05,
-      "loss": 0.0943,
-      "step": 2250
-    },
-    {
-      "epoch": 0.3924914675767918,
-      "grad_norm": 0.31954771280288696,
-      "learning_rate": 1.7337744955397012e-05,
-      "loss": 0.0965,
-      "step": 2300
-    },
-    {
-      "epoch": 0.40102389078498296,
-      "grad_norm": 0.2830738425254822,
-      "learning_rate": 1.7017415565180293e-05,
-      "loss": 0.0964,
-      "step": 2350
-    },
-    {
-      "epoch": 0.40955631399317405,
-      "grad_norm": 0.23557038605213165,
-      "learning_rate": 1.669363637415601e-05,
-      "loss": 0.096,
-      "step": 2400
-    },
-    {
-      "epoch": 0.4180887372013652,
-      "grad_norm": 0.3127359449863434,
-      "learning_rate": 1.636665464178004e-05,
-      "loss": 0.0951,
-      "step": 2450
-    },
-    {
-      "epoch": 0.42662116040955633,
-      "grad_norm": 0.2751990556716919,
-      "learning_rate": 1.603672007318316e-05,
-      "loss": 0.0962,
-      "step": 2500
-    },
-    {
-      "epoch": 0.4351535836177474,
-      "grad_norm": 0.23224587738513947,
-      "learning_rate": 1.5704084628479443e-05,
-      "loss": 0.0975,
-      "step": 2550
-    },
-    {
-      "epoch": 0.44368600682593856,
-      "grad_norm": 0.2621734142303467,
-      "learning_rate": 1.536900233035271e-05,
-      "loss": 0.0947,
-      "step": 2600
-    },
-    {
-      "epoch": 0.4522184300341297,
-      "grad_norm": 0.2019677609205246,
-      "learning_rate": 1.5031729070067773e-05,
-      "loss": 0.0967,
-      "step": 2650
-    },
-    {
-      "epoch": 0.46075085324232085,
-      "grad_norm": 0.2038186639547348,
-      "learning_rate": 1.4692522412054772e-05,
-      "loss": 0.095,
-      "step": 2700
-    },
-    {
-      "epoch": 0.46928327645051193,
-      "grad_norm": 0.25815144181251526,
-      "learning_rate": 1.4351641397215703e-05,
-      "loss": 0.0935,
-      "step": 2750
-    },
-    {
-      "epoch": 0.4778156996587031,
-      "grad_norm": 0.2345559149980545,
-      "learning_rate": 1.4009346345103494e-05,
-      "loss": 0.0947,
-      "step": 2800
-    },
-    {
-      "epoch": 0.4863481228668942,
-      "grad_norm": 0.20084676146507263,
-      "learning_rate": 1.366589865512454e-05,
-      "loss": 0.0946,
-      "step": 2850
-    },
-    {
-      "epoch": 0.4948805460750853,
-      "grad_norm": 0.29759591817855835,
-      "learning_rate": 1.3321560606916652e-05,
-      "loss": 0.0951,
-      "step": 2900
-    },
-    {
-      "epoch": 0.5,
-      "eval_entity_f1": 0.7423178801283339,
-      "eval_entity_precision": 0.6650263167961575,
-      "eval_entity_recall": 0.8804226585921899,
-      "eval_loss": 0.08911468833684921,
-      "eval_runtime": 790.8047,
-      "eval_samples_per_second": 1517.442,
-      "eval_steps_per_second": 23.71,
-      "step": 2930
-    },
-    {
-      "epoch": 0.5034129692832765,
-      "grad_norm": 0.2829442322254181,
-      "learning_rate": 1.2976595160054744e-05,
-      "loss": 0.0956,
-      "step": 2950
-    },
-    {
-      "epoch": 0.5119453924914675,
-      "grad_norm": 0.3120681345462799,
-      "learning_rate": 1.263126575323735e-05,
-      "loss": 0.0922,
-      "step": 3000
-    },
-    {
-      "epoch": 0.5204778156996587,
-      "grad_norm": 0.2506762444972992,
-      "learning_rate": 1.228583610310716e-05,
-      "loss": 0.0943,
-      "step": 3050
-    },
-    {
-      "epoch": 0.5290102389078498,
-      "grad_norm": 0.19913755357265472,
-      "learning_rate": 1.1940570002859372e-05,
-      "loss": 0.0944,
-      "step": 3100
-    },
-    {
-      "epoch": 0.537542662116041,
-      "grad_norm": 0.2714909613132477,
-      "learning_rate": 1.1595731120791551e-05,
-      "loss": 0.0924,
-      "step": 3150
-    },
-    {
-      "epoch": 0.5460750853242321,
-      "grad_norm": 0.22429101169109344,
-      "learning_rate": 1.1251582798948877e-05,
-      "loss": 0.0924,
-      "step": 3200
-    },
-    {
-      "epoch": 0.5546075085324232,
-      "grad_norm": 0.18535013496875763,
-      "learning_rate": 1.0908387852018519e-05,
-      "loss": 0.0943,
-      "step": 3250
-    },
-    {
-      "epoch": 0.5631399317406144,
-      "grad_norm": 0.24078741669654846,
-      "learning_rate": 1.0566408366626783e-05,
-      "loss": 0.0955,
-      "step": 3300
-    },
-    {
-      "epoch": 0.5716723549488054,
-      "grad_norm": 0.21610242128372192,
-      "learning_rate": 1.0225905501192207e-05,
-      "loss": 0.0929,
-      "step": 3350
-    },
-    {
-      "epoch": 0.5802047781569966,
-      "grad_norm": 0.21978144347667694,
-      "learning_rate": 9.887139286487521e-06,
-      "loss": 0.0949,
-      "step": 3400
-    },
-    {
-      "epoch": 0.5887372013651877,
-      "grad_norm": 0.25699079036712646,
-      "learning_rate": 9.550368427062745e-06,
-      "loss": 0.0914,
-      "step": 3450
-    },
-    {
-      "epoch": 0.5972696245733788,
-      "grad_norm": 0.21956154704093933,
-      "learning_rate": 9.215850103681096e-06,
-      "loss": 0.0946,
-      "step": 3500
-    },
-    {
-      "epoch": 0.60580204778157,
-      "grad_norm": 0.23602405190467834,
-      "learning_rate": 8.883839776918538e-06,
-      "loss": 0.0935,
-      "step": 3550
-    },
-    {
-      "epoch": 0.6143344709897611,
-      "grad_norm": 0.24697446823120117,
-      "learning_rate": 8.554590992077e-06,
-      "loss": 0.092,
-      "step": 3600
-    },
-    {
-      "epoch": 0.6228668941979523,
-      "grad_norm": 0.24195240437984467,
-      "learning_rate": 8.228355185560196e-06,
-      "loss": 0.0927,
-      "step": 3650
-    },
-    {
-      "epoch": 0.6313993174061433,
-      "grad_norm": 0.25726425647735596,
-      "learning_rate": 7.905381492859997e-06,
-      "loss": 0.0942,
-      "step": 3700
-    },
-    {
-      "epoch": 0.6399317406143344,
-      "grad_norm": 0.230339914560318,
-      "learning_rate": 7.5859165582998655e-06,
-      "loss": 0.0947,
-      "step": 3750
-    },
-    {
-      "epoch": 0.6484641638225256,
-      "grad_norm": 0.34315890073776245,
-      "learning_rate": 7.270204346680777e-06,
-      "loss": 0.0924,
-      "step": 3800
-    },
-    {
-      "epoch": 0.6569965870307167,
-      "grad_norm": 0.21445219218730927,
-      "learning_rate": 6.958485956973332e-06,
-      "loss": 0.0965,
-      "step": 3850
-    },
-    {
-      "epoch": 0.6655290102389079,
-      "grad_norm": 0.2074640691280365,
-      "learning_rate": 6.650999438198499e-06,
-      "loss": 0.093,
-      "step": 3900
-    },
-    {
-      "epoch": 0.674061433447099,
-      "grad_norm": 0.1953040212392807,
-      "learning_rate": 6.347979607637408e-06,
-      "loss": 0.0923,
-      "step": 3950
-    },
-    {
-      "epoch": 0.6825938566552902,
-      "grad_norm": 0.19871976971626282,
-      "learning_rate": 6.049657871509198e-06,
-      "loss": 0.0925,
-      "step": 4000
-    },
-    {
-      "epoch": 0.6911262798634812,
-      "grad_norm": 0.2851618230342865,
-      "learning_rate": 5.756262048253709e-06,
-      "loss": 0.0936,
-      "step": 4050
-    },
-    {
-      "epoch": 0.6996587030716723,
-      "grad_norm": 0.25152096152305603,
-      "learning_rate": 5.468016194554112e-06,
-      "loss": 0.0904,
-      "step": 4100
-    },
-    {
-      "epoch": 0.7081911262798635,
-      "grad_norm": 0.29341500997543335,
-      "learning_rate": 5.185140434232203e-06,
-      "loss": 0.0915,
-      "step": 4150
-    },
-    {
-      "epoch": 0.7167235494880546,
-      "grad_norm": 0.19771753251552582,
-      "learning_rate": 4.907850790147146e-06,
-      "loss": 0.0921,
-      "step": 4200
-    },
-    {
-      "epoch": 0.7252559726962458,
-      "grad_norm": 0.2772028148174286,
-      "learning_rate": 4.636359019225947e-06,
-      "loss": 0.0926,
-      "step": 4250
-    },
-    {
-      "epoch": 0.7337883959044369,
-      "grad_norm": 0.27628397941589355,
-      "learning_rate": 4.370872450751694e-06,
-      "loss": 0.0896,
-      "step": 4300
-    },
-    {
-      "epoch": 0.742320819112628,
-      "grad_norm": 0.23001554608345032,
-      "learning_rate": 4.111593828033067e-06,
-      "loss": 0.0877,
-      "step": 4350
-    },
-    {
-      "epoch": 0.75,
-      "eval_entity_f1": 0.7449346267936984,
-      "eval_entity_precision": 0.668926309530101,
-      "eval_entity_recall": 0.878515580366616,
-      "eval_loss": 0.08664915710687637,
-      "eval_runtime": 792.2218,
-      "eval_samples_per_second": 1514.727,
-      "eval_steps_per_second": 23.668,
-      "step": 4395
-    },
-    {
-      "epoch": 0.7508532423208191,
-      "grad_norm": 0.23986265063285828,
-      "learning_rate": 3.858721153575945e-06,
-      "loss": 0.0898,
-      "step": 4400
-    },
-    {
-      "epoch": 0.7593856655290102,
-      "grad_norm": 0.17509053647518158,
-      "learning_rate": 3.6124475378754783e-06,
-      "loss": 0.0927,
-      "step": 4450
-    },
-    {
-      "epoch": 0.7679180887372014,
-      "grad_norm": 0.23464491963386536,
-      "learning_rate": 3.3729610519439585e-06,
-      "loss": 0.0911,
-      "step": 4500
-    },
-    {
-      "epoch": 0.7764505119453925,
-      "grad_norm": 0.2226947546005249,
-      "learning_rate": 3.140444583687245e-06,
-      "loss": 0.0934,
-      "step": 4550
-    },
-    {
-      "epoch": 0.7849829351535836,
-      "grad_norm": 0.21338465809822083,
-      "learning_rate": 2.915075698239285e-06,
-      "loss": 0.0901,
-      "step": 4600
-    },
-    {
-      "epoch": 0.7935153583617748,
-      "grad_norm": 0.1981608271598816,
-      "learning_rate": 2.6970265023615297e-06,
-      "loss": 0.0904,
-      "step": 4650
-    },
-    {
-      "epoch": 0.8020477815699659,
-      "grad_norm": 0.19228731095790863,
-      "learning_rate": 2.4864635130106645e-06,
-      "loss": 0.0893,
-      "step": 4700
-    },
-    {
-      "epoch": 0.810580204778157,
-      "grad_norm": 0.309893935918808,
-      "learning_rate": 2.283547530175148e-06,
-      "loss": 0.0896,
-      "step": 4750
-    },
-    {
-      "epoch": 0.8191126279863481,
-      "grad_norm": 0.20562194287776947,
-      "learning_rate": 2.0884335140775522e-06,
-      "loss": 0.0922,
-      "step": 4800
-    },
-    {
-      "epoch": 0.8276450511945392,
-      "grad_norm": 0.2413642853498459,
-      "learning_rate": 1.901270466836584e-06,
-      "loss": 0.0886,
-      "step": 4850
-    },
-    {
-      "epoch": 0.8361774744027304,
-      "grad_norm": 0.20934799313545227,
-      "learning_rate": 1.7222013186790995e-06,
-      "loss": 0.0912,
-      "step": 4900
-    },
-    {
-      "epoch": 0.8447098976109215,
-      "grad_norm": 0.20717120170593262,
-      "learning_rate": 1.5513628187890136e-06,
-      "loss": 0.0929,
-      "step": 4950
-    },
-    {
-      "epoch": 0.8532423208191127,
-      "grad_norm": 0.24309539794921875,
-      "learning_rate": 1.3888854308764631e-06,
-      "loss": 0.0907,
-      "step": 5000
-    },
-    {
-      "epoch": 0.8617747440273038,
-      "grad_norm": 0.23167067766189575,
-      "learning_rate": 1.2348932335469992e-06,
-      "loss": 0.092,
-      "step": 5050
-    },
-    {
-      "epoch": 0.8703071672354948,
-      "grad_norm": 0.24426911771297455,
-      "learning_rate": 1.0895038255468643e-06,
-      "loss": 0.0913,
-      "step": 5100
-    },
-    {
-      "epoch": 0.878839590443686,
-      "grad_norm": 0.21875974535942078,
-      "learning_rate": 9.528282359567153e-07,
-      "loss": 0.0919,
-      "step": 5150
-    },
-    {
-      "epoch": 0.8873720136518771,
-      "grad_norm": 0.15874198079109192,
-      "learning_rate": 8.249708394023767e-07,
-      "loss": 0.0911,
-      "step": 5200
-    },
-    {
-      "epoch": 0.8959044368600683,
-      "grad_norm": 0.26125800609588623,
-      "learning_rate": 7.060292763474142e-07,
-      "loss": 0.0908,
-      "step": 5250
-    },
-    {
-      "epoch": 0.9044368600682594,
-      "grad_norm": 0.25548022985458374,
-      "learning_rate": 5.960943785283293e-07,
-      "loss": 0.0907,
-      "step": 5300
-    },
-    {
-      "epoch": 0.9129692832764505,
-      "grad_norm": 0.18535113334655762,
-      "learning_rate": 4.9525009958937e-07,
-      "loss": 0.0883,
-      "step": 5350
-    },
-    {
-      "epoch": 0.9215017064846417,
-      "grad_norm": 0.29517892003059387,
-      "learning_rate": 4.03573450969906e-07,
-      "loss": 0.0902,
-      "step": 5400
-    },
-    {
-      "epoch": 0.9300341296928327,
-      "grad_norm": 0.21196790039539337,
-      "learning_rate": 3.211344430933516e-07,
-      "loss": 0.0899,
-      "step": 5450
-    },
-    {
-      "epoch": 0.9385665529010239,
-      "grad_norm": 0.21812868118286133,
-      "learning_rate": 2.479960319025129e-07,
-      "loss": 0.0903,
-      "step": 5500
-    },
-    {
-      "epoch": 0.947098976109215,
-      "grad_norm": 0.21973644196987152,
-      "learning_rate": 1.8421407078221404e-07,
-      "loss": 0.0906,
-      "step": 5550
-    },
-    {
-      "epoch": 0.9556313993174061,
-      "grad_norm": 0.24090933799743652,
-      "learning_rate": 1.2983726790592592e-07,
-      "loss": 0.0896,
-      "step": 5600
-    },
-    {
-      "epoch": 0.9641638225255973,
-      "grad_norm": 0.21915237605571747,
-      "learning_rate": 8.490714903894025e-08,
-      "loss": 0.091,
-      "step": 5650
-    },
-    {
-      "epoch": 0.9726962457337884,
-      "grad_norm": 0.1731707900762558,
-      "learning_rate": 4.94580258265126e-08,
-      "loss": 0.0893,
-      "step": 5700
-    },
-    {
-      "epoch": 0.9812286689419796,
-      "grad_norm": 0.2428259402513504,
-      "learning_rate": 2.3516969591198813e-08,
-      "loss": 0.0926,
-      "step": 5750
-    },
-    {
-      "epoch": 0.9897610921501706,
-      "grad_norm": 0.23712676763534546,
-      "learning_rate": 7.103790659380993e-09,
-      "loss": 0.0948,
-      "step": 5800
-    },
-    {
-      "epoch": 0.9982935153583617,
-      "grad_norm": 0.24450626969337463,
-      "learning_rate": 2.3102323277596205e-10,
-      "loss": 0.0888,
-      "step": 5850
-    },
-    {
-      "epoch": 1.0,
-      "eval_entity_f1": 0.744815399729913,
-      "eval_entity_precision": 0.6683184682739546,
-      "eval_entity_recall": 0.8811592369510689,
-      "eval_loss": 0.08635299652814865,
-      "eval_runtime": 790.7469,
-      "eval_samples_per_second": 1517.552,
-      "eval_steps_per_second": 23.712,
-      "step": 5860
-    }
-  ],
-  "logging_steps": 50,
-  "max_steps": 5860,
-  "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
-  "save_steps": 1465,
-  "stateful_callbacks": {
-    "TrainerControl": {
-      "args": {
-        "should_epoch_stop": false,
-        "should_evaluate": false,
-        "should_log": false,
-        "should_save": true,
-        "should_training_stop": true
-      },
-      "attributes": {}
-    }
-  },
-  "total_flos": 7.412859979369021e+17,
-  "train_batch_size": 256,
-  "trial_name": null,
-  "trial_params": null
-}

last-checkpoint/training_args.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:5cd0cadb15b38d5d62eb7ba1c8d8cf6d8ff7a7453651ed77c16de5239c1b3221
-size 5905

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c1afa714bdd56bfbbb1efbf628f4c15f0b6ae266654356a88e0048e0cc7982eb
+oid sha256:ade3a987bda6b550c43dc13485050e50f44bad5a2c59f156a6c24f534e2b131d
 size 735396724

spm.model

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c679fbf93643d19aab7ee10c0b99e460bdbc02fedf34b92b05af343b4af586fd
-size 2464616

train.ipynb

@@ -0,0 +1,466 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b00e4cd9",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "!hf download SaladTechnologies/fiction-ner-750m --quiet --repo-type=dataset --local-dir .\n",
+    "!unzip -q data.zip"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b1be4895",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import string\n",
+    "import random\n",
+    "\n",
+    "def get_random_string(length=8):\n",
+    "    \"\"\"Generate a random string of fixed length.\"\"\"\n",
+    "    letters = string.ascii_letters\n",
+    "    return ''.join(random.choice(letters) for i in range(length))\n",
+    "\n",
+    "run_name = f\"ner-{get_random_string(8)}\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f21e8995",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from accelerate import notebook_launcher\n",
+    "import os\n",
+    "\n",
+    "\n",
+    "cuda_visible_devices = os.getenv(\"CUDA_VISIBLE_DEVICES\", \"0\")\n",
+    "num_devices = len(cuda_visible_devices.split(\",\"))\n",
+    "\n",
+    "\n",
+    "def train_fn():\n",
+    "    global num_processes\n",
+    "    from datasets import Dataset, concatenate_datasets\n",
+    "    import pandas as pd\n",
+    "    from pathlib import Path\n",
+    "    import random\n",
+    "    from transformers import AutoTokenizer\n",
+    "    import torch\n",
+    "    import numpy as np\n",
+    "    from transformers import AutoModelForTokenClassification\n",
+    "    from transformers.data.data_collator import DataCollatorForTokenClassification\n",
+    "    from transformers.training_args import TrainingArguments\n",
+    "    from transformers.trainer import Trainer\n",
+    "    from transformers.trainer_callback import TrainerCallback\n",
+    "    import numpy as np\n",
+    "    from sklearn.metrics import precision_recall_fscore_support\n",
+    "    import os\n",
+    "    import wandb\n",
+    "\n",
+    "    num_epochs = int(os.getenv(\"NUM_EPOCHS\", 3))\n",
+    "    output_dir = os.getenv(\"OUTPUT_DIR\", \"./model\")\n",
+    "    seed = int(os.getenv(\"RANDOM_SEED\", 42))\n",
+    "    model_id = os.getenv(\"MODEL_ID\")\n",
+    "    hub_token = os.getenv(\"HF_TOKEN\")\n",
+    "    save_steps = float(os.getenv(\"SAVE_STEPS\", 100))\n",
+    "    if save_steps.is_integer():\n",
+    "        save_steps = int(save_steps)\n",
+    "    train_size = float(os.getenv(\"TRAIN_SIZE\", 4_000_000))\n",
+    "    test_size = float(os.getenv(\"TEST_SIZE\", 400_000))\n",
+    "    if train_size.is_integer():\n",
+    "        train_size = int(train_size)\n",
+    "    if test_size.is_integer():\n",
+    "        test_size = int(test_size)\n",
+    "    hidden_dropout_prob = float(os.getenv(\"HIDDEN_DROPOUT_PROB\", 0.14))\n",
+    "    attention_probs_dropout_prob = float(os.getenv(\"ATTENTION_PROBS_DROPOUT_PROB\", 0.14))\n",
+    "    frequency_exponent = float(os.getenv(\"FREQUENCY_EXPONENT\", 0.35))\n",
+    "    gamma = float(os.getenv(\"GAMMA\", 2.1))\n",
+    "    learning_rate = float(os.getenv(\"LEARNING_RATE\", 2.5e-5))\n",
+    "    lr_scheduler_type = os.getenv(\"LR_SCHEDULER_TYPE\", \"cosine\")\n",
+    "    weight_decay = float(os.getenv(\"WEIGHT_DECAY\", 0.007))\n",
+    "    warmup_ratio = float(os.getenv(\"WARMUP_RATIO\", 0.03))\n",
+    "    per_device_train_batch_size = int(os.getenv(\"PER_DEVICE_TRAIN_BATCH_SIZE\", 256))\n",
+    "    max_saved_checkpoints = int(os.getenv(\"MAX_SAVED_CHECKPOINTS\", 8))\n",
+    "    patience = max_saved_checkpoints - 1\n",
+    "    n_eval_samples = int(os.getenv(\"N_EVAL_SAMPLES\", 5))  # Number of samples to show\n",
+    "    log_predictions_to_wandb = os.getenv(\"LOG_PREDICTIONS_TO_WANDB\", \"true\").lower() == \"true\"\n",
+    "    log_predictions_to_console = os.getenv(\"LOG_PREDICTIONS_TO_CONSOLE\", \"false\").lower() == \"true\"\n",
+    "\n",
+    "    num_processes = torch.cuda.device_count()\n",
+    "    \n",
+    "    tokenizer = AutoTokenizer.from_pretrained(\"microsoft/deberta-v3-base\")\n",
+    "    \n",
+    "    data_dir = Path(\"data\")\n",
+    "    output = Path(output_dir)\n",
+    "    random.seed(seed)\n",
+    "    torch.manual_seed(seed)\n",
+    "    np.random.seed(seed)\n",
+    "\n",
+    "    \n",
+    "    label_list = [\n",
+    "        \"O\",\n",
+    "        \"B-CHA\",\n",
+    "        \"I-CHA\",\n",
+    "        \"B-LOC\",\n",
+    "        \"I-LOC\",\n",
+    "        \"B-FAC\",\n",
+    "        \"I-FAC\",\n",
+    "        \"B-OBJ\",\n",
+    "        \"I-OBJ\",\n",
+    "        \"B-EVT\",\n",
+    "        \"I-EVT\",\n",
+    "        \"B-ORG\",\n",
+    "        \"I-ORG\",\n",
+    "        \"B-MISC\",\n",
+    "        \"I-MISC\"\n",
+    "    ]\n",
+    "    label_to_id = {label: i for i, label in enumerate(label_list)}\n",
+    "    id_to_label = {i: label for i, label in enumerate(label_list)}\n",
+    "\n",
+    "    datasets = []\n",
+    "    for parquet_file in sorted(data_dir.glob(\"*.parquet\")):\n",
+    "        ds = Dataset.from_parquet(str(parquet_file))\n",
+    "        datasets.append(ds)\n",
+    "\n",
+    "    full_ds = concatenate_datasets(datasets)\n",
+    "    splits = full_ds.train_test_split(train_size=train_size, test_size=test_size, seed=seed)\n",
+    "\n",
+    "    train_ds = splits['train']\n",
+    "    eval_ds = splits['test']\n",
+    "\n",
+    "    stats_file = \"label_counts.csv\"\n",
+    "    stats_df = pd.read_csv(stats_file)\n",
+    "    stats_df.head()\n",
+    "\n",
+    "    total_count = stats_df[\"total\"].sum()\n",
+    "    label_frequencies = {\n",
+    "        label: stats_df[label].sum() / total_count for label in label_list\n",
+    "    }\n",
+    "    \n",
+    "    label_weights = {}\n",
+    "    for label, freq in label_frequencies.items():\n",
+    "        label_weights[label] = 1.0 / freq ** frequency_exponent\n",
+    "\n",
+    "    weight_tensor = torch.tensor([label_weights[label] for label in label_list], dtype=torch.float32)\n",
+    "\n",
+    "    model = AutoModelForTokenClassification.from_pretrained(\n",
+    "        \"microsoft/deberta-v3-base\",\n",
+    "        num_labels=len(label_list),\n",
+    "        id2label=id_to_label,\n",
+    "        label2id=label_to_id,\n",
+    "        ignore_mismatched_sizes=True,\n",
+    "        hidden_dropout_prob=hidden_dropout_prob,\n",
+    "        attention_probs_dropout_prob=attention_probs_dropout_prob\n",
+    "    )\n",
+    "    \n",
+    "    data_collator = DataCollatorForTokenClassification(\n",
+    "        tokenizer=tokenizer,\n",
+    "        padding=True\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "    def create_compute_metrics_fn(eval_dataset):\n",
+    "        \"\"\"\n",
+    "        Factory function that creates a compute_metrics function with access to eval_dataset.\n",
+    "        \"\"\"\n",
+    "        def compute_metrics(eval_pred):\n",
+    "            predictions, labels = eval_pred\n",
+    "            predictions_raw = predictions  # Keep raw predictions for logging\n",
+    "            predictions = np.argmax(predictions, axis=2)\n",
+    "            \n",
+    "            # Remove ignored indices\n",
+    "            true_predictions = [\n",
+    "                [id_to_label[p] for (p, l) in zip(pred, label) if l != -100]\n",
+    "                for pred, label in zip(predictions, labels)\n",
+    "            ]\n",
+    "            true_labels = [\n",
+    "                [id_to_label[l] for (p, l) in zip(pred, label) if l != -100]\n",
+    "                for pred, label in zip(predictions, labels)\n",
+    "            ]\n",
+    "            \n",
+    "            # Flatten\n",
+    "            all_predictions = [item for sublist in true_predictions for item in sublist]\n",
+    "            all_labels = [item for sublist in true_labels for item in sublist]\n",
+    "            \n",
+    "            # Calculate metrics excluding 'O' class\n",
+    "            entity_labels = [l for l in label_list if l != 'O']\n",
+    "            \n",
+    "            precision, recall, f1, support = precision_recall_fscore_support(\n",
+    "                all_labels,\n",
+    "                all_predictions,\n",
+    "                labels=entity_labels,\n",
+    "                average='weighted',\n",
+    "                zero_division=0\n",
+    "            )\n",
+    "\n",
+    "            return {\n",
+    "                'entity_precision': precision,\n",
+    "                'entity_recall': recall,\n",
+    "                'entity_f1': f1,\n",
+    "            }\n",
+    "        \n",
+    "        return compute_metrics\n",
+    "\n",
+    "    # Create the compute_metrics function with access to eval_ds\n",
+    "    compute_metrics = create_compute_metrics_fn(eval_ds)\n",
+    "\n",
+    "    class FocalLoss(torch.nn.Module):\n",
+    "        def __init__(self, alpha=None, gamma=2.0, reduction='mean', ignore_index=-100):\n",
+    "            \"\"\"\n",
+    "            alpha: class weights tensor\n",
+    "            gamma: focusing parameter (higher = more focus on hard examples)\n",
+    "            ignore_index: label to ignore (for padding tokens)\n",
+    "            \"\"\"\n",
+    "            super().__init__()\n",
+    "            self.alpha = alpha\n",
+    "            self.gamma = gamma\n",
+    "            self.reduction = reduction\n",
+    "            self.ignore_index = ignore_index\n",
+    "            \n",
+    "        def forward(self, logits, labels):\n",
+    "            # logits shape: (batch_size, seq_len, num_classes)\n",
+    "            # labels shape: (batch_size, seq_len)\n",
+    "            \n",
+    "            # Reshape for loss calculation\n",
+    "            logits_flat = logits.view(-1, logits.size(-1))  # (batch*seq_len, num_classes)\n",
+    "            labels_flat = labels.view(-1)  # (batch*seq_len)\n",
+    "            \n",
+    "            # Calculate cross entropy (without reduction)\n",
+    "            ce_loss = torch.nn.functional.cross_entropy(\n",
+    "                logits_flat, \n",
+    "                labels_flat, \n",
+    "                reduction='none',\n",
+    "                ignore_index=self.ignore_index\n",
+    "            )\n",
+    "            \n",
+    "            # Get the probabilities for the correct class\n",
+    "            p = torch.exp(-ce_loss)\n",
+    "            \n",
+    "            # Calculate focal term: (1 - p)^gamma\n",
+    "            focal_term = (1 - p) ** self.gamma\n",
+    "            \n",
+    "            # Apply focal term to loss\n",
+    "            focal_loss = focal_term * ce_loss\n",
+    "            \n",
+    "            # Apply class weights if provided\n",
+    "            if self.alpha is not None:\n",
+    "                # Create a mask for valid (non-ignored) tokens\n",
+    "                valid_mask = labels_flat != self.ignore_index\n",
+    "                \n",
+    "                # Gather the weights for each sample's true class\n",
+    "                # Only for valid labels to avoid index errors\n",
+    "                valid_labels = labels_flat.clone()\n",
+    "                valid_labels[~valid_mask] = 0  # Set ignored labels to 0 to avoid index errors\n",
+    "                \n",
+    "                alpha_t = self.alpha.gather(0, valid_labels)\n",
+    "                # Apply mask to weights\n",
+    "                alpha_t = alpha_t * valid_mask.float()\n",
+    "                \n",
+    "                focal_loss = alpha_t * focal_loss\n",
+    "            \n",
+    "            # Apply reduction\n",
+    "            if self.reduction == 'mean':\n",
+    "                # Only average over non-ignored tokens\n",
+    "                valid_tokens = (labels_flat != self.ignore_index).sum()\n",
+    "                return focal_loss.sum() / valid_tokens.clamp(min=1)\n",
+    "            elif self.reduction == 'sum':\n",
+    "                return focal_loss.sum()\n",
+    "            else:\n",
+    "                return focal_loss\n",
+    "    \n",
+    "    class FocalLossTrainer(Trainer):\n",
+    "        def __init__(self, *args, class_weights=None, gamma=2.0, **kwargs):\n",
+    "            super().__init__(*args, **kwargs)\n",
+    "            self.class_weights = class_weights\n",
+    "            self.gamma = gamma\n",
+    "            \n",
+    "        def compute_loss(self, model, inputs, return_outputs=False, num_items_in_batch=None):\n",
+    "            \"\"\"\n",
+    "            Override compute_loss to use focal loss.\n",
+    "            num_items_in_batch parameter added for compatibility with newer transformers versions.\n",
+    "            \"\"\"\n",
+    "            labels = inputs.get(\"labels\")\n",
+    "            outputs = model(**inputs)\n",
+    "            logits = outputs.get(\"logits\")\n",
+    "            \n",
+    "            # Move weights to the same device as logits\n",
+    "            if self.class_weights is not None:\n",
+    "                weights = self.class_weights.to(logits.device)\n",
+    "            else:\n",
+    "                weights = None\n",
+    "            \n",
+    "            # Initialize focal loss\n",
+    "            loss_fct = FocalLoss(\n",
+    "                alpha=weights,\n",
+    "                gamma=self.gamma,\n",
+    "                ignore_index=-100\n",
+    "            )\n",
+    "            \n",
+    "            # Calculate loss\n",
+    "            loss = loss_fct(logits, labels)\n",
+    "            \n",
+    "            return (loss, outputs) if return_outputs else loss\n",
+    "\n",
+    "    \n",
+    "\n",
+    "    training_args = TrainingArguments(\n",
+    "        output_dir=str(output),\n",
+    "        learning_rate=learning_rate,\n",
+    "        lr_scheduler_type=lr_scheduler_type,\n",
+    "        per_device_train_batch_size=per_device_train_batch_size,\n",
+    "        weight_decay=weight_decay,\n",
+    "        warmup_ratio=warmup_ratio,\n",
+    "        gradient_accumulation_steps=1,\n",
+    "        logging_steps=50,\n",
+    "        num_train_epochs=num_epochs,\n",
+    "        save_strategy=\"steps\",\n",
+    "        save_steps=save_steps,\n",
+    "        save_total_limit=3,\n",
+    "        eval_strategy=\"steps\",\n",
+    "        eval_steps=save_steps,\n",
+    "        load_best_model_at_end=True,\n",
+    "        metric_for_best_model=\"eval_entity_f1\",\n",
+    "        greater_is_better=True,\n",
+    "        bf16=True,\n",
+    "        tf32=True,\n",
+    "        report_to='wandb',\n",
+    "        run_name=run_name,\n",
+    "        push_to_hub=True,\n",
+    "        hub_strategy=\"checkpoint\",\n",
+    "        hub_token=hub_token,\n",
+    "        dataloader_persistent_workers=True,\n",
+    "        dataloader_num_workers=2,\n",
+    "        dataloader_pin_memory=True,\n",
+    "        ddp_find_unused_parameters=False,\n",
+    "        gradient_checkpointing=False,\n",
+    "        hub_model_id=model_id,\n",
+    "        hub_private_repo=True\n",
+    "    )\n",
+    "\n",
+    "    class CustomEarlyStoppingCallback(TrainerCallback):\n",
+    "        def __init__(self, patience=2, threshold=0.001):\n",
+    "            self.patience = patience\n",
+    "            self.threshold = threshold\n",
+    "            self.best_metric = None\n",
+    "            self.wait = 0\n",
+    "        \n",
+    "        def on_evaluate(self, args, state, control, metrics=None, **kwargs):\n",
+    "            if metrics is None or \"eval_entity_f1\" not in metrics:\n",
+    "                return control\n",
+    "            metric_value = metrics.get(\"eval_entity_f1\")\n",
+    "            \n",
+    "            if self.best_metric is None:\n",
+    "                self.best_metric = metric_value\n",
+    "            elif metric_value > self.best_metric + self.threshold:\n",
+    "                self.best_metric = metric_value\n",
+    "                self.wait = 0\n",
+    "            else:\n",
+    "                self.wait += 1\n",
+    "                if self.wait >= self.patience:\n",
+    "                    control.should_training_stop = True\n",
+    "                    print(f\"Early stopping triggered. Best F1: {self.best_metric:.4f}\")\n",
+    "            \n",
+    "            return control\n",
+    "            \n",
+    "\n",
+    "    trainer = FocalLossTrainer(\n",
+    "        model=model,\n",
+    "        args=training_args,\n",
+    "        train_dataset=train_ds,\n",
+    "        eval_dataset=eval_ds,\n",
+    "        processing_class=tokenizer,\n",
+    "        data_collator=data_collator,\n",
+    "        compute_metrics=compute_metrics,\n",
+    "        class_weights=weight_tensor,\n",
+    "        gamma=gamma,\n",
+    "        callbacks=[CustomEarlyStoppingCallback(patience=patience, threshold=0.0001)]\n",
+    "    )\n",
+    "    \n",
+    "    if wandb.run is not None:\n",
+    "        # Add custom config values\n",
+    "        wandb.config.update({\n",
+    "            # Data configuration\n",
+    "            \"train_samples\": len(train_ds),\n",
+    "            \"eval_samples\": len(eval_ds),\n",
+    "            \"train_size_requested\": train_size,\n",
+    "            \"test_size_requested\": test_size,\n",
+    "            \"actual_train_size\": len(train_ds),\n",
+    "            \"actual_eval_size\": len(eval_ds),\n",
+    "\n",
+    "            # Model architecture details\n",
+    "            \"model_architecture\": \"deberta-v3-base\",\n",
+    "            \"num_labels\": len(label_list),\n",
+    "            \"label_list\": label_list,\n",
+    "\n",
+    "            # Loss function configuration\n",
+    "            \"loss_function\": \"focal_loss\",\n",
+    "            \"focal_gamma\": gamma,\n",
+    "            \"focal_alpha\": \"weighted\",\n",
+    "            \"frequency_exponent\": frequency_exponent,\n",
+    "\n",
+    "            # Dropout configuration\n",
+    "            \"hidden_dropout_prob\": hidden_dropout_prob,\n",
+    "            \"attention_probs_dropout_prob\": attention_probs_dropout_prob,\n",
+    "\n",
+    "            # Training configuration not in TrainingArguments\n",
+    "            \"max_saved_checkpoints\": max_saved_checkpoints,\n",
+    "            \"early_stopping_patience\": patience,\n",
+    "            \"early_stopping_threshold\": 0.001,\n",
+    "\n",
+    "            # Environment info\n",
+    "            \"cuda_devices\": cuda_visible_devices,\n",
+    "            \"num_gpus\": num_devices,\n",
+    "\n",
+    "            # Data processing\n",
+    "            \"tokenizer\": \"microsoft/deberta-v3-base\"\n",
+    "\n",
+    "            # Experiment metadata\n",
+    "            \"experiment_type\": \"ner_fiction\",\n",
+    "            \"data_source\": \"gutenberg_ao3_mixed\",\n",
+    "            \"random_seed\": seed,\n",
+    "\n",
+    "            # Logging configuration\n",
+    "            \"n_eval_samples\": n_eval_samples,\n",
+    "            \"log_predictions_to_wandb\": log_predictions_to_wandb,\n",
+    "        })\n",
+    "\n",
+    "    has_checkpoints = bool([f for f in os.scandir(output_dir) if f.is_dir() and \"checkpoint\" in f.name])\n",
+    "    if has_checkpoints:\n",
+    "        trainer.train(resume_from_checkpoint=True)\n",
+    "    else:\n",
+    "        trainer.train()\n",
+    "\n",
+    "notebook_launcher(train_fn, num_processes=num_devices)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

training_args.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:5cd0cadb15b38d5d62eb7ba1c8d8cf6d8ff7a7453651ed77c16de5239c1b3221
-size 5905