Training in progress, step 5000

config.json

@@ -5,6 +5,7 @@
   "attention_probs_dropout_prob": 0.1,
   "classifier_dropout": null,
   "dtype": "bfloat16",
+  "gradient_checkpointing": false,
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
   "hidden_size": 384,
@@ -14,7 +15,7 @@
   "max_position_embeddings": 512,
   "model_type": "bert",
   "num_attention_heads": 12,
-  "num_hidden_layers": 12,
+  "num_hidden_layers": 6,
   "pad_token_id": 0,
   "position_embedding_type": "absolute",
   "transformers_version": "4.57.0",

eval/Information-Retrieval_evaluation_test_results.csv

@@ -60,3 +60,27 @@ epoch,steps,cosine-Accuracy@1,cosine-Precision@1,cosine-Recall@1,cosine-MRR@1,co
 0.49206544470414565,4000,0.5136291460832745,0.5136291460832745,0.49609913213697615,0.5136291460832745,0.7094132063717694,0.6539726048972176,{1: np.float64(0.3119784463901498)},{1: np.float64(0.21389626227951036)}
 0.5535736252921638,4500,0.5121736062103035,0.5121736062103035,0.49458110696642804,0.5121736062103035,0.7075051193793919,0.6522760525794585,{1: np.float64(0.31343196470110707)},{1: np.float64(0.21483213049895286)}
 0.615081805880182,5000,0.5073659139026111,0.5073659139026111,0.48995940030917096,0.5073659139026111,0.704125968555372,0.6486868844121669,{1: np.float64(0.30976968186914705)},{1: np.float64(0.21846349203066784)}
+0.6765899864682002,5500,0.5051164431898377,0.5051164431898377,0.4877760904997144,0.5051164431898377,0.7022776742806202,0.6468460558896545,{1: np.float64(0.30797653909235867)},{1: np.float64(0.21939950112168932)}
+0.7380981670562184,6000,0.5052487649964714,0.5052487649964714,0.48791208791208796,0.5052487649964714,0.7018006758534852,0.6463104839757691,{1: np.float64(0.30939388097644616)},{1: np.float64(0.22077790819696838)}
+0.7996063476442367,6500,0.5031757233592096,0.5031757233592096,0.4858743320899284,0.5031757233592096,0.7018024941985244,0.6457940441222814,{1: np.float64(0.3071103744572787)},{1: np.float64(0.21924260973263582)}
+0.8611145282322549,7000,0.5014555398729711,0.5014555398729711,0.48431366989279834,0.5014555398729711,0.7003065535212539,0.644486083880378,{1: np.float64(0.3059839444807557)},{1: np.float64(0.22119951578936942)}
+0.9226227088202731,7500,0.5005733944954128,0.5005733944954128,0.48360060237927216,0.5005733944954128,0.6997240529681923,0.6437306889155522,{1: np.float64(0.3058524170445839)},{1: np.float64(0.2215523302463902)}
+0,0,0.5309191954834157,0.5309191954834157,0.5126434011325066,0.5309191954834157,0.7314305279871582,0.675229160358019,{1: np.float64(0.3015556447544318)},{1: np.float64(0.19690931872370762)}
+0.03075598203850649,500,0.5370059985885673,0.5370059985885673,0.5186901401350943,0.5370059985885673,0.737809170544929,0.6818464315190007,{1: np.float64(0.30481837104809123)},{1: np.float64(0.17553712520412681)}
+0.06151196407701298,1000,0.5384615384615384,0.5384615384615384,0.5200659193635111,0.5384615384615384,0.7358701729703853,0.6804762985676286,{1: np.float64(0.31569810249687086)},{1: np.float64(0.18389194101031053)}
+0,0,0.5309191954834157,0.5309191954834157,0.5126434011325066,0.5309191954834157,0.7314305279871582,0.675229160358019,{1: np.float64(0.3015556447544318)},{1: np.float64(0.19690931872370762)}
+0,0,0.5474153140437544,0.5474153140437544,0.5284808511442686,0.5474153140437544,0.7464206027541721,0.6905111169633417,{1: np.float64(0.3145837338283234)},{1: np.float64(0.16086033196070731)}
+0.03075598203850649,500,0.547106563161609,0.547106563161609,0.5281706300198272,0.547106563161609,0.7461821929919408,0.690293009969878,{1: np.float64(0.31459186837145553)},{1: np.float64(0.16280243465720218)}
+0.06151196407701298,1000,0.5404463655610444,0.5404463655610444,0.5219959799374937,0.5404463655610444,0.7407090011662882,0.6845898197371095,{1: np.float64(0.31411398761743997)},{1: np.float64(0.179996654024503)}
+0.09226794611551947,1500,0.531227946365561,0.531227946365561,0.5130385287495379,0.531227946365561,0.7319342349371158,0.6752496263522522,{1: np.float64(0.3114153905151002)},{1: np.float64(0.20177076753513354)}
+0,0,0.5474153140437544,0.5474153140437544,0.5284808511442686,0.5474153140437544,0.7464206027541721,0.6905111169633417,{1: np.float64(0.3145837338283234)},{1: np.float64(0.16086033196070731)}
+0.03075598203850649,500,0.547106563161609,0.547106563161609,0.5281706300198272,0.547106563161609,0.7461821929919408,0.690293009969878,{1: np.float64(0.31459186837145553)},{1: np.float64(0.16280243465720218)}
+0.06151196407701298,1000,0.5404463655610444,0.5404463655610444,0.5219959799374937,0.5404463655610444,0.7407090011662882,0.6845898197371095,{1: np.float64(0.31411398761743997)},{1: np.float64(0.179996654024503)}
+0.09226794611551947,1500,0.531227946365561,0.531227946365561,0.5130385287495379,0.531227946365561,0.7319342349371158,0.6752496263522522,{1: np.float64(0.3114153905151002)},{1: np.float64(0.20177076753513354)}
+0.12302392815402596,2000,0.5258468595624559,0.5258468595624559,0.5078706270793427,0.5258468595624559,0.7245864644994908,0.6685842514505764,{1: np.float64(0.3123872498414073)},{1: np.float64(0.20395092275330823)}
+0.15377991019253245,2500,0.5213920254057869,0.5213920254057869,0.5035958976039251,0.5213920254057869,0.719590786147121,0.6638421006585614,{1: np.float64(0.31129992235883797)},{1: np.float64(0.2047724041817989)}
+0.18453589223103894,3000,0.5178193366266761,0.5178193366266761,0.5000673160936923,0.5178193366266761,0.7163131766086044,0.6603664447787778,{1: np.float64(0.3085206311438812)},{1: np.float64(0.20528798172967666)}
+0.21529187426954544,3500,0.5159668313338038,0.5159668313338038,0.49833243018449447,0.5159668313338038,0.7145314571439964,0.6586537491588038,{1: np.float64(0.30641447356250734)},{1: np.float64(0.20732705885352593)}
+0.24604785630805193,4000,0.5132762879322512,0.5132762879322512,0.4957499495916927,0.5132762879322512,0.7123072279482698,0.6561196211679712,{1: np.float64(0.3058214968650276)},{1: np.float64(0.21048968308185473)}
+0.2768038383465584,4500,0.5087332392378264,0.5087332392378264,0.4910709034848943,0.5087332392378264,0.709310692558922,0.652656638617702,{1: np.float64(0.30174652682850833)},{1: np.float64(0.21226196776001288)}
+0.3075598203850649,5000,0.5047194777699365,0.5047194777699365,0.48727988372483777,0.5047194777699365,0.7059068523784022,0.6490213484284776,{1: np.float64(0.3008660367505195)},{1: np.float64(0.21497544837116472)}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:551613faa4cf473038ea65dedbd058fb620e0c7f8d94dc653d50e5ee4508fe69
-size 66742184
+oid sha256:f94a3ce836e18aa6c43796ded8fc6e30cc777f68148f860fc7554a79b4fcbb59
+size 45437864

tokenizer_config.json

@@ -41,17 +41,25 @@
       "special": true
     }
   },
-  "clean_up_tokenization_spaces": true,
+  "clean_up_tokenization_spaces": false,
   "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
   "do_lower_case": true,
   "extra_special_tokens": {},
   "mask_token": "[MASK]",
   "max_length": 64,
-  "model_max_length": 512,
+  "model_max_length": 256,
+  "never_split": null,
+  "pad_to_multiple_of": null,
   "pad_token": "[PAD]",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
   "sep_token": "[SEP]",
+  "stride": 0,
   "strip_accents": null,
   "tokenize_chinese_chars": true,
   "tokenizer_class": "BertTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
   "unk_token": "[UNK]"
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d4ec5e27753a1494ed86180cda30a69319d61af98b822adc19821142a45f8eae
+oid sha256:317bd5b83332174b76fe35a4fc644ffadedecb5bcbd748cb002e6652cc3d9041
 size 6353