Add new SentenceTransformer model

1_Pooling/config.json

@@ -1,7 +1,7 @@
 {
-    "word_embedding_dimension": 384,
-    "pooling_mode_cls_token": false,
-    "pooling_mode_mean_tokens": true,
+    "word_embedding_dimension": 768,
+    "pooling_mode_cls_token": true,
+    "pooling_mode_mean_tokens": false,
     "pooling_mode_max_tokens": false,
     "pooling_mode_mean_sqrt_len_tokens": false,
     "pooling_mode_weightedmean_tokens": false,

README.md

@@ -7,7 +7,7 @@ tags:
 - generated_from_trainer
 - dataset_size:111470
 - loss:MultipleNegativesRankingLoss
-base_model: sentence-transformers/all-MiniLM-L12-v2
+base_model: Alibaba-NLP/gte-modernbert-base
 widget:
 - source_sentence: why are some rocks radioactive
   sentences:
@@ -106,7 +106,7 @@ metrics:
 - cosine_mrr@10
 - cosine_map@100
 model-index:
-- name: SentenceTransformer based on sentence-transformers/all-MiniLM-L12-v2
+- name: SentenceTransformer based on Alibaba-NLP/gte-modernbert-base
   results:
   - task:
       type: information-retrieval
@@ -116,49 +116,49 @@ model-index:
       type: NanoMSMARCO
     metrics:
     - type: cosine_accuracy@1
-      value: 0.36
+      value: 0.42
       name: Cosine Accuracy@1
     - type: cosine_accuracy@3
-      value: 0.58
+      value: 0.64
       name: Cosine Accuracy@3
     - type: cosine_accuracy@5
-      value: 0.62
+      value: 0.78
       name: Cosine Accuracy@5
     - type: cosine_accuracy@10
-      value: 0.7
+      value: 0.84
       name: Cosine Accuracy@10
     - type: cosine_precision@1
-      value: 0.36
+      value: 0.42
       name: Cosine Precision@1
     - type: cosine_precision@3
-      value: 0.19333333333333333
+      value: 0.21333333333333332
       name: Cosine Precision@3
     - type: cosine_precision@5
-      value: 0.124
+      value: 0.156
       name: Cosine Precision@5
     - type: cosine_precision@10
-      value: 0.07
+      value: 0.08399999999999999
       name: Cosine Precision@10
     - type: cosine_recall@1
-      value: 0.36
+      value: 0.42
       name: Cosine Recall@1
     - type: cosine_recall@3
-      value: 0.58
+      value: 0.64
       name: Cosine Recall@3
     - type: cosine_recall@5
-      value: 0.62
+      value: 0.78
       name: Cosine Recall@5
     - type: cosine_recall@10
-      value: 0.7
+      value: 0.84
       name: Cosine Recall@10
     - type: cosine_ndcg@10
-      value: 0.5293969846378502
+      value: 0.6273713143801162
       name: Cosine Ndcg@10
     - type: cosine_mrr@10
-      value: 0.47507936507936505
+      value: 0.5593571428571429
       name: Cosine Mrr@10
     - type: cosine_map@100
-      value: 0.4876720668963073
+      value: 0.567451526639622
       name: Cosine Map@100
   - task:
       type: information-retrieval
@@ -168,49 +168,49 @@ model-index:
       type: NanoNQ
     metrics:
     - type: cosine_accuracy@1
-      value: 0.4
+      value: 0.44
       name: Cosine Accuracy@1
     - type: cosine_accuracy@3
       value: 0.56
       name: Cosine Accuracy@3
     - type: cosine_accuracy@5
-      value: 0.58
+      value: 0.62
       name: Cosine Accuracy@5
     - type: cosine_accuracy@10
       value: 0.74
       name: Cosine Accuracy@10
     - type: cosine_precision@1
-      value: 0.4
+      value: 0.44
       name: Cosine Precision@1
     - type: cosine_precision@3
-      value: 0.19333333333333333
+      value: 0.18666666666666665
       name: Cosine Precision@3
     - type: cosine_precision@5
-      value: 0.12
+      value: 0.128
       name: Cosine Precision@5
     - type: cosine_precision@10
-      value: 0.07600000000000001
+      value: 0.08
       name: Cosine Precision@10
     - type: cosine_recall@1
-      value: 0.37
+      value: 0.4
       name: Cosine Recall@1
     - type: cosine_recall@3
-      value: 0.53
+      value: 0.52
       name: Cosine Recall@3
     - type: cosine_recall@5
-      value: 0.55
+      value: 0.59
       name: Cosine Recall@5
     - type: cosine_recall@10
-      value: 0.7
+      value: 0.71
       name: Cosine Recall@10
     - type: cosine_ndcg@10
-      value: 0.5283136453383426
+      value: 0.5468372621429358
       name: Cosine Ndcg@10
     - type: cosine_mrr@10
-      value: 0.49621428571428566
+      value: 0.5185555555555555
       name: Cosine Mrr@10
     - type: cosine_map@100
-      value: 0.47570813068007134
+      value: 0.49953000242452567
       name: Cosine Map@100
   - task:
       type: nano-beir
@@ -220,63 +220,63 @@ model-index:
       type: NanoBEIR_mean
     metrics:
     - type: cosine_accuracy@1
-      value: 0.38
+      value: 0.43
       name: Cosine Accuracy@1
     - type: cosine_accuracy@3
-      value: 0.5700000000000001
+      value: 0.6000000000000001
       name: Cosine Accuracy@3
     - type: cosine_accuracy@5
-      value: 0.6
+      value: 0.7
       name: Cosine Accuracy@5
     - type: cosine_accuracy@10
-      value: 0.72
+      value: 0.79
       name: Cosine Accuracy@10
     - type: cosine_precision@1
-      value: 0.38
+      value: 0.43
       name: Cosine Precision@1
     - type: cosine_precision@3
-      value: 0.19333333333333333
+      value: 0.19999999999999998
       name: Cosine Precision@3
     - type: cosine_precision@5
-      value: 0.122
+      value: 0.14200000000000002
       name: Cosine Precision@5
     - type: cosine_precision@10
-      value: 0.07300000000000001
+      value: 0.08199999999999999
       name: Cosine Precision@10
     - type: cosine_recall@1
-      value: 0.365
+      value: 0.41000000000000003
       name: Cosine Recall@1
     - type: cosine_recall@3
-      value: 0.5549999999999999
+      value: 0.5800000000000001
       name: Cosine Recall@3
     - type: cosine_recall@5
-      value: 0.585
+      value: 0.685
       name: Cosine Recall@5
     - type: cosine_recall@10
-      value: 0.7
+      value: 0.7749999999999999
       name: Cosine Recall@10
     - type: cosine_ndcg@10
-      value: 0.5288553149880963
+      value: 0.587104288261526
       name: Cosine Ndcg@10
     - type: cosine_mrr@10
-      value: 0.48564682539682535
+      value: 0.5389563492063492
       name: Cosine Mrr@10
     - type: cosine_map@100
-      value: 0.48169009878818936
+      value: 0.5334907645320738
       name: Cosine Map@100
 ---
 
-# SentenceTransformer based on sentence-transformers/all-MiniLM-L12-v2
+# SentenceTransformer based on Alibaba-NLP/gte-modernbert-base
 
-This is a [sentence-transformers](https://www.SBERT.net) model finetuned from [sentence-transformers/all-MiniLM-L12-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L12-v2). It maps sentences & paragraphs to a 384-dimensional dense vector space and can be used for semantic textual similarity, semantic search, paraphrase mining, text classification, clustering, and more.
+This is a [sentence-transformers](https://www.SBERT.net) model finetuned from [Alibaba-NLP/gte-modernbert-base](https://huggingface.co/Alibaba-NLP/gte-modernbert-base). It maps sentences & paragraphs to a 768-dimensional dense vector space and can be used for semantic textual similarity, semantic search, paraphrase mining, text classification, clustering, and more.
 
 ## Model Details
 
 ### Model Description
 - **Model Type:** Sentence Transformer
-- **Base model:** [sentence-transformers/all-MiniLM-L12-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L12-v2) <!-- at revision 936af83a2ecce5fe87a09109ff5cbcefe073173a -->
+- **Base model:** [Alibaba-NLP/gte-modernbert-base](https://huggingface.co/Alibaba-NLP/gte-modernbert-base) <!-- at revision e7f32e3c00f91d699e8c43b53106206bcc72bb22 -->
 - **Maximum Sequence Length:** 128 tokens
-- **Output Dimensionality:** 384 dimensions
+- **Output Dimensionality:** 768 dimensions
 - **Similarity Function:** Cosine Similarity
 <!-- - **Training Dataset:** Unknown -->
 <!-- - **Language:** Unknown -->
@@ -292,9 +292,8 @@ This is a [sentence-transformers](https://www.SBERT.net) model finetuned from [s
 
 ```
 SentenceTransformer(
-  (0): Transformer({'max_seq_length': 128, 'do_lower_case': False, 'architecture': 'BertModel'})
-  (1): Pooling({'word_embedding_dimension': 384, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False, 'pooling_mode_weightedmean_tokens': False, 'pooling_mode_lasttoken': False, 'include_prompt': True})
-  (2): Normalize()
+  (0): Transformer({'max_seq_length': 128, 'do_lower_case': False, 'architecture': 'ModernBertModel'})
+  (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': True, 'pooling_mode_mean_tokens': False, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False, 'pooling_mode_weightedmean_tokens': False, 'pooling_mode_lasttoken': False, 'include_prompt': True})
 )
 ```
 
@@ -322,14 +321,14 @@ sentences = [
 ]
 embeddings = model.encode(sentences)
 print(embeddings.shape)
-# [3, 384]
+# [3, 768]
 
 # Get the similarity scores for the embeddings
 similarities = model.similarity(embeddings, embeddings)
 print(similarities)
-# tensor([[1.0000, 1.0000, 0.9401],
-#         [1.0000, 1.0000, 0.9401],
-#         [0.9401, 0.9401, 1.0000]])
+# tensor([[1.0000, 1.0000, 0.3177],
+#         [1.0000, 1.0000, 0.3177],
+#         [0.3177, 0.3177, 1.0000]])
 ```
 
 <!--
@@ -367,21 +366,21 @@ You can finetune this model on your own dataset.
 
 | Metric              | NanoMSMARCO | NanoNQ     |
 |:--------------------|:------------|:-----------|
-| cosine_accuracy@1   | 0.36        | 0.4        |
-| cosine_accuracy@3   | 0.58        | 0.56       |
-| cosine_accuracy@5   | 0.62        | 0.58       |
-| cosine_accuracy@10  | 0.7         | 0.74       |
-| cosine_precision@1  | 0.36        | 0.4        |
-| cosine_precision@3  | 0.1933      | 0.1933     |
-| cosine_precision@5  | 0.124       | 0.12       |
-| cosine_precision@10 | 0.07        | 0.076      |
-| cosine_recall@1     | 0.36        | 0.37       |
-| cosine_recall@3     | 0.58        | 0.53       |
-| cosine_recall@5     | 0.62        | 0.55       |
-| cosine_recall@10    | 0.7         | 0.7        |
-| **cosine_ndcg@10**  | **0.5294**  | **0.5283** |
-| cosine_mrr@10       | 0.4751      | 0.4962     |
-| cosine_map@100      | 0.4877      | 0.4757     |
+| cosine_accuracy@1   | 0.42        | 0.44       |
+| cosine_accuracy@3   | 0.64        | 0.56       |
+| cosine_accuracy@5   | 0.78        | 0.62       |
+| cosine_accuracy@10  | 0.84        | 0.74       |
+| cosine_precision@1  | 0.42        | 0.44       |
+| cosine_precision@3  | 0.2133      | 0.1867     |
+| cosine_precision@5  | 0.156       | 0.128      |
+| cosine_precision@10 | 0.084       | 0.08       |
+| cosine_recall@1     | 0.42        | 0.4        |
+| cosine_recall@3     | 0.64        | 0.52       |
+| cosine_recall@5     | 0.78        | 0.59       |
+| cosine_recall@10    | 0.84        | 0.71       |
+| **cosine_ndcg@10**  | **0.6274**  | **0.5468** |
+| cosine_mrr@10       | 0.5594      | 0.5186     |
+| cosine_map@100      | 0.5675      | 0.4995     |
 
 #### Nano BEIR
 
@@ -399,21 +398,21 @@ You can finetune this model on your own dataset.
 
 | Metric              | Value      |
 |:--------------------|:-----------|
-| cosine_accuracy@1   | 0.38       |
-| cosine_accuracy@3   | 0.57       |
-| cosine_accuracy@5   | 0.6        |
-| cosine_accuracy@10  | 0.72       |
-| cosine_precision@1  | 0.38       |
-| cosine_precision@3  | 0.1933     |
-| cosine_precision@5  | 0.122      |
-| cosine_precision@10 | 0.073      |
-| cosine_recall@1     | 0.365      |
-| cosine_recall@3     | 0.555      |
-| cosine_recall@5     | 0.585      |
-| cosine_recall@10    | 0.7        |
-| **cosine_ndcg@10**  | **0.5289** |
-| cosine_mrr@10       | 0.4856     |
-| cosine_map@100      | 0.4817     |
+| cosine_accuracy@1   | 0.43       |
+| cosine_accuracy@3   | 0.6        |
+| cosine_accuracy@5   | 0.7        |
+| cosine_accuracy@10  | 0.79       |
+| cosine_precision@1  | 0.43       |
+| cosine_precision@3  | 0.2        |
+| cosine_precision@5  | 0.142      |
+| cosine_precision@10 | 0.082      |
+| cosine_recall@1     | 0.41       |
+| cosine_recall@3     | 0.58       |
+| cosine_recall@5     | 0.685      |
+| cosine_recall@10    | 0.775      |
+| **cosine_ndcg@10**  | **0.5871** |
+| cosine_mrr@10       | 0.539      |
+| cosine_map@100      | 0.5335     |
 
 <!--
 ## Bias, Risks and Limitations
@@ -439,7 +438,7 @@ You can finetune this model on your own dataset.
   |         | anchor                                                                            | positive                                                                           | negative                                                                           |
   |:--------|:----------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------|
   | type    | string                                                                            | string                                                                             | string                                                                             |
-  | details | <ul><li>min: 4 tokens</li><li>mean: 10.95 tokens</li><li>max: 60 tokens</li></ul> | <ul><li>min: 6 tokens</li><li>mean: 67.57 tokens</li><li>max: 128 tokens</li></ul> | <ul><li>min: 7 tokens</li><li>mean: 66.64 tokens</li><li>max: 128 tokens</li></ul> |
+  | details | <ul><li>min: 4 tokens</li><li>mean: 11.17 tokens</li><li>max: 59 tokens</li></ul> | <ul><li>min: 6 tokens</li><li>mean: 68.53 tokens</li><li>max: 128 tokens</li></ul> | <ul><li>min: 7 tokens</li><li>mean: 67.56 tokens</li><li>max: 128 tokens</li></ul> |
 * Samples:
   | anchor                                                                | positive                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               | negative                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     |
   |:----------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
@@ -465,7 +464,7 @@ You can finetune this model on your own dataset.
   |         | anchor                                                                            | positive                                                                           | negative                                                                           |
   |:--------|:----------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------|
   | type    | string                                                                            | string                                                                             | string                                                                             |
-  | details | <ul><li>min: 4 tokens</li><li>mean: 11.11 tokens</li><li>max: 66 tokens</li></ul> | <ul><li>min: 7 tokens</li><li>mean: 67.99 tokens</li><li>max: 128 tokens</li></ul> | <ul><li>min: 7 tokens</li><li>mean: 66.08 tokens</li><li>max: 128 tokens</li></ul> |
+  | details | <ul><li>min: 4 tokens</li><li>mean: 11.35 tokens</li><li>max: 64 tokens</li></ul> | <ul><li>min: 7 tokens</li><li>mean: 68.67 tokens</li><li>max: 128 tokens</li></ul> | <ul><li>min: 6 tokens</li><li>mean: 67.03 tokens</li><li>max: 128 tokens</li></ul> |
 * Samples:
   | anchor                                                                                                    | positive                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                | negative                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
   |:----------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
@@ -487,9 +486,9 @@ You can finetune this model on your own dataset.
 - `eval_strategy`: steps
 - `per_device_train_batch_size`: 128
 - `per_device_eval_batch_size`: 128
-- `learning_rate`: 8e-05
-- `weight_decay`: 0.005
-- `max_steps`: 3375
+- `learning_rate`: 4e-05
+- `weight_decay`: 0.01
+- `max_steps`: 703
 - `warmup_ratio`: 0.1
 - `fp16`: True
 - `dataloader_drop_last`: True
@@ -516,14 +515,14 @@ You can finetune this model on your own dataset.
 - `gradient_accumulation_steps`: 1
 - `eval_accumulation_steps`: None
 - `torch_empty_cache_steps`: None
-- `learning_rate`: 8e-05
-- `weight_decay`: 0.005
+- `learning_rate`: 4e-05
+- `weight_decay`: 0.01
 - `adam_beta1`: 0.9
 - `adam_beta2`: 0.999
 - `adam_epsilon`: 1e-08
 - `max_grad_norm`: 1.0
 - `num_train_epochs`: 3.0
-- `max_steps`: 3375
+- `max_steps`: 703
 - `lr_scheduler_type`: linear
 - `lr_scheduler_kwargs`: {}
 - `warmup_ratio`: 0.1
@@ -630,20 +629,9 @@ You can finetune this model on your own dataset.
 ### Training Logs
 | Epoch  | Step | Training Loss | Validation Loss | NanoMSMARCO_cosine_ndcg@10 | NanoNQ_cosine_ndcg@10 | NanoBEIR_mean_cosine_ndcg@10 |
 |:------:|:----:|:-------------:|:---------------:|:--------------------------:|:---------------------:|:----------------------------:|
-| 0      | 0    | -             | 1.1142          | 0.5887                     | 0.5786                | 0.5836                       |
-| 0.2874 | 250  | 1.0308        | 0.8274          | 0.5370                     | 0.5295                | 0.5332                       |
-| 0.5747 | 500  | 0.9418        | 0.8097          | 0.5222                     | 0.5367                | 0.5295                       |
-| 0.8621 | 750  | 0.9198        | 0.7884          | 0.5348                     | 0.5194                | 0.5271                       |
-| 1.1494 | 1000 | 0.8563        | 0.7848          | 0.5172                     | 0.5148                | 0.5160                       |
-| 1.4368 | 1250 | 0.8147        | 0.7826          | 0.5236                     | 0.4794                | 0.5015                       |
-| 1.7241 | 1500 | 0.8074        | 0.7742          | 0.5312                     | 0.5117                | 0.5214                       |
-| 2.0115 | 1750 | 0.8021        | 0.7775          | 0.5468                     | 0.4985                | 0.5226                       |
-| 2.2989 | 2000 | 0.7212        | 0.7763          | 0.5045                     | 0.4867                | 0.4956                       |
-| 2.5862 | 2250 | 0.7197        | 0.7796          | 0.5123                     | 0.4800                | 0.4962                       |
-| 2.8736 | 2500 | 0.7116        | 0.7781          | 0.5451                     | 0.5259                | 0.5355                       |
-| 3.1609 | 2750 | 0.6905        | 0.7827          | 0.5065                     | 0.5346                | 0.5205                       |
-| 3.4483 | 3000 | 0.6656        | 0.7844          | 0.5287                     | 0.5066                | 0.5176                       |
-| 3.7356 | 3250 | 0.6632        | 0.7836          | 0.5294                     | 0.5283                | 0.5289                       |
+| 0      | 0    | -             | 2.5772          | 0.6530                     | 0.6552                | 0.6541                       |
+| 0.2874 | 250  | 0.9649        | 0.7574          | 0.6170                     | 0.5720                | 0.5945                       |
+| 0.5747 | 500  | 0.7456        | 0.7372          | 0.6274                     | 0.5468                | 0.5871                       |
 
 
 ### Framework Versions

config_sentence_transformers.json

@@ -4,11 +4,11 @@
     "transformers": "4.57.3",
     "pytorch": "2.9.1+cu128"
   },
-  "model_type": "SentenceTransformer",
   "prompts": {
     "query": "",
     "document": ""
   },
   "default_prompt_name": null,
-  "similarity_fn_name": "cosine"
+  "similarity_fn_name": "cosine",
+  "model_type": "SentenceTransformer"
 }

modules.json

@@ -10,11 +10,5 @@
     "name": "1",
     "path": "1_Pooling",
     "type": "sentence_transformers.models.Pooling"
-  },
-  {
-    "idx": 2,
-    "name": "2",
-    "path": "2_Normalize",
-    "type": "sentence_transformers.models.Normalize"
   }
 ]