Add new SentenceTransformer model

1_Pooling/config.json

@@ -1,7 +1,7 @@
 {
-    "word_embedding_dimension": 768,
-    "pooling_mode_cls_token": true,
-    "pooling_mode_mean_tokens": false,
+    "word_embedding_dimension": 384,
+    "pooling_mode_cls_token": false,
+    "pooling_mode_mean_tokens": true,
     "pooling_mode_max_tokens": false,
     "pooling_mode_mean_sqrt_len_tokens": false,
     "pooling_mode_weightedmean_tokens": false,

README.md

@@ -7,7 +7,7 @@ tags:
 - generated_from_trainer
 - dataset_size:90000
 - loss:MultipleNegativesRankingLoss
-base_model: Alibaba-NLP/gte-modernbert-base
+base_model: thenlper/gte-small
 widget:
 - source_sentence: who is the publisher of the norton anthology american literature
   sentences:
@@ -154,7 +154,7 @@ metrics:
 - cosine_mrr@10
 - cosine_map@100
 model-index:
-- name: SentenceTransformer based on Alibaba-NLP/gte-modernbert-base
+- name: SentenceTransformer based on thenlper/gte-small
   results:
   - task:
       type: information-retrieval
@@ -167,46 +167,46 @@ model-index:
       value: 0.38
       name: Cosine Accuracy@1
     - type: cosine_accuracy@3
-      value: 0.68
+      value: 0.58
       name: Cosine Accuracy@3
     - type: cosine_accuracy@5
-      value: 0.8
+      value: 0.66
       name: Cosine Accuracy@5
     - type: cosine_accuracy@10
-      value: 0.86
+      value: 0.8
       name: Cosine Accuracy@10
     - type: cosine_precision@1
       value: 0.38
       name: Cosine Precision@1
     - type: cosine_precision@3
-      value: 0.22666666666666668
+      value: 0.19333333333333336
       name: Cosine Precision@3
     - type: cosine_precision@5
-      value: 0.16
+      value: 0.132
       name: Cosine Precision@5
     - type: cosine_precision@10
-      value: 0.08599999999999998
+      value: 0.08
       name: Cosine Precision@10
     - type: cosine_recall@1
       value: 0.38
       name: Cosine Recall@1
     - type: cosine_recall@3
-      value: 0.68
+      value: 0.58
       name: Cosine Recall@3
     - type: cosine_recall@5
-      value: 0.8
+      value: 0.66
       name: Cosine Recall@5
     - type: cosine_recall@10
-      value: 0.86
+      value: 0.8
       name: Cosine Recall@10
     - type: cosine_ndcg@10
-      value: 0.6232981077766904
+      value: 0.5747352409361379
       name: Cosine Ndcg@10
     - type: cosine_mrr@10
-      value: 0.5465555555555556
+      value: 0.5051349206349205
       name: Cosine Mrr@10
     - type: cosine_map@100
-      value: 0.5540526315789474
+      value: 0.5163932476955072
       name: Cosine Map@100
   - task:
       type: information-retrieval
@@ -216,49 +216,49 @@ model-index:
       type: NanoNQ
     metrics:
     - type: cosine_accuracy@1
-      value: 0.64
+      value: 0.46
       name: Cosine Accuracy@1
     - type: cosine_accuracy@3
-      value: 0.7
+      value: 0.64
       name: Cosine Accuracy@3
     - type: cosine_accuracy@5
-      value: 0.78
+      value: 0.72
       name: Cosine Accuracy@5
     - type: cosine_accuracy@10
-      value: 0.82
+      value: 0.74
       name: Cosine Accuracy@10
     - type: cosine_precision@1
-      value: 0.64
+      value: 0.46
       name: Cosine Precision@1
     - type: cosine_precision@3
-      value: 0.24
+      value: 0.21333333333333332
       name: Cosine Precision@3
     - type: cosine_precision@5
-      value: 0.16
+      value: 0.15200000000000002
       name: Cosine Precision@5
     - type: cosine_precision@10
-      value: 0.08800000000000001
+      value: 0.08
       name: Cosine Precision@10
     - type: cosine_recall@1
-      value: 0.61
+      value: 0.44
       name: Cosine Recall@1
     - type: cosine_recall@3
-      value: 0.66
+      value: 0.6
       name: Cosine Recall@3
     - type: cosine_recall@5
-      value: 0.73
+      value: 0.69
       name: Cosine Recall@5
     - type: cosine_recall@10
-      value: 0.78
+      value: 0.72
       name: Cosine Recall@10
     - type: cosine_ndcg@10
-      value: 0.6987067579229547
+      value: 0.5958872018988118
       name: Cosine Ndcg@10
     - type: cosine_mrr@10
-      value: 0.69
+      value: 0.5641904761904761
       name: Cosine Mrr@10
     - type: cosine_map@100
-      value: 0.6733088641959746
+      value: 0.5591780429569271
       name: Cosine Map@100
   - task:
       type: nano-beir
@@ -268,63 +268,63 @@ model-index:
       type: NanoBEIR_mean
     metrics:
     - type: cosine_accuracy@1
-      value: 0.51
+      value: 0.42000000000000004
       name: Cosine Accuracy@1
     - type: cosine_accuracy@3
-      value: 0.69
+      value: 0.61
       name: Cosine Accuracy@3
     - type: cosine_accuracy@5
-      value: 0.79
+      value: 0.69
       name: Cosine Accuracy@5
     - type: cosine_accuracy@10
-      value: 0.84
+      value: 0.77
       name: Cosine Accuracy@10
     - type: cosine_precision@1
-      value: 0.51
+      value: 0.42000000000000004
       name: Cosine Precision@1
     - type: cosine_precision@3
-      value: 0.23333333333333334
+      value: 0.20333333333333334
       name: Cosine Precision@3
     - type: cosine_precision@5
-      value: 0.16
+      value: 0.14200000000000002
       name: Cosine Precision@5
     - type: cosine_precision@10
-      value: 0.087
+      value: 0.08
       name: Cosine Precision@10
     - type: cosine_recall@1
-      value: 0.495
+      value: 0.41000000000000003
       name: Cosine Recall@1
     - type: cosine_recall@3
-      value: 0.67
+      value: 0.59
       name: Cosine Recall@3
     - type: cosine_recall@5
-      value: 0.765
+      value: 0.675
       name: Cosine Recall@5
     - type: cosine_recall@10
-      value: 0.8200000000000001
+      value: 0.76
       name: Cosine Recall@10
     - type: cosine_ndcg@10
-      value: 0.6610024328498225
+      value: 0.5853112214174749
       name: Cosine Ndcg@10
     - type: cosine_mrr@10
-      value: 0.6182777777777777
+      value: 0.5346626984126983
       name: Cosine Mrr@10
     - type: cosine_map@100
-      value: 0.613680747887461
+      value: 0.5377856453262171
       name: Cosine Map@100
 ---
 
-# SentenceTransformer based on Alibaba-NLP/gte-modernbert-base
+# SentenceTransformer based on thenlper/gte-small
 
-This is a [sentence-transformers](https://www.SBERT.net) model finetuned from [Alibaba-NLP/gte-modernbert-base](https://huggingface.co/Alibaba-NLP/gte-modernbert-base). It maps sentences & paragraphs to a 768-dimensional dense vector space and can be used for semantic textual similarity, semantic search, paraphrase mining, text classification, clustering, and more.
+This is a [sentence-transformers](https://www.SBERT.net) model finetuned from [thenlper/gte-small](https://huggingface.co/thenlper/gte-small). It maps sentences & paragraphs to a 384-dimensional dense vector space and can be used for semantic textual similarity, semantic search, paraphrase mining, text classification, clustering, and more.
 
 ## Model Details
 
 ### Model Description
 - **Model Type:** Sentence Transformer
-- **Base model:** [Alibaba-NLP/gte-modernbert-base](https://huggingface.co/Alibaba-NLP/gte-modernbert-base) <!-- at revision e7f32e3c00f91d699e8c43b53106206bcc72bb22 -->
+- **Base model:** [thenlper/gte-small](https://huggingface.co/thenlper/gte-small) <!-- at revision 17e1f347d17fe144873b1201da91788898c639cd -->
 - **Maximum Sequence Length:** 128 tokens
-- **Output Dimensionality:** 768 dimensions
+- **Output Dimensionality:** 384 dimensions
 - **Similarity Function:** Cosine Similarity
 <!-- - **Training Dataset:** Unknown -->
 <!-- - **Language:** Unknown -->
@@ -340,8 +340,9 @@ This is a [sentence-transformers](https://www.SBERT.net) model finetuned from [A
 
 ```
 SentenceTransformer(
-  (0): Transformer({'max_seq_length': 128, 'do_lower_case': False, 'architecture': 'ModernBertModel'})
-  (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': True, 'pooling_mode_mean_tokens': False, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False, 'pooling_mode_weightedmean_tokens': False, 'pooling_mode_lasttoken': False, 'include_prompt': True})
+  (0): Transformer({'max_seq_length': 128, 'do_lower_case': False, 'architecture': 'BertModel'})
+  (1): Pooling({'word_embedding_dimension': 384, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False, 'pooling_mode_weightedmean_tokens': False, 'pooling_mode_lasttoken': False, 'include_prompt': True})
+  (2): Normalize()
 )
 ```
 
@@ -369,14 +370,14 @@ sentences = [
 ]
 embeddings = model.encode(sentences)
 print(embeddings.shape)
-# [3, 768]
+# [3, 384]
 
 # Get the similarity scores for the embeddings
 similarities = model.similarity(embeddings, embeddings)
 print(similarities)
-# tensor([[ 1.0001,  0.6919, -0.0133],
-#         [ 0.6919,  1.0000, -0.0985],
-#         [-0.0133, -0.0985,  1.0000]])
+# tensor([[1.0000, 0.7343, 0.0079],
+#         [0.7343, 1.0000, 0.0383],
+#         [0.0079, 0.0383, 1.0000]])
 ```
 
 <!--
@@ -414,21 +415,21 @@ You can finetune this model on your own dataset.
 
 | Metric              | NanoMSMARCO | NanoNQ     |
 |:--------------------|:------------|:-----------|
-| cosine_accuracy@1   | 0.38        | 0.64       |
-| cosine_accuracy@3   | 0.68        | 0.7        |
-| cosine_accuracy@5   | 0.8         | 0.78       |
-| cosine_accuracy@10  | 0.86        | 0.82       |
-| cosine_precision@1  | 0.38        | 0.64       |
-| cosine_precision@3  | 0.2267      | 0.24       |
-| cosine_precision@5  | 0.16        | 0.16       |
-| cosine_precision@10 | 0.086       | 0.088      |
-| cosine_recall@1     | 0.38        | 0.61       |
-| cosine_recall@3     | 0.68        | 0.66       |
-| cosine_recall@5     | 0.8         | 0.73       |
-| cosine_recall@10    | 0.86        | 0.78       |
-| **cosine_ndcg@10**  | **0.6233**  | **0.6987** |
-| cosine_mrr@10       | 0.5466      | 0.69       |
-| cosine_map@100      | 0.5541      | 0.6733     |
+| cosine_accuracy@1   | 0.38        | 0.46       |
+| cosine_accuracy@3   | 0.58        | 0.64       |
+| cosine_accuracy@5   | 0.66        | 0.72       |
+| cosine_accuracy@10  | 0.8         | 0.74       |
+| cosine_precision@1  | 0.38        | 0.46       |
+| cosine_precision@3  | 0.1933      | 0.2133     |
+| cosine_precision@5  | 0.132       | 0.152      |
+| cosine_precision@10 | 0.08        | 0.08       |
+| cosine_recall@1     | 0.38        | 0.44       |
+| cosine_recall@3     | 0.58        | 0.6        |
+| cosine_recall@5     | 0.66        | 0.69       |
+| cosine_recall@10    | 0.8         | 0.72       |
+| **cosine_ndcg@10**  | **0.5747**  | **0.5959** |
+| cosine_mrr@10       | 0.5051      | 0.5642     |
+| cosine_map@100      | 0.5164      | 0.5592     |
 
 #### Nano BEIR
 
@@ -444,23 +445,23 @@ You can finetune this model on your own dataset.
   }
   ```
 
-| Metric              | Value     |
-|:--------------------|:----------|
-| cosine_accuracy@1   | 0.51      |
-| cosine_accuracy@3   | 0.69      |
-| cosine_accuracy@5   | 0.79      |
-| cosine_accuracy@10  | 0.84      |
-| cosine_precision@1  | 0.51      |
-| cosine_precision@3  | 0.2333    |
-| cosine_precision@5  | 0.16      |
-| cosine_precision@10 | 0.087     |
-| cosine_recall@1     | 0.495     |
-| cosine_recall@3     | 0.67      |
-| cosine_recall@5     | 0.765     |
-| cosine_recall@10    | 0.82      |
-| **cosine_ndcg@10**  | **0.661** |
-| cosine_mrr@10       | 0.6183    |
-| cosine_map@100      | 0.6137    |
+| Metric              | Value      |
+|:--------------------|:-----------|
+| cosine_accuracy@1   | 0.42       |
+| cosine_accuracy@3   | 0.61       |
+| cosine_accuracy@5   | 0.69       |
+| cosine_accuracy@10  | 0.77       |
+| cosine_precision@1  | 0.42       |
+| cosine_precision@3  | 0.2033     |
+| cosine_precision@5  | 0.142      |
+| cosine_precision@10 | 0.08       |
+| cosine_recall@1     | 0.41       |
+| cosine_recall@3     | 0.59       |
+| cosine_recall@5     | 0.675      |
+| cosine_recall@10    | 0.76       |
+| **cosine_ndcg@10**  | **0.5853** |
+| cosine_mrr@10       | 0.5347     |
+| cosine_map@100      | 0.5378     |
 
 <!--
 ## Bias, Risks and Limitations
@@ -483,10 +484,10 @@ You can finetune this model on your own dataset.
 * Size: 90,000 training samples
 * Columns: <code>anchor</code>, <code>positive</code>, and <code>negative</code>
 * Approximate statistics based on the first 1000 samples:
-  |         | anchor                                                                             | positive                                                                             | negative                                                                             |
-  |:--------|:-----------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------|
-  | type    | string                                                                             | string                                                                               | string                                                                               |
-  | details | <ul><li>min: 10 tokens</li><li>mean: 12.57 tokens</li><li>max: 28 tokens</li></ul> | <ul><li>min: 19 tokens</li><li>mean: 107.04 tokens</li><li>max: 128 tokens</li></ul> | <ul><li>min: 15 tokens</li><li>mean: 105.42 tokens</li><li>max: 128 tokens</li></ul> |
+  |         | anchor                                                                            | positive                                                                            | negative                                                                             |
+  |:--------|:----------------------------------------------------------------------------------|:------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------|
+  | type    | string                                                                            | string                                                                              | string                                                                               |
+  | details | <ul><li>min: 9 tokens</li><li>mean: 11.82 tokens</li><li>max: 27 tokens</li></ul> | <ul><li>min: 19 tokens</li><li>mean: 106.2 tokens</li><li>max: 128 tokens</li></ul> | <ul><li>min: 16 tokens</li><li>mean: 104.63 tokens</li><li>max: 128 tokens</li></ul> |
 * Samples:
   | anchor                                                          | positive                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      | negative                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
   |:----------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
@@ -512,7 +513,7 @@ You can finetune this model on your own dataset.
   |         | anchor                                                                            | positive                                                                             | negative                                                                             |
   |:--------|:----------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------|
   | type    | string                                                                            | string                                                                               | string                                                                               |
-  | details | <ul><li>min: 9 tokens</li><li>mean: 12.46 tokens</li><li>max: 25 tokens</li></ul> | <ul><li>min: 16 tokens</li><li>mean: 106.89 tokens</li><li>max: 128 tokens</li></ul> | <ul><li>min: 11 tokens</li><li>mean: 106.57 tokens</li><li>max: 128 tokens</li></ul> |
+  | details | <ul><li>min: 9 tokens</li><li>mean: 11.76 tokens</li><li>max: 24 tokens</li></ul> | <ul><li>min: 15 tokens</li><li>mean: 105.95 tokens</li><li>max: 128 tokens</li></ul> | <ul><li>min: 11 tokens</li><li>mean: 105.69 tokens</li><li>max: 128 tokens</li></ul> |
 * Samples:
   | anchor                                                                                           | positive                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 | negative                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
   |:-------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
@@ -534,9 +535,9 @@ You can finetune this model on your own dataset.
 - `eval_strategy`: steps
 - `per_device_train_batch_size`: 128
 - `per_device_eval_batch_size`: 128
-- `learning_rate`: 4e-05
-- `weight_decay`: 0.01
-- `max_steps`: 500
+- `learning_rate`: 8e-05
+- `weight_decay`: 0.005
+- `max_steps`: 1125
 - `warmup_ratio`: 0.1
 - `fp16`: True
 - `dataloader_drop_last`: True
@@ -563,14 +564,14 @@ You can finetune this model on your own dataset.
 - `gradient_accumulation_steps`: 1
 - `eval_accumulation_steps`: None
 - `torch_empty_cache_steps`: None
-- `learning_rate`: 4e-05
-- `weight_decay`: 0.01
+- `learning_rate`: 8e-05
+- `weight_decay`: 0.005
 - `adam_beta1`: 0.9
 - `adam_beta2`: 0.999
 - `adam_epsilon`: 1e-08
 - `max_grad_norm`: 1.0
 - `num_train_epochs`: 3.0
-- `max_steps`: 500
+- `max_steps`: 1125
 - `lr_scheduler_type`: linear
 - `lr_scheduler_kwargs`: {}
 - `warmup_ratio`: 0.1
@@ -675,13 +676,14 @@ You can finetune this model on your own dataset.
 </details>
 
 ### Training Logs
-| Epoch      | Step    | Training Loss | Validation Loss | NanoMSMARCO_cosine_ndcg@10 | NanoNQ_cosine_ndcg@10 | NanoBEIR_mean_cosine_ndcg@10 |
-|:----------:|:-------:|:-------------:|:---------------:|:--------------------------:|:---------------------:|:----------------------------:|
-| 0          | 0       | -             | 0.4265          | 0.6530                     | 0.6552                | 0.6541                       |
-| 0.3556     | 250     | 0.0816        | 0.0565          | 0.6334                     | 0.6822                | 0.6578                       |
-| **0.7112** | **500** | **0.0517**    | **0.052**       | **0.6233**                 | **0.6987**            | **0.661**                    |
+| Epoch  | Step | Training Loss | Validation Loss | NanoMSMARCO_cosine_ndcg@10 | NanoNQ_cosine_ndcg@10 | NanoBEIR_mean_cosine_ndcg@10 |
+|:------:|:----:|:-------------:|:---------------:|:--------------------------:|:---------------------:|:----------------------------:|
+| 0      | 0    | -             | 2.1869          | 0.6259                     | 0.6583                | 0.6421                       |
+| 0.3556 | 250  | 0.3907        | 0.0761          | 0.5880                     | 0.6146                | 0.6013                       |
+| 0.7112 | 500  | 0.0814        | 0.0680          | 0.5666                     | 0.6170                | 0.5918                       |
+| 1.0669 | 750  | 0.0714        | 0.0634          | 0.5580                     | 0.5846                | 0.5713                       |
+| 1.4225 | 1000 | 0.0406        | 0.0614          | 0.5747                     | 0.5959                | 0.5853                       |
 
-* The bold row denotes the saved checkpoint.
 
 ### Framework Versions
 - Python: 3.10.18

config_sentence_transformers.json

@@ -1,4 +1,5 @@
 {
+  "model_type": "SentenceTransformer",
   "__version__": {
     "sentence_transformers": "5.2.0",
     "transformers": "4.57.3",
@@ -9,6 +10,5 @@
     "document": ""
   },
   "default_prompt_name": null,
-  "similarity_fn_name": "cosine",
-  "model_type": "SentenceTransformer"
+  "similarity_fn_name": "cosine"
 }

modules.json

@@ -10,5 +10,11 @@
     "name": "1",
     "path": "1_Pooling",
     "type": "sentence_transformers.models.Pooling"
+  },
+  {
+    "idx": 2,
+    "name": "2",
+    "path": "2_Normalize",
+    "type": "sentence_transformers.models.Normalize"
   }
 ]