Add new SentenceTransformer model

README.md

@@ -67,49 +67,49 @@ model-index:
       type: NanoMSMARCO
     metrics:
     - type: cosine_accuracy@1
-      value: 0.3
+      value: 0.36
       name: Cosine Accuracy@1
     - type: cosine_accuracy@3
-      value: 0.62
+      value: 0.6
       name: Cosine Accuracy@3
     - type: cosine_accuracy@5
-      value: 0.7
+      value: 0.68
       name: Cosine Accuracy@5
     - type: cosine_accuracy@10
-      value: 0.76
+      value: 0.78
       name: Cosine Accuracy@10
     - type: cosine_precision@1
-      value: 0.3
+      value: 0.36
       name: Cosine Precision@1
     - type: cosine_precision@3
-      value: 0.20666666666666664
+      value: 0.2
       name: Cosine Precision@3
     - type: cosine_precision@5
-      value: 0.14
+      value: 0.136
       name: Cosine Precision@5
     - type: cosine_precision@10
-      value: 0.07600000000000001
+      value: 0.07800000000000001
       name: Cosine Precision@10
     - type: cosine_recall@1
-      value: 0.3
+      value: 0.36
       name: Cosine Recall@1
     - type: cosine_recall@3
-      value: 0.62
+      value: 0.6
       name: Cosine Recall@3
     - type: cosine_recall@5
-      value: 0.7
+      value: 0.68
       name: Cosine Recall@5
     - type: cosine_recall@10
-      value: 0.76
+      value: 0.78
       name: Cosine Recall@10
     - type: cosine_ndcg@10
-      value: 0.5415969332190478
+      value: 0.5686788105462819
       name: Cosine Ndcg@10
     - type: cosine_mrr@10
-      value: 0.47016666666666657
+      value: 0.5018888888888889
       name: Cosine Mrr@10
     - type: cosine_map@100
-      value: 0.4817150619220429
+      value: 0.5110826036192063
       name: Cosine Map@100
   - task:
       type: information-retrieval
@@ -119,49 +119,49 @@ model-index:
       type: NanoNQ
     metrics:
     - type: cosine_accuracy@1
-      value: 0.38
+      value: 0.36
       name: Cosine Accuracy@1
     - type: cosine_accuracy@3
-      value: 0.56
+      value: 0.66
       name: Cosine Accuracy@3
     - type: cosine_accuracy@5
-      value: 0.62
+      value: 0.68
       name: Cosine Accuracy@5
     - type: cosine_accuracy@10
-      value: 0.7
+      value: 0.76
       name: Cosine Accuracy@10
     - type: cosine_precision@1
-      value: 0.38
+      value: 0.36
       name: Cosine Precision@1
     - type: cosine_precision@3
-      value: 0.19333333333333333
+      value: 0.22
       name: Cosine Precision@3
     - type: cosine_precision@5
-      value: 0.132
+      value: 0.14
       name: Cosine Precision@5
     - type: cosine_precision@10
-      value: 0.07400000000000001
+      value: 0.08
       name: Cosine Precision@10
     - type: cosine_recall@1
-      value: 0.35
+      value: 0.33
       name: Cosine Recall@1
     - type: cosine_recall@3
-      value: 0.54
+      value: 0.62
       name: Cosine Recall@3
     - type: cosine_recall@5
-      value: 0.6
+      value: 0.65
       name: Cosine Recall@5
     - type: cosine_recall@10
-      value: 0.68
+      value: 0.72
       name: Cosine Recall@10
     - type: cosine_ndcg@10
-      value: 0.5219456964879438
+      value: 0.547217901995397
       name: Cosine Ndcg@10
     - type: cosine_mrr@10
-      value: 0.4853333333333334
+      value: 0.5098571428571428
       name: Cosine Mrr@10
     - type: cosine_map@100
-      value: 0.47424546661452993
+      value: 0.4872849044614519
       name: Cosine Map@100
   - task:
       type: nano-beir
@@ -171,49 +171,49 @@ model-index:
       type: NanoBEIR_mean
     metrics:
     - type: cosine_accuracy@1
-      value: 0.33999999999999997
+      value: 0.36
       name: Cosine Accuracy@1
     - type: cosine_accuracy@3
-      value: 0.5900000000000001
+      value: 0.63
       name: Cosine Accuracy@3
     - type: cosine_accuracy@5
-      value: 0.6599999999999999
+      value: 0.68
       name: Cosine Accuracy@5
     - type: cosine_accuracy@10
-      value: 0.73
+      value: 0.77
       name: Cosine Accuracy@10
     - type: cosine_precision@1
-      value: 0.33999999999999997
+      value: 0.36
       name: Cosine Precision@1
     - type: cosine_precision@3
-      value: 0.19999999999999998
+      value: 0.21000000000000002
       name: Cosine Precision@3
     - type: cosine_precision@5
-      value: 0.136
+      value: 0.138
       name: Cosine Precision@5
     - type: cosine_precision@10
-      value: 0.07500000000000001
+      value: 0.07900000000000001
       name: Cosine Precision@10
     - type: cosine_recall@1
-      value: 0.32499999999999996
+      value: 0.345
       name: Cosine Recall@1
     - type: cosine_recall@3
-      value: 0.5800000000000001
+      value: 0.61
       name: Cosine Recall@3
     - type: cosine_recall@5
-      value: 0.6499999999999999
+      value: 0.665
       name: Cosine Recall@5
     - type: cosine_recall@10
-      value: 0.72
+      value: 0.75
       name: Cosine Recall@10
     - type: cosine_ndcg@10
-      value: 0.5317713148534957
+      value: 0.5579483562708394
       name: Cosine Ndcg@10
     - type: cosine_mrr@10
-      value: 0.47775
+      value: 0.5058730158730158
       name: Cosine Mrr@10
     - type: cosine_map@100
-      value: 0.47798026426828644
+      value: 0.4991837540403291
       name: Cosine Map@100
 ---
 
@@ -278,9 +278,9 @@ print(embeddings.shape)
 # Get the similarity scores for the embeddings
 similarities = model.similarity(embeddings, embeddings)
 print(similarities)
-# tensor([[1.0000, 0.7584, 0.1067],
-#         [0.7584, 1.0000, 0.0503],
-#         [0.1067, 0.0503, 1.0000]])
+# tensor([[1.0000, 0.7041, 0.1992],
+#         [0.7041, 1.0000, 0.0598],
+#         [0.1992, 0.0598, 1.0000]])
 ```
 
 <!--
@@ -318,21 +318,21 @@ You can finetune this model on your own dataset.
 
 | Metric              | NanoMSMARCO | NanoNQ     |
 |:--------------------|:------------|:-----------|
-| cosine_accuracy@1   | 0.3         | 0.38       |
-| cosine_accuracy@3   | 0.62        | 0.56       |
-| cosine_accuracy@5   | 0.7         | 0.62       |
-| cosine_accuracy@10  | 0.76        | 0.7        |
-| cosine_precision@1  | 0.3         | 0.38       |
-| cosine_precision@3  | 0.2067      | 0.1933     |
-| cosine_precision@5  | 0.14        | 0.132      |
-| cosine_precision@10 | 0.076       | 0.074      |
-| cosine_recall@1     | 0.3         | 0.35       |
-| cosine_recall@3     | 0.62        | 0.54       |
-| cosine_recall@5     | 0.7         | 0.6        |
-| cosine_recall@10    | 0.76        | 0.68       |
-| **cosine_ndcg@10**  | **0.5416**  | **0.5219** |
-| cosine_mrr@10       | 0.4702      | 0.4853     |
-| cosine_map@100      | 0.4817      | 0.4742     |
+| cosine_accuracy@1   | 0.36        | 0.36       |
+| cosine_accuracy@3   | 0.6         | 0.66       |
+| cosine_accuracy@5   | 0.68        | 0.68       |
+| cosine_accuracy@10  | 0.78        | 0.76       |
+| cosine_precision@1  | 0.36        | 0.36       |
+| cosine_precision@3  | 0.2         | 0.22       |
+| cosine_precision@5  | 0.136       | 0.14       |
+| cosine_precision@10 | 0.078       | 0.08       |
+| cosine_recall@1     | 0.36        | 0.33       |
+| cosine_recall@3     | 0.6         | 0.62       |
+| cosine_recall@5     | 0.68        | 0.65       |
+| cosine_recall@10    | 0.78        | 0.72       |
+| **cosine_ndcg@10**  | **0.5687**  | **0.5472** |
+| cosine_mrr@10       | 0.5019      | 0.5099     |
+| cosine_map@100      | 0.5111      | 0.4873     |
 
 #### Nano BEIR
 
@@ -350,21 +350,21 @@ You can finetune this model on your own dataset.
 
 | Metric              | Value      |
 |:--------------------|:-----------|
-| cosine_accuracy@1   | 0.34       |
-| cosine_accuracy@3   | 0.59       |
-| cosine_accuracy@5   | 0.66       |
-| cosine_accuracy@10  | 0.73       |
-| cosine_precision@1  | 0.34       |
-| cosine_precision@3  | 0.2        |
-| cosine_precision@5  | 0.136      |
-| cosine_precision@10 | 0.075      |
-| cosine_recall@1     | 0.325      |
-| cosine_recall@3     | 0.58       |
-| cosine_recall@5     | 0.65       |
-| cosine_recall@10    | 0.72       |
-| **cosine_ndcg@10**  | **0.5318** |
-| cosine_mrr@10       | 0.4778     |
-| cosine_map@100      | 0.478      |
+| cosine_accuracy@1   | 0.36       |
+| cosine_accuracy@3   | 0.63       |
+| cosine_accuracy@5   | 0.68       |
+| cosine_accuracy@10  | 0.77       |
+| cosine_precision@1  | 0.36       |
+| cosine_precision@3  | 0.21       |
+| cosine_precision@5  | 0.138      |
+| cosine_precision@10 | 0.079      |
+| cosine_recall@1     | 0.345      |
+| cosine_recall@3     | 0.61       |
+| cosine_recall@5     | 0.665      |
+| cosine_recall@10    | 0.75       |
+| **cosine_ndcg@10**  | **0.5579** |
+| cosine_mrr@10       | 0.5059     |
+| cosine_map@100      | 0.4992     |
 
 <!--
 ## Bias, Risks and Limitations
@@ -438,8 +438,8 @@ You can finetune this model on your own dataset.
 - `eval_strategy`: steps
 - `per_device_train_batch_size`: 128
 - `per_device_eval_batch_size`: 128
-- `learning_rate`: 2e-05
-- `weight_decay`: 0.0001
+- `learning_rate`: 1e-06
+- `weight_decay`: 0.001
 - `max_steps`: 3000
 - `warmup_ratio`: 0.1
 - `fp16`: True
@@ -467,8 +467,8 @@ You can finetune this model on your own dataset.
 - `gradient_accumulation_steps`: 1
 - `eval_accumulation_steps`: None
 - `torch_empty_cache_steps`: None
-- `learning_rate`: 2e-05
-- `weight_decay`: 0.0001
+- `learning_rate`: 1e-06
+- `weight_decay`: 0.001
 - `adam_beta1`: 0.9
 - `adam_beta2`: 0.999
 - `adam_epsilon`: 1e-08
@@ -582,18 +582,18 @@ You can finetune this model on your own dataset.
 | Epoch  | Step | Training Loss | Validation Loss | NanoMSMARCO_cosine_ndcg@10 | NanoNQ_cosine_ndcg@10 | NanoBEIR_mean_cosine_ndcg@10 |
 |:------:|:----:|:-------------:|:---------------:|:--------------------------:|:---------------------:|:----------------------------:|
 | 0      | 0    | -             | 3.6614          | 0.6259                     | 0.6583                | 0.6421                       |
-| 0.3556 | 250  | 2.1172        | 0.4652          | 0.5662                     | 0.5558                | 0.5610                       |
-| 0.7112 | 500  | 0.5877        | 0.4272          | 0.5926                     | 0.5289                | 0.5607                       |
-| 1.0669 | 750  | 0.5436        | 0.4134          | 0.5476                     | 0.5478                | 0.5477                       |
-| 1.4225 | 1000 | 0.5214        | 0.4041          | 0.5461                     | 0.5451                | 0.5456                       |
-| 1.7781 | 1250 | 0.5119        | 0.3994          | 0.5316                     | 0.5377                | 0.5347                       |
-| 2.1337 | 1500 | 0.5006        | 0.3957          | 0.5357                     | 0.5305                | 0.5331                       |
-| 2.4893 | 1750 | 0.494         | 0.3934          | 0.5444                     | 0.5319                | 0.5382                       |
-| 2.8450 | 2000 | 0.4886        | 0.3906          | 0.5322                     | 0.5246                | 0.5284                       |
-| 3.2006 | 2250 | 0.4839        | 0.3889          | 0.5413                     | 0.5307                | 0.5360                       |
-| 3.5562 | 2500 | 0.4795        | 0.3881          | 0.5490                     | 0.5214                | 0.5352                       |
-| 3.9118 | 2750 | 0.479         | 0.3872          | 0.5413                     | 0.5219                | 0.5316                       |
-| 4.2674 | 3000 | 0.4765        | 0.3869          | 0.5416                     | 0.5219                | 0.5318                       |
+| 0.3556 | 250  | 3.8825        | 3.4013          | 0.6200                     | 0.6575                | 0.6388                       |
+| 0.7112 | 500  | 3.3083        | 2.1977          | 0.6287                     | 0.6387                | 0.6337                       |
+| 1.0669 | 750  | 1.7439        | 0.6392          | 0.5543                     | 0.5530                | 0.5537                       |
+| 1.4225 | 1000 | 0.8977        | 0.5267          | 0.5526                     | 0.5274                | 0.5400                       |
+| 1.7781 | 1250 | 0.7869        | 0.5083          | 0.5426                     | 0.5390                | 0.5408                       |
+| 2.1337 | 1500 | 0.7442        | 0.4991          | 0.5412                     | 0.5482                | 0.5447                       |
+| 2.4893 | 1750 | 0.7213        | 0.4941          | 0.5553                     | 0.5484                | 0.5518                       |
+| 2.8450 | 2000 | 0.7054        | 0.4872          | 0.5635                     | 0.5506                | 0.5571                       |
+| 3.2006 | 2250 | 0.6943        | 0.4863          | 0.5696                     | 0.5503                | 0.5599                       |
+| 3.5562 | 2500 | 0.6864        | 0.4839          | 0.5681                     | 0.5472                | 0.5576                       |
+| 3.9118 | 2750 | 0.6851        | 0.4832          | 0.5687                     | 0.5472                | 0.5579                       |
+| 4.2674 | 3000 | 0.6825        | 0.4825          | 0.5687                     | 0.5472                | 0.5579                       |
 
 
 ### Framework Versions