Add new SentenceTransformer model

README.md

@@ -7,7 +7,7 @@ tags:
 - generated_from_trainer
 - dataset_size:90000
 - loss:MultipleNegativesRankingLoss
-base_model: sentence-transformers/all-MiniLM-L6-v2
+base_model: sentence-transformers/all-MiniLM-L12-v2
 widget:
 - source_sentence: what is the maximum i can contribute to a traditional ira
   sentences:
@@ -121,7 +121,7 @@ metrics:
 - cosine_mrr@10
 - cosine_map@100
 model-index:
-- name: SentenceTransformer based on sentence-transformers/all-MiniLM-L6-v2
+- name: SentenceTransformer based on sentence-transformers/all-MiniLM-L12-v2
   results:
   - task:
       type: information-retrieval
@@ -131,49 +131,49 @@ model-index:
       type: NanoMSMARCO
     metrics:
     - type: cosine_accuracy@1
-      value: 0.3
+      value: 0.34
       name: Cosine Accuracy@1
     - type: cosine_accuracy@3
-      value: 0.58
+      value: 0.56
       name: Cosine Accuracy@3
     - type: cosine_accuracy@5
-      value: 0.6
+      value: 0.64
       name: Cosine Accuracy@5
     - type: cosine_accuracy@10
-      value: 0.7
+      value: 0.74
       name: Cosine Accuracy@10
     - type: cosine_precision@1
-      value: 0.3
+      value: 0.34
       name: Cosine Precision@1
     - type: cosine_precision@3
-      value: 0.19333333333333333
+      value: 0.18666666666666668
       name: Cosine Precision@3
     - type: cosine_precision@5
-      value: 0.12
+      value: 0.128
       name: Cosine Precision@5
     - type: cosine_precision@10
-      value: 0.07
+      value: 0.07400000000000001
       name: Cosine Precision@10
     - type: cosine_recall@1
-      value: 0.3
+      value: 0.34
       name: Cosine Recall@1
     - type: cosine_recall@3
-      value: 0.58
+      value: 0.56
       name: Cosine Recall@3
     - type: cosine_recall@5
-      value: 0.6
+      value: 0.64
       name: Cosine Recall@5
     - type: cosine_recall@10
-      value: 0.7
+      value: 0.74
       name: Cosine Recall@10
     - type: cosine_ndcg@10
-      value: 0.5101349275378135
+      value: 0.5374407455925777
       name: Cosine Ndcg@10
     - type: cosine_mrr@10
-      value: 0.44874603174603167
+      value: 0.47276190476190466
       name: Cosine Mrr@10
     - type: cosine_map@100
-      value: 0.4606212065533731
+      value: 0.48561801675925215
       name: Cosine Map@100
   - task:
       type: information-retrieval
@@ -183,49 +183,49 @@ model-index:
       type: NanoNQ
     metrics:
     - type: cosine_accuracy@1
-      value: 0.28
+      value: 0.4
       name: Cosine Accuracy@1
     - type: cosine_accuracy@3
-      value: 0.48
+      value: 0.5
       name: Cosine Accuracy@3
     - type: cosine_accuracy@5
-      value: 0.52
+      value: 0.56
       name: Cosine Accuracy@5
     - type: cosine_accuracy@10
-      value: 0.58
+      value: 0.7
       name: Cosine Accuracy@10
     - type: cosine_precision@1
-      value: 0.28
+      value: 0.4
       name: Cosine Precision@1
     - type: cosine_precision@3
-      value: 0.16
+      value: 0.1733333333333333
       name: Cosine Precision@3
     - type: cosine_precision@5
-      value: 0.11200000000000002
+      value: 0.12
       name: Cosine Precision@5
     - type: cosine_precision@10
-      value: 0.06400000000000002
+      value: 0.07400000000000001
       name: Cosine Precision@10
     - type: cosine_recall@1
-      value: 0.26
+      value: 0.37
       name: Cosine Recall@1
     - type: cosine_recall@3
-      value: 0.44
+      value: 0.47
       name: Cosine Recall@3
     - type: cosine_recall@5
-      value: 0.5
+      value: 0.54
       name: Cosine Recall@5
     - type: cosine_recall@10
-      value: 0.57
+      value: 0.66
       name: Cosine Recall@10
     - type: cosine_ndcg@10
-      value: 0.42522283720602283
+      value: 0.5075548380780193
       name: Cosine Ndcg@10
     - type: cosine_mrr@10
-      value: 0.3893888888888889
+      value: 0.47743650793650794
       name: Cosine Mrr@10
     - type: cosine_map@100
-      value: 0.38784914899138384
+      value: 0.46217171679843827
       name: Cosine Map@100
   - task:
       type: nano-beir
@@ -235,61 +235,61 @@ model-index:
       type: NanoBEIR_mean
     metrics:
     - type: cosine_accuracy@1
-      value: 0.29000000000000004
+      value: 0.37
       name: Cosine Accuracy@1
     - type: cosine_accuracy@3
       value: 0.53
       name: Cosine Accuracy@3
     - type: cosine_accuracy@5
-      value: 0.56
+      value: 0.6000000000000001
       name: Cosine Accuracy@5
     - type: cosine_accuracy@10
-      value: 0.6399999999999999
+      value: 0.72
       name: Cosine Accuracy@10
     - type: cosine_precision@1
-      value: 0.29000000000000004
+      value: 0.37
       name: Cosine Precision@1
     - type: cosine_precision@3
-      value: 0.17666666666666667
+      value: 0.18
       name: Cosine Precision@3
     - type: cosine_precision@5
-      value: 0.116
+      value: 0.124
       name: Cosine Precision@5
     - type: cosine_precision@10
-      value: 0.067
+      value: 0.07400000000000001
       name: Cosine Precision@10
     - type: cosine_recall@1
-      value: 0.28
+      value: 0.355
       name: Cosine Recall@1
     - type: cosine_recall@3
-      value: 0.51
+      value: 0.515
       name: Cosine Recall@3
     - type: cosine_recall@5
-      value: 0.55
+      value: 0.5900000000000001
       name: Cosine Recall@5
     - type: cosine_recall@10
-      value: 0.635
+      value: 0.7
       name: Cosine Recall@10
     - type: cosine_ndcg@10
-      value: 0.4676788823719182
+      value: 0.5224977918352984
       name: Cosine Ndcg@10
     - type: cosine_mrr@10
-      value: 0.4190674603174603
+      value: 0.4750992063492063
       name: Cosine Mrr@10
     - type: cosine_map@100
-      value: 0.42423517777237846
+      value: 0.4738948667788452
       name: Cosine Map@100
 ---
 
-# SentenceTransformer based on sentence-transformers/all-MiniLM-L6-v2
+# SentenceTransformer based on sentence-transformers/all-MiniLM-L12-v2
 
-This is a [sentence-transformers](https://www.SBERT.net) model finetuned from [sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2). It maps sentences & paragraphs to a 384-dimensional dense vector space and can be used for semantic textual similarity, semantic search, paraphrase mining, text classification, clustering, and more.
+This is a [sentence-transformers](https://www.SBERT.net) model finetuned from [sentence-transformers/all-MiniLM-L12-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L12-v2). It maps sentences & paragraphs to a 384-dimensional dense vector space and can be used for semantic textual similarity, semantic search, paraphrase mining, text classification, clustering, and more.
 
 ## Model Details
 
 ### Model Description
 - **Model Type:** Sentence Transformer
-- **Base model:** [sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) <!-- at revision c9745ed1d9f207416be6d2e6f8de32d1f16199bf -->
+- **Base model:** [sentence-transformers/all-MiniLM-L12-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L12-v2) <!-- at revision 936af83a2ecce5fe87a09109ff5cbcefe073173a -->
 - **Maximum Sequence Length:** 128 tokens
 - **Output Dimensionality:** 384 dimensions
 - **Similarity Function:** Cosine Similarity
@@ -342,9 +342,9 @@ print(embeddings.shape)
 # Get the similarity scores for the embeddings
 similarities = model.similarity(embeddings, embeddings)
 print(similarities)
-# tensor([[0.9999, 0.4254, 0.6155],
-#         [0.4254, 0.9999, 0.2541],
-#         [0.6155, 0.2541, 1.0000]])
+# tensor([[1.0000, 0.5519, 0.4707],
+#         [0.5519, 1.0001, 0.0831],
+#         [0.4707, 0.0831, 1.0000]])
 ```
 
 <!--
@@ -382,21 +382,21 @@ You can finetune this model on your own dataset.
 
 | Metric              | NanoMSMARCO | NanoNQ     |
 |:--------------------|:------------|:-----------|
-| cosine_accuracy@1   | 0.3         | 0.28       |
-| cosine_accuracy@3   | 0.58        | 0.48       |
-| cosine_accuracy@5   | 0.6         | 0.52       |
-| cosine_accuracy@10  | 0.7         | 0.58       |
-| cosine_precision@1  | 0.3         | 0.28       |
-| cosine_precision@3  | 0.1933      | 0.16       |
-| cosine_precision@5  | 0.12        | 0.112      |
-| cosine_precision@10 | 0.07        | 0.064      |
-| cosine_recall@1     | 0.3         | 0.26       |
-| cosine_recall@3     | 0.58        | 0.44       |
-| cosine_recall@5     | 0.6         | 0.5        |
-| cosine_recall@10    | 0.7         | 0.57       |
-| **cosine_ndcg@10**  | **0.5101**  | **0.4252** |
-| cosine_mrr@10       | 0.4487      | 0.3894     |
-| cosine_map@100      | 0.4606      | 0.3878     |
+| cosine_accuracy@1   | 0.34        | 0.4        |
+| cosine_accuracy@3   | 0.56        | 0.5        |
+| cosine_accuracy@5   | 0.64        | 0.56       |
+| cosine_accuracy@10  | 0.74        | 0.7        |
+| cosine_precision@1  | 0.34        | 0.4        |
+| cosine_precision@3  | 0.1867      | 0.1733     |
+| cosine_precision@5  | 0.128       | 0.12       |
+| cosine_precision@10 | 0.074       | 0.074      |
+| cosine_recall@1     | 0.34        | 0.37       |
+| cosine_recall@3     | 0.56        | 0.47       |
+| cosine_recall@5     | 0.64        | 0.54       |
+| cosine_recall@10    | 0.74        | 0.66       |
+| **cosine_ndcg@10**  | **0.5374**  | **0.5076** |
+| cosine_mrr@10       | 0.4728      | 0.4774     |
+| cosine_map@100      | 0.4856      | 0.4622     |
 
 #### Nano BEIR
 
@@ -414,21 +414,21 @@ You can finetune this model on your own dataset.
 
 | Metric              | Value      |
 |:--------------------|:-----------|
-| cosine_accuracy@1   | 0.29       |
+| cosine_accuracy@1   | 0.37       |
 | cosine_accuracy@3   | 0.53       |
-| cosine_accuracy@5   | 0.56       |
-| cosine_accuracy@10  | 0.64       |
-| cosine_precision@1  | 0.29       |
-| cosine_precision@3  | 0.1767     |
-| cosine_precision@5  | 0.116      |
-| cosine_precision@10 | 0.067      |
-| cosine_recall@1     | 0.28       |
-| cosine_recall@3     | 0.51       |
-| cosine_recall@5     | 0.55       |
-| cosine_recall@10    | 0.635      |
-| **cosine_ndcg@10**  | **0.4677** |
-| cosine_mrr@10       | 0.4191     |
-| cosine_map@100      | 0.4242     |
+| cosine_accuracy@5   | 0.6        |
+| cosine_accuracy@10  | 0.72       |
+| cosine_precision@1  | 0.37       |
+| cosine_precision@3  | 0.18       |
+| cosine_precision@5  | 0.124      |
+| cosine_precision@10 | 0.074      |
+| cosine_recall@1     | 0.355      |
+| cosine_recall@3     | 0.515      |
+| cosine_recall@5     | 0.59       |
+| cosine_recall@10    | 0.7        |
+| **cosine_ndcg@10**  | **0.5225** |
+| cosine_mrr@10       | 0.4751     |
+| cosine_map@100      | 0.4739     |
 
 <!--
 ## Bias, Risks and Limitations
@@ -502,9 +502,9 @@ You can finetune this model on your own dataset.
 - `eval_strategy`: steps
 - `per_device_train_batch_size`: 128
 - `per_device_eval_batch_size`: 128
-- `learning_rate`: 0.0001
+- `learning_rate`: 8e-05
 - `weight_decay`: 0.005
-- `max_steps`: 2250
+- `max_steps`: 1687
 - `warmup_ratio`: 0.1
 - `fp16`: True
 - `dataloader_drop_last`: True
@@ -531,14 +531,14 @@ You can finetune this model on your own dataset.
 - `gradient_accumulation_steps`: 1
 - `eval_accumulation_steps`: None
 - `torch_empty_cache_steps`: None
-- `learning_rate`: 0.0001
+- `learning_rate`: 8e-05
 - `weight_decay`: 0.005
 - `adam_beta1`: 0.9
 - `adam_beta2`: 0.999
 - `adam_epsilon`: 1e-08
 - `max_grad_norm`: 1.0
 - `num_train_epochs`: 3.0
-- `max_steps`: 2250
+- `max_steps`: 1687
 - `lr_scheduler_type`: linear
 - `lr_scheduler_kwargs`: {}
 - `warmup_ratio`: 0.1
@@ -645,16 +645,13 @@ You can finetune this model on your own dataset.
 ### Training Logs
 | Epoch  | Step | Training Loss | Validation Loss | NanoMSMARCO_cosine_ndcg@10 | NanoNQ_cosine_ndcg@10 | NanoBEIR_mean_cosine_ndcg@10 |
 |:------:|:----:|:-------------:|:---------------:|:--------------------------:|:---------------------:|:----------------------------:|
-| 0      | 0    | -             | 1.2542          | 0.5540                     | 0.5931                | 0.5735                       |
-| 0.3556 | 250  | 1.164         | 0.9510          | 0.4905                     | 0.4669                | 0.4787                       |
-| 0.7112 | 500  | 1.0862        | 0.9271          | 0.5075                     | 0.4222                | 0.4648                       |
-| 1.0669 | 750  | 1.0451        | 0.9133          | 0.5116                     | 0.4339                | 0.4727                       |
-| 1.4225 | 1000 | 0.9486        | 0.9157          | 0.4948                     | 0.4660                | 0.4804                       |
-| 1.7781 | 1250 | 0.9464        | 0.9022          | 0.5148                     | 0.4456                | 0.4802                       |
-| 2.1337 | 1500 | 0.9034        | 0.9062          | 0.5134                     | 0.4155                | 0.4645                       |
-| 2.4893 | 1750 | 0.851         | 0.9029          | 0.5317                     | 0.4236                | 0.4776                       |
-| 2.8450 | 2000 | 0.8506        | 0.8996          | 0.5259                     | 0.4389                | 0.4824                       |
-| 3.2006 | 2250 | 0.8204        | 0.9039          | 0.5101                     | 0.4252                | 0.4677                       |
+| 0      | 0    | -             | 1.2073          | 0.5887                     | 0.5786                | 0.5836                       |
+| 0.3556 | 250  | 1.0978        | 0.9124          | 0.4990                     | 0.5159                | 0.5075                       |
+| 0.7112 | 500  | 1.0307        | 0.8904          | 0.5523                     | 0.4790                | 0.5157                       |
+| 1.0669 | 750  | 0.993         | 0.8816          | 0.5518                     | 0.4577                | 0.5047                       |
+| 1.4225 | 1000 | 0.8908        | 0.8770          | 0.5529                     | 0.5130                | 0.5329                       |
+| 1.7781 | 1250 | 0.8857        | 0.8703          | 0.5288                     | 0.5212                | 0.5250                       |
+| 2.1337 | 1500 | 0.8526        | 0.8677          | 0.5374                     | 0.5076                | 0.5225                       |
 
 
 ### Framework Versions