Training in progress, step 12000

1_Pooling/config.json

@@ -1,7 +1,7 @@
 {
-    "word_embedding_dimension": 768,
-    "pooling_mode_cls_token": false,
-    "pooling_mode_mean_tokens": true,
+    "word_embedding_dimension": 512,
+    "pooling_mode_cls_token": true,
+    "pooling_mode_mean_tokens": false,
     "pooling_mode_max_tokens": false,
     "pooling_mode_mean_sqrt_len_tokens": false,
     "pooling_mode_weightedmean_tokens": false,

Information-Retrieval_evaluation_val_results.csv

@@ -10,3 +10,4 @@ epoch,steps,cosine-Accuracy@1,cosine-Accuracy@3,cosine-Accuracy@5,cosine-Precisi
 -1,-1,0.8288,0.899775,0.925775,0.8288,0.8288,0.29992499999999994,0.899775,0.185155,0.925775,0.8288,0.8661879166666627,0.8703450396825356,0.8910978019383597,0.8726020537429935
 -1,-1,0.826575,0.900725,0.92805,0.826575,0.826575,0.30024166666666663,0.900725,0.18561000000000002,0.92805,0.826575,0.8658308333333287,0.8701137103174557,0.891705546917102,0.8723575730144177
 -1,-1,0.82585,0.902175,0.930075,0.82585,0.82585,0.30072499999999996,0.902175,0.186015,0.930075,0.82585,0.8661279166666617,0.8703281448412645,0.8922105025555344,0.8724788643099791
+-1,-1,0.00065,0.7986,0.880825,0.00065,0.00065,0.26619999999999994,0.7986,0.17616500000000002,0.880825,0.00065,0.288667083333407,0.2951483234127803,0.45147470340355694,0.2980051496600344

README.md

@@ -5,124 +5,51 @@ tags:
 - feature-extraction
 - dense
 - generated_from_trainer
-- dataset_size:359997
+- dataset_size:100000
 - loss:MultipleNegativesRankingLoss
-base_model: google/embeddinggemma-300m
+base_model: prajjwal1/bert-small
 widget:
-- source_sentence: When do you use Ms. or Mrs.? Is one for a married woman and one
-    for one that's not married? Which one is for what?
+- source_sentence: How do I polish my English skills?
   sentences:
-  - When do you use Ms. or Mrs.? Is one for a married woman and one for one that's
-    not married? Which one is for what?
-  - Nations that do/does otherwise? Which one do I use?
-  - Why don't bikes have a gear indicator?
-- source_sentence: Which ointment is applied to the face of UFC fighters at the commencement
-    of a bout? What does it do?
+  - How can we polish English skills?
+  - Why should I move to Israel as a Jew?
+  - What are vitamins responsible for?
+- source_sentence: Can I use the Kozuka Gothic Pro font as a font-face on my web site?
   sentences:
-  - How can I save a Snapchat video that others posted?
-  - Which ointment is applied to the face of UFC fighters at the commencement of a
-    bout? What does it do?
-  - How do I get the body of a UFC Fighter?
-- source_sentence: Do you love the life you live?
+  - Can I use the Kozuka Gothic Pro font as a font-face on my web site?
+  - Why are Google, Facebook, YouTube and other social networking sites banned in
+    China?
+  - What font is used in Bloomberg Terminal?
+- source_sentence: Is Quora the best Q&A site?
   sentences:
-  - Can I do shoulder and triceps workout on same day? What other combinations like
-    this can I do?
-  - Do you love the life you're living?
-  - Where can you find an online TI-84 calculator?
-- source_sentence: Ordered food on Swiggy 3 days ago.After accepting my money, said
-    no more on Menu! When if ever will I atleast get refund in cr card a/c?
+  - What was the best Quora question ever?
+  - Is Quora the best inquiry site?
+  - Where do I buy Oway hair products online?
+- source_sentence: How can I customize my walking speed on Google Maps?
   sentences:
-  - Is getting to the Tel Aviv airport to catch a 5:30 AM flight very expensive?
-  - How do I die and make it look like an accident?
-  - Ordered food on Swiggy 3 days ago.After accepting my money, said no more on Menu!
-    When if ever will I atleast get refund in cr card a/c?
-- source_sentence: How do you earn money on Quora?
+  - How do I bring back Google maps icon in my home screen?
+  - How many pages are there in all the Harry Potter books combined?
+  - How can I customize my walking speed on Google Maps?
+- source_sentence: DId something exist before the Big Bang?
   sentences:
-  - What is a cheap healthy diet I can keep the same and eat every day?
-  - What are some things new employees should know going into their first day at Maximus?
-  - What is the best way to make money on Quora?
+  - How can I improve my memory problem?
+  - Where can I buy Fairy Tail Manga?
+  - Is there a scientific name for what existed before the Big Bang?
 pipeline_tag: sentence-similarity
 library_name: sentence-transformers
-metrics:
-- cosine_accuracy@1
-- cosine_accuracy@3
-- cosine_accuracy@5
-- cosine_precision@1
-- cosine_precision@3
-- cosine_precision@5
-- cosine_recall@1
-- cosine_recall@3
-- cosine_recall@5
-- cosine_ndcg@10
-- cosine_mrr@1
-- cosine_mrr@5
-- cosine_mrr@10
-- cosine_map@100
-model-index:
-- name: SentenceTransformer based on google/embeddinggemma-300m
-  results:
-  - task:
-      type: information-retrieval
-      name: Information Retrieval
-    dataset:
-      name: val
-      type: val
-    metrics:
-    - type: cosine_accuracy@1
-      value: 0.00065
-      name: Cosine Accuracy@1
-    - type: cosine_accuracy@3
-      value: 0.7986
-      name: Cosine Accuracy@3
-    - type: cosine_accuracy@5
-      value: 0.880825
-      name: Cosine Accuracy@5
-    - type: cosine_precision@1
-      value: 0.00065
-      name: Cosine Precision@1
-    - type: cosine_precision@3
-      value: 0.26619999999999994
-      name: Cosine Precision@3
-    - type: cosine_precision@5
-      value: 0.17616500000000002
-      name: Cosine Precision@5
-    - type: cosine_recall@1
-      value: 0.00065
-      name: Cosine Recall@1
-    - type: cosine_recall@3
-      value: 0.7986
-      name: Cosine Recall@3
-    - type: cosine_recall@5
-      value: 0.880825
-      name: Cosine Recall@5
-    - type: cosine_ndcg@10
-      value: 0.45147470340355694
-      name: Cosine Ndcg@10
-    - type: cosine_mrr@1
-      value: 0.00065
-      name: Cosine Mrr@1
-    - type: cosine_mrr@5
-      value: 0.288667083333407
-      name: Cosine Mrr@5
-    - type: cosine_mrr@10
-      value: 0.2951483234127803
-      name: Cosine Mrr@10
-    - type: cosine_map@100
-      value: 0.2980051496600344
-      name: Cosine Map@100
 ---
 
-# SentenceTransformer based on google/embeddinggemma-300m
+# SentenceTransformer based on prajjwal1/bert-small
 
-This is a [sentence-transformers](https://www.SBERT.net) model finetuned from [google/embeddinggemma-300m](https://huggingface.co/google/embeddinggemma-300m). It maps sentences & paragraphs to a 768-dimensional dense vector space and can be used for semantic textual similarity, semantic search, paraphrase mining, text classification, clustering, and more.
+This is a [sentence-transformers](https://www.SBERT.net) model finetuned from [prajjwal1/bert-small](https://huggingface.co/prajjwal1/bert-small). It maps sentences & paragraphs to a 512-dimensional dense vector space and can be used for semantic textual similarity, semantic search, paraphrase mining, text classification, clustering, and more.
 
 ## Model Details
 
 ### Model Description
 - **Model Type:** Sentence Transformer
-- **Base model:** [google/embeddinggemma-300m](https://huggingface.co/google/embeddinggemma-300m) <!-- at revision 57c266a740f537b4dc058e1b0cda161fd15afa75 -->
+- **Base model:** [prajjwal1/bert-small](https://huggingface.co/prajjwal1/bert-small) <!-- at revision 0ec5f86f27c1a77d704439db5e01c307ea11b9d4 -->
 - **Maximum Sequence Length:** 128 tokens
-- **Output Dimensionality:** 768 dimensions
+- **Output Dimensionality:** 512 dimensions
 - **Similarity Function:** Cosine Similarity
 <!-- - **Training Dataset:** Unknown -->
 <!-- - **Language:** Unknown -->
@@ -138,11 +65,8 @@ This is a [sentence-transformers](https://www.SBERT.net) model finetuned from [g
 
 ```
 SentenceTransformer(
-  (0): Transformer({'max_seq_length': 128, 'do_lower_case': False, 'architecture': 'Gemma3TextModel'})
-  (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False, 'pooling_mode_weightedmean_tokens': False, 'pooling_mode_lasttoken': False, 'include_prompt': True})
-  (2): Dense({'in_features': 768, 'out_features': 3072, 'bias': False, 'activation_function': 'torch.nn.modules.linear.Identity'})
-  (3): Dense({'in_features': 3072, 'out_features': 768, 'bias': False, 'activation_function': 'torch.nn.modules.linear.Identity'})
-  (4): Normalize()
+  (0): Transformer({'max_seq_length': 128, 'do_lower_case': False, 'architecture': 'BertModel'})
+  (1): Pooling({'word_embedding_dimension': 512, 'pooling_mode_cls_token': True, 'pooling_mode_mean_tokens': False, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False, 'pooling_mode_weightedmean_tokens': False, 'pooling_mode_lasttoken': False, 'include_prompt': True})
 )
 ```
 
@@ -161,25 +85,23 @@ Then you can load this model and run inference.
 from sentence_transformers import SentenceTransformer
 
 # Download from the 🤗 Hub
-model = SentenceTransformer("redis/model-a-baseline")
+model = SentenceTransformer("sentence_transformers_model_id")
 # Run inference
-queries = [
-    "How do you earn money on Quora?",
+sentences = [
+    'DId something exist before the Big Bang?',
+    'Is there a scientific name for what existed before the Big Bang?',
+    'Where can I buy Fairy Tail Manga?',
 ]
-documents = [
-    'What is the best way to make money on Quora?',
-    'What are some things new employees should know going into their first day at Maximus?',
-    'What is a cheap healthy diet I can keep the same and eat every day?',
-]
-query_embeddings = model.encode_query(queries)
-document_embeddings = model.encode_document(documents)
-print(query_embeddings.shape, document_embeddings.shape)
-# [1, 768] [3, 768]
+embeddings = model.encode(sentences)
+print(embeddings.shape)
+# [3, 512]
 
 # Get the similarity scores for the embeddings
-similarities = model.similarity(query_embeddings, document_embeddings)
+similarities = model.similarity(embeddings, embeddings)
 print(similarities)
-# tensor([[ 0.9829, -0.0321, -0.0412]])
+# tensor([[ 1.0000,  0.7596, -0.0398],
+#         [ 0.7596,  1.0000, -0.0308],
+#         [-0.0398, -0.0308,  1.0000]])
 ```
 
 <!--
@@ -206,32 +128,6 @@ You can finetune this model on your own dataset.
 *List how the model may foreseeably be misused and address what users ought not to do with the model.*
 -->
 
-## Evaluation
-
-### Metrics
-
-#### Information Retrieval
-
-* Dataset: `val`
-* Evaluated with [<code>InformationRetrievalEvaluator</code>](https://sbert.net/docs/package_reference/sentence_transformer/evaluation.html#sentence_transformers.evaluation.InformationRetrievalEvaluator)
-
-| Metric             | Value      |
-|:-------------------|:-----------|
-| cosine_accuracy@1  | 0.0006     |
-| cosine_accuracy@3  | 0.7986     |
-| cosine_accuracy@5  | 0.8808     |
-| cosine_precision@1 | 0.0006     |
-| cosine_precision@3 | 0.2662     |
-| cosine_precision@5 | 0.1762     |
-| cosine_recall@1    | 0.0006     |
-| cosine_recall@3    | 0.7986     |
-| cosine_recall@5    | 0.8808     |
-| **cosine_ndcg@10** | **0.4515** |
-| cosine_mrr@1       | 0.0006     |
-| cosine_mrr@5       | 0.2887     |
-| cosine_mrr@10      | 0.2951     |
-| cosine_map@100     | 0.298      |
-
 <!--
 ## Bias, Risks and Limitations
 
@@ -250,49 +146,23 @@ You can finetune this model on your own dataset.
 
 #### Unnamed Dataset
 
-* Size: 359,997 training samples
-* Columns: <code>anchor</code>, <code>positive</code>, and <code>negative</code>
-* Approximate statistics based on the first 1000 samples:
-  |         | anchor                                                                            | positive                                                                          | negative                                                                           |
-  |:--------|:----------------------------------------------------------------------------------|:----------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------|
-  | type    | string                                                                            | string                                                                            | string                                                                             |
-  | details | <ul><li>min: 4 tokens</li><li>mean: 15.38 tokens</li><li>max: 60 tokens</li></ul> | <ul><li>min: 4 tokens</li><li>mean: 15.48 tokens</li><li>max: 60 tokens</li></ul> | <ul><li>min: 5 tokens</li><li>mean: 16.77 tokens</li><li>max: 128 tokens</li></ul> |
-* Samples:
-  | anchor                                                                          | positive                                                                                             | negative                                                                                                                 |
-  |:--------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------|
-  | <code>Shall I upgrade my iPhone 5s to iOS 10 final version?</code>              | <code>Should I upgrade an iPhone 5s to iOS 10?</code>                                                | <code>Whether extension of CA-articleship is to be served at same firm/company?</code>                                   |
-  | <code>Is Donald Trump really going to be the president of United States?</code> | <code>Do you think Donald Trump could conceivably be the next President of the United States?</code> | <code>Since solid carbon dioxide is dry ice and incredibly cold, why doesn't it have an effect on global warming?</code> |
-  | <code>What are real tips to improve work life balance?</code>                   | <code>What are the best ways to create a work life balance?</code>                                   | <code>How do you open a briefcase combination lock without the combination?</code>                                       |
-* Loss: [<code>MultipleNegativesRankingLoss</code>](https://sbert.net/docs/package_reference/sentence_transformer/losses.html#multiplenegativesrankingloss) with these parameters:
-  ```json
-  {
-      "scale": 7.0,
-      "similarity_fct": "cos_sim",
-      "gather_across_devices": false
-  }
-  ```
-
-### Evaluation Dataset
-
-#### Unnamed Dataset
-
-* Size: 40,000 evaluation samples
-* Columns: <code>anchor</code>, <code>positive</code>, and <code>negative</code>
+* Size: 100,000 training samples
+* Columns: <code>sentence_0</code>, <code>sentence_1</code>, and <code>sentence_2</code>
 * Approximate statistics based on the first 1000 samples:
-  |         | anchor                                                                            | positive                                                                          | negative                                                                          |
-  |:--------|:----------------------------------------------------------------------------------|:----------------------------------------------------------------------------------|:----------------------------------------------------------------------------------|
-  | type    | string                                                                            | string                                                                            | string                                                                            |
-  | details | <ul><li>min: 6 tokens</li><li>mean: 15.61 tokens</li><li>max: 70 tokens</li></ul> | <ul><li>min: 6 tokens</li><li>mean: 15.66 tokens</li><li>max: 70 tokens</li></ul> | <ul><li>min: 6 tokens</li><li>mean: 16.81 tokens</li><li>max: 77 tokens</li></ul> |
+  |         | sentence_0                                                                        | sentence_1                                                                       | sentence_2                                                                         |
+  |:--------|:----------------------------------------------------------------------------------|:---------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------|
+  | type    | string                                                                            | string                                                                           | string                                                                             |
+  | details | <ul><li>min: 3 tokens</li><li>mean: 15.53 tokens</li><li>max: 59 tokens</li></ul> | <ul><li>min: 3 tokens</li><li>mean: 15.5 tokens</li><li>max: 59 tokens</li></ul> | <ul><li>min: 6 tokens</li><li>mean: 16.87 tokens</li><li>max: 128 tokens</li></ul> |
 * Samples:
-  | anchor                                                                                                            | positive                                                                                                          | negative                                                                                                                                  |
-  |:------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------|
-  | <code>Why were feathered dinosaur fossils only found in the last 20 years?</code>                                 | <code>Why were feathered dinosaur fossils only found in the last 20 years?</code>                                 | <code>Why are only few people aware that many dinosaurs had feathers?</code>                                                              |
-  | <code>If FOX News is the conservative news station, which cable news network is for liberals/progressives?</code> | <code>If FOX News is the conservative news station, which cable news network is for liberals/progressives?</code> | <code>How much did Fox News and conservative leaning media networks stoke the anger that contributed to Donald Trump's popularity?</code> |
-  | <code>How can guys last longer during sex?</code>                                                                 | <code>How do I last longer in sex?</code>                                                                         | <code>What is a permanent solution for rough and puffy hair?</code>                                                                       |
+  | sentence_0                                                                              | sentence_1                                                                              | sentence_2                                                             |
+  |:----------------------------------------------------------------------------------------|:----------------------------------------------------------------------------------------|:-----------------------------------------------------------------------|
+  | <code>Is there visitor entry facility in Jaipur airport. How much is the ticket?</code> | <code>Is there visitor entry facility in Jaipur airport. How much is the ticket?</code> | <code>How much is the airport tax in bogota?</code>                    |
+  | <code>Which concept is more important: good planning or hard work?</code>               | <code>Which concept is more important: good planning or hard work?</code>               | <code>What is important in life: luck or hard work?</code>             |
+  | <code>What is the most efficient way to make money?</code>                              | <code>How can I make my money make money?</code>                                        | <code>What can one learn about Quantum Mechanics in 10 minutes?</code> |
 * Loss: [<code>MultipleNegativesRankingLoss</code>](https://sbert.net/docs/package_reference/sentence_transformer/losses.html#multiplenegativesrankingloss) with these parameters:
   ```json
   {
-      "scale": 7.0,
+      "scale": 20.0,
       "similarity_fct": "cos_sim",
       "gather_across_devices": false
   }
@@ -301,30 +171,17 @@ You can finetune this model on your own dataset.
 ### Training Hyperparameters
 #### Non-Default Hyperparameters
 
-- `eval_strategy`: steps
 - `per_device_train_batch_size`: 64
 - `per_device_eval_batch_size`: 64
-- `learning_rate`: 2e-05
-- `weight_decay`: 0.0001
-- `max_steps`: 5000
-- `warmup_ratio`: 0.1
 - `fp16`: True
-- `dataloader_drop_last`: True
-- `dataloader_num_workers`: 1
-- `dataloader_prefetch_factor`: 1
-- `load_best_model_at_end`: True
-- `optim`: adamw_torch
-- `ddp_find_unused_parameters`: False
-- `push_to_hub`: True
-- `hub_model_id`: redis/model-a-baseline
-- `eval_on_start`: True
+- `multi_dataset_batch_sampler`: round_robin
 
 #### All Hyperparameters
 <details><summary>Click to expand</summary>
 
 - `overwrite_output_dir`: False
 - `do_predict`: False
-- `eval_strategy`: steps
+- `eval_strategy`: no
 - `prediction_loss_only`: True
 - `per_device_train_batch_size`: 64
 - `per_device_eval_batch_size`: 64
@@ -333,17 +190,17 @@ You can finetune this model on your own dataset.
 - `gradient_accumulation_steps`: 1
 - `eval_accumulation_steps`: None
 - `torch_empty_cache_steps`: None
-- `learning_rate`: 2e-05
-- `weight_decay`: 0.0001
+- `learning_rate`: 5e-05
+- `weight_decay`: 0.0
 - `adam_beta1`: 0.9
 - `adam_beta2`: 0.999
 - `adam_epsilon`: 1e-08
-- `max_grad_norm`: 1.0
-- `num_train_epochs`: 3.0
-- `max_steps`: 5000
+- `max_grad_norm`: 1
+- `num_train_epochs`: 3
+- `max_steps`: -1
 - `lr_scheduler_type`: linear
 - `lr_scheduler_kwargs`: {}
-- `warmup_ratio`: 0.1
+- `warmup_ratio`: 0.0
 - `warmup_steps`: 0
 - `log_level`: passive
 - `log_level_replica`: warning
@@ -371,14 +228,14 @@ You can finetune this model on your own dataset.
 - `tpu_num_cores`: None
 - `tpu_metrics_debug`: False
 - `debug`: []
-- `dataloader_drop_last`: True
-- `dataloader_num_workers`: 1
-- `dataloader_prefetch_factor`: 1
+- `dataloader_drop_last`: False
+- `dataloader_num_workers`: 0
+- `dataloader_prefetch_factor`: None
 - `past_index`: -1
 - `disable_tqdm`: False
 - `remove_unused_columns`: True
 - `label_names`: None
-- `load_best_model_at_end`: True
+- `load_best_model_at_end`: False
 - `ignore_data_skip`: False
 - `fsdp`: []
 - `fsdp_min_num_params`: 0
@@ -388,23 +245,23 @@ You can finetune this model on your own dataset.
 - `parallelism_config`: None
 - `deepspeed`: None
 - `label_smoothing_factor`: 0.0
-- `optim`: adamw_torch
+- `optim`: adamw_torch_fused
 - `optim_args`: None
 - `adafactor`: False
 - `group_by_length`: False
 - `length_column_name`: length
 - `project`: huggingface
 - `trackio_space_id`: trackio
-- `ddp_find_unused_parameters`: False
+- `ddp_find_unused_parameters`: None
 - `ddp_bucket_cap_mb`: None
 - `ddp_broadcast_buffers`: False
 - `dataloader_pin_memory`: True
 - `dataloader_persistent_workers`: False
 - `skip_memory_metrics`: True
 - `use_legacy_prediction_loop`: False
-- `push_to_hub`: True
+- `push_to_hub`: False
 - `resume_from_checkpoint`: None
-- `hub_model_id`: redis/model-a-baseline
+- `hub_model_id`: None
 - `hub_strategy`: every_save
 - `hub_private_repo`: None
 - `hub_always_push`: False
@@ -431,43 +288,31 @@ You can finetune this model on your own dataset.
 - `neftune_noise_alpha`: None
 - `optim_target_modules`: None
 - `batch_eval_metrics`: False
-- `eval_on_start`: True
+- `eval_on_start`: False
 - `use_liger_kernel`: False
 - `liger_kernel_config`: None
 - `eval_use_gather_object`: False
 - `average_tokens_across_devices`: True
 - `prompts`: None
 - `batch_sampler`: batch_sampler
-- `multi_dataset_batch_sampler`: proportional
+- `multi_dataset_batch_sampler`: round_robin
 - `router_mapping`: {}
 - `learning_rate_mapping`: {}
 
 </details>
 
 ### Training Logs
-| Epoch  | Step | Training Loss | Validation Loss | val_cosine_ndcg@10 |
-|:------:|:----:|:-------------:|:---------------:|:------------------:|
-| 0      | 0    | -             | 0.9732          | 0.8903             |
-| 0.0445 | 250  | 0.3014        | 0.2430          | 0.8863             |
-| 0.0889 | 500  | 0.2442        | 0.2489          | 0.8875             |
-| 0.1334 | 750  | 0.2504        | nan             | 0.0293             |
-| 0.1778 | 1000 | 0.2492        | nan             | 0.8871             |
-| 0.2223 | 1250 | 0.2676        | nan             | 0.4514             |
-| 0.2667 | 1500 | 0.2804        | nan             | 0.4514             |
-| 0.3112 | 1750 | 0.2782        | nan             | 0.4514             |
-| 0.3556 | 2000 | 0.28          | nan             | 0.4515             |
-| 0.4001 | 2250 | 0.2814        | nan             | 0.4515             |
-| 0.4445 | 2500 | 0.2758        | nan             | 0.4515             |
-| 0.4890 | 2750 | 0.2792        | nan             | 0.4515             |
-| 0.5334 | 3000 | 0.2807        | nan             | 0.4515             |
-| 0.5779 | 3250 | 0.282         | nan             | 0.4515             |
-| 0.6223 | 3500 | 0.2785        | nan             | 0.4515             |
-| 0.6668 | 3750 | 0.28          | nan             | 0.4515             |
-| 0.7112 | 4000 | 0.2813        | nan             | 0.4515             |
-| 0.7557 | 4250 | 0.2823        | nan             | 0.4515             |
-| 0.8001 | 4500 | 0.2791        | nan             | 0.4515             |
-| 0.8446 | 4750 | 0.2789        | nan             | 0.4515             |
-| 0.8890 | 5000 | 0.282         | nan             | 0.4515             |
+| Epoch  | Step | Training Loss |
+|:------:|:----:|:-------------:|
+| 0.3199 | 500  | 0.2284        |
+| 0.6398 | 1000 | 0.0571        |
+| 0.9597 | 1500 | 0.0486        |
+| 1.2796 | 2000 | 0.0378        |
+| 1.5995 | 2500 | 0.0367        |
+| 1.9194 | 3000 | 0.0338        |
+| 2.2393 | 3500 | 0.0327        |
+| 2.5592 | 4000 | 0.0285        |
+| 2.8791 | 4500 | 0.0285        |
 
 
 ### Framework Versions

config.json

@@ -1,60 +1,25 @@
 {
-  "_sliding_window_pattern": 6,
   "architectures": [
-    "Gemma3TextModel"
+    "BertModel"
   ],
-  "attention_bias": false,
-  "attention_dropout": 0.0,
-  "attn_logit_softcapping": null,
-  "bos_token_id": 2,
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
   "dtype": "float32",
-  "eos_token_id": 1,
-  "final_logit_softcapping": null,
-  "head_dim": 256,
-  "hidden_activation": "gelu_pytorch_tanh",
-  "hidden_size": 768,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 384,
   "initializer_range": 0.02,
-  "intermediate_size": 1152,
-  "layer_types": [
-    "sliding_attention",
-    "sliding_attention",
-    "sliding_attention",
-    "sliding_attention",
-    "sliding_attention",
-    "full_attention",
-    "sliding_attention",
-    "sliding_attention",
-    "sliding_attention",
-    "sliding_attention",
-    "sliding_attention",
-    "full_attention",
-    "sliding_attention",
-    "sliding_attention",
-    "sliding_attention",
-    "sliding_attention",
-    "sliding_attention",
-    "full_attention",
-    "sliding_attention",
-    "sliding_attention",
-    "sliding_attention",
-    "sliding_attention",
-    "sliding_attention",
-    "full_attention"
-  ],
-  "max_position_embeddings": 2048,
-  "model_type": "gemma3_text",
-  "num_attention_heads": 3,
-  "num_hidden_layers": 24,
-  "num_key_value_heads": 1,
+  "intermediate_size": 1536,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 6,
   "pad_token_id": 0,
-  "query_pre_attn_scalar": 256,
-  "rms_norm_eps": 1e-06,
-  "rope_local_base_freq": 10000.0,
-  "rope_scaling": null,
-  "rope_theta": 1000000.0,
-  "sliding_window": 257,
+  "position_embedding_type": "absolute",
   "transformers_version": "4.57.3",
-  "use_bidirectional_attention": true,
+  "type_vocab_size": 2,
   "use_cache": true,
-  "vocab_size": 262144
+  "vocab_size": 30522
 }

config_sentence_transformers.json

@@ -6,20 +6,8 @@
     "pytorch": "2.9.1+cu128"
   },
   "prompts": {
-    "query": "task: search result | query: ",
-    "document": "title: none | text: ",
-    "BitextMining": "task: search result | query: ",
-    "Clustering": "task: clustering | query: ",
-    "Classification": "task: classification | query: ",
-    "InstructionRetrieval": "task: code retrieval | query: ",
-    "MultilabelClassification": "task: classification | query: ",
-    "PairClassification": "task: sentence similarity | query: ",
-    "Reranking": "task: search result | query: ",
-    "Retrieval": "task: search result | query: ",
-    "Retrieval-query": "task: search result | query: ",
-    "Retrieval-document": "title: none | text: ",
-    "STS": "task: sentence similarity | query: ",
-    "Summarization": "task: summarization | query: "
+    "query": "",
+    "document": ""
   },
   "default_prompt_name": null,
   "similarity_fn_name": "cosine"

eval/Information-Retrieval_evaluation_val_results.csv

@@ -818,3 +818,52 @@ epoch,steps,cosine-Accuracy@1,cosine-Accuracy@3,cosine-Accuracy@5,cosine-Precisi
 0.8001422475106685,4500,0.00065,0.7986,0.880825,0.00065,0.00065,0.26619999999999994,0.7986,0.17616500000000002,0.880825,0.00065,0.288667083333407,0.2951483234127803,0.45147470340355694,0.2980051496600344
 0.8445945945945946,4750,0.00065,0.7986,0.880825,0.00065,0.00065,0.26619999999999994,0.7986,0.17616500000000002,0.880825,0.00065,0.288667083333407,0.2951483234127803,0.45147470340355694,0.2980051496600344
 0.8890469416785206,5000,0.00065,0.7986,0.880825,0.00065,0.00065,0.26619999999999994,0.7986,0.17616500000000002,0.880825,0.00065,0.288667083333407,0.2951483234127803,0.45147470340355694,0.2980051496600344
+0,0,0.83175,0.903975,0.928825,0.83175,0.83175,0.3013249999999999,0.903975,0.185765,0.928825,0.83175,0.8696037499999943,0.8736409623015803,0.8942125677045817,0.8758019595132278
+0.17780938833570412,250,0.8298,0.901375,0.9263,0.8298,0.8298,0.30045833333333327,0.901375,0.18526,0.9263,0.8298,0.8672679166666633,0.871277857142852,0.8917710037156674,0.8735051857910221
+0.35561877667140823,500,0.829425,0.9009,0.924475,0.829425,0.829425,0.30029999999999996,0.9009,0.184895,0.924475,0.829425,0.8666179166666625,0.8707307837301547,0.8910529119995637,0.8730143621228817
+0.5334281650071123,750,0.829525,0.9008,0.924625,0.829525,0.829525,0.3002666666666666,0.9008,0.18492500000000003,0.924625,0.829525,0.8665862499999952,0.8706709126984068,0.8909881032602778,0.8729676890722908
+0.7112375533428165,1000,0.82945,0.9004,0.9248,0.82945,0.82945,0.3001333333333333,0.9004,0.18495999999999999,0.9248,0.82945,0.8666070833333291,0.8706706746031699,0.8909814879426163,0.8729971704579322
+0.8890469416785206,1250,0.828825,0.89945,0.9248,0.828825,0.828825,0.2998166666666666,0.89945,0.18496000000000004,0.9248,0.828825,0.8660399999999953,0.8701422023809464,0.8906698796585799,0.872440351264829
+1.0668563300142249,1500,0.828325,0.899425,0.925175,0.828325,0.828325,0.29980833333333323,0.899425,0.18503500000000003,0.925175,0.828325,0.8658666666666628,0.8699482440476135,0.8906064784084294,0.8722410872423765
+1.2446657183499288,1750,0.829,0.899575,0.9252,0.829,0.829,0.2998583333333333,0.899575,0.18504000000000007,0.9252,0.829,0.8661520833333282,0.8703107043650742,0.8909704705166476,0.8726021051327861
+1.422475106685633,2000,0.828125,0.899475,0.924875,0.828125,0.828125,0.299825,0.899475,0.18497500000000006,0.924875,0.828125,0.8655224999999951,0.8696983928571376,0.8904182641298983,0.8720281345427299
+1.600284495021337,2250,0.828125,0.8996,0.9256,0.828125,0.828125,0.29986666666666667,0.8996,0.18512000000000003,0.9256,0.828125,0.8658949999999959,0.8700507043650736,0.8909141261388864,0.8723249622424765
+1.7780938833570412,2500,0.8284,0.89975,0.92525,0.8284,0.8284,0.2999166666666666,0.89975,0.18505000000000002,0.92525,0.8284,0.8659133333333284,0.8701298015872949,0.8909826453993707,0.8724114097634953
+1.9559032716927454,2750,0.8284,0.899625,0.92565,0.8284,0.8284,0.29987499999999995,0.899625,0.18513000000000004,0.92565,0.8284,0.8658908333333284,0.8700691567460257,0.8909817662470981,0.8723517690381816
+2.1337126600284497,3000,0.82815,0.899175,0.925225,0.82815,0.82815,0.299725,0.899175,0.18504500000000002,0.925225,0.82815,0.8656183333333282,0.8698032936507867,0.8906445442171722,0.8721210376144165
+2.3115220483641536,3250,0.828725,0.89985,0.925725,0.828725,0.828725,0.29994999999999994,0.89985,0.185145,0.925725,0.828725,0.8661716666666621,0.8703707539682477,0.8912340046527859,0.872660098072118
+2.4893314366998576,3500,0.828275,0.8994,0.925475,0.828275,0.828275,0.29979999999999996,0.8994,0.18509500000000004,0.925475,0.828275,0.8656466666666616,0.8698591170634867,0.8907648949594115,0.87219205242926
+2.667140825035562,3750,0.8283,0.89875,0.9253,0.8283,0.8283,0.2995833333333333,0.89875,0.18506000000000003,0.9253,0.8283,0.8656337499999947,0.8699330952380903,0.8909264696431981,0.8722177974435448
+2.844950213371266,4000,0.82835,0.8994,0.92555,0.82835,0.82835,0.2997999999999999,0.8994,0.18511,0.92555,0.82835,0.8658199999999949,0.8701035515872961,0.8910883504036013,0.8723985335287471
+3.0227596017069702,4250,0.82825,0.8996,0.9263,0.82825,0.82825,0.29986666666666667,0.8996,0.18526,0.9263,0.82825,0.8659529166666614,0.8701048511904707,0.8910734609029339,0.8724252490135679
+3.200568990042674,4500,0.8277,0.899675,0.9266,0.8277,0.8277,0.29989166666666667,0.899675,0.18532000000000004,0.9266,0.8277,0.8657191666666609,0.8698755753968195,0.8909808746095252,0.8721680969534944
+3.3783783783783785,4750,0.82785,0.89975,0.92645,0.82785,0.82785,0.29991666666666666,0.89975,0.18529000000000004,0.92645,0.82785,0.8657941666666611,0.8699176984126923,0.8909228072077822,0.8722519939581889
+3.5561877667140824,5000,0.8281,0.899425,0.9263,0.8281,0.8281,0.2998083333333333,0.899425,0.18526000000000006,0.9263,0.8281,0.8657945833333274,0.8699932638888828,0.8910608683159033,0.8723111498240054
+3.733997155049787,5250,0.828,0.89925,0.926475,0.828,0.828,0.29974999999999996,0.89925,0.18529500000000002,0.926475,0.828,0.865916249999995,0.8701195634920589,0.8912279872499286,0.8724104724821727
+3.9118065433854907,5500,0.828375,0.899675,0.9263,0.828375,0.828375,0.2998916666666666,0.899675,0.18526,0.9263,0.828375,0.8660124999999942,0.870194007936502,0.8911974141937964,0.872517170678436
+4.089615931721195,5750,0.82815,0.8999,0.926625,0.82815,0.82815,0.2999666666666666,0.8999,0.18532500000000002,0.926625,0.82815,0.8661049999999954,0.8702374900793608,0.8912279100182797,0.8725786528407918
+4.2674253200568995,6000,0.827875,0.89955,0.9264,0.827875,0.827875,0.2998499999999999,0.89955,0.18528000000000003,0.9264,0.827875,0.8658287499999946,0.8700195932539642,0.8911118555195989,0.8723399615981953
+4.445234708392603,6250,0.828075,0.9002,0.926475,0.828075,0.828075,0.30006666666666665,0.9002,0.18529500000000002,0.926475,0.828075,0.866062083333328,0.8703042460317422,0.8914192358779078,0.8726139945581133
+4.623044096728307,6500,0.82815,0.9002,0.9268,0.82815,0.82815,0.3000666666666666,0.9002,0.18536000000000002,0.9268,0.82815,0.8661799999999951,0.8703642460317412,0.8914384292682349,0.8726787629715127
+4.800853485064011,6750,0.82815,0.900375,0.926775,0.82815,0.82815,0.300125,0.900375,0.18535500000000002,0.926775,0.82815,0.8662083333333293,0.870440942460312,0.8916079401774063,0.8727270914730314
+4.978662873399715,7000,0.828175,0.900325,0.9269,0.828175,0.828175,0.30010833333333325,0.900325,0.18538000000000002,0.9269,0.828175,0.866215833333329,0.8703757638888837,0.8914345869423077,0.8727073322059316
+5.15647226173542,7250,0.828125,0.9003,0.927,0.828125,0.828125,0.3000999999999999,0.9003,0.1854,0.927,0.828125,0.8662441666666615,0.8704108333333277,0.8914839027483926,0.8727429892775405
+5.334281650071124,7500,0.82835,0.9005,0.9271,0.82835,0.82835,0.30016666666666664,0.9005,0.18542000000000003,0.9271,0.82835,0.8664845833333292,0.870711855158725,0.8918662873245725,0.8729953191357858
+5.512091038406828,7750,0.828025,0.9003,0.9266,0.828025,0.828025,0.3001,0.9003,0.18532000000000004,0.9266,0.828025,0.866135833333329,0.8704414285714227,0.8916759547261216,0.8727139575100616
+5.689900426742532,8000,0.828,0.90035,0.92695,0.828,0.828,0.3001166666666666,0.90035,0.18539000000000003,0.92695,0.828,0.8662174999999955,0.8704429861111054,0.8916180877077008,0.8727453218360391
+5.867709815078236,8250,0.827825,0.900425,0.92715,0.827825,0.827825,0.3001416666666666,0.900425,0.18543,0.92715,0.827825,0.8662124999999951,0.8704174801587243,0.8915933814560918,0.8727258542799402
+6.0455192034139404,8500,0.828,0.900575,0.927025,0.828,0.828,0.3001916666666666,0.900575,0.18540500000000001,0.927025,0.828,0.8662441666666627,0.8704399900793606,0.8915614826595735,0.8727748598025921
+6.223328591749644,8750,0.8279,0.90035,0.92635,0.8279,0.8279,0.3001166666666666,0.90035,0.18527000000000002,0.92635,0.8279,0.8660104166666617,0.8702868551587247,0.8914372805079627,0.8726103535171617
+6.401137980085348,9000,0.8281,0.900375,0.92675,0.8281,0.8281,0.300125,0.900375,0.18535000000000004,0.92675,0.8281,0.866213333333329,0.8704590277777723,0.8916245077673864,0.8727687702395993
+6.578947368421053,9250,0.82765,0.9004,0.927175,0.82765,0.82765,0.3001333333333333,0.9004,0.18543500000000002,0.927175,0.82765,0.8661099999999957,0.8703067559523755,0.8915377883823766,0.872607454746048
+6.756756756756757,9500,0.82765,0.900575,0.926725,0.82765,0.82765,0.3001916666666666,0.900575,0.185345,0.926725,0.82765,0.8659654166666624,0.8702720138888835,0.8915565456654305,0.8725550336297769
+6.934566145092461,9750,0.827975,0.900575,0.927175,0.827975,0.827975,0.3001916666666666,0.900575,0.18543500000000004,0.927175,0.827975,0.8662537499999953,0.8704973809523754,0.8917383936706421,0.8727792761081553
+7.112375533428165,10000,0.82785,0.9008,0.927025,0.82785,0.82785,0.3002666666666666,0.9008,0.18540500000000001,0.927025,0.82785,0.8662083333333285,0.8704846130952314,0.8917715435604863,0.8727515648994657
+7.290184921763869,10250,0.8278,0.9008,0.927075,0.8278,0.8278,0.3002666666666666,0.9008,0.185415,0.927075,0.8278,0.866154166666662,0.8704102777777718,0.8916861480256432,0.8726937691969677
+7.467994310099574,10500,0.827825,0.90055,0.927225,0.827825,0.827825,0.3001833333333333,0.90055,0.185445,0.927225,0.827825,0.8661895833333294,0.8704195734126929,0.8916986664790416,0.8727028796088485
+7.6458036984352775,10750,0.827875,0.9006,0.927225,0.827875,0.827875,0.30019999999999997,0.9006,0.18544500000000003,0.927225,0.827875,0.8662158333333293,0.8704450793650736,0.8917174408022205,0.8727262878002741
+7.823613086770981,11000,0.827775,0.900525,0.927225,0.827775,0.827775,0.30017499999999997,0.900525,0.18544500000000003,0.927225,0.827775,0.8661854166666623,0.8703987599206292,0.891665903804086,0.8726874745211679
+8.001422475106686,11250,0.827775,0.90045,0.9273,0.827775,0.827775,0.30015,0.90045,0.18546000000000004,0.9273,0.827775,0.8661808333333287,0.8704179563492007,0.8917319897154259,0.87269052120409
+8.17923186344239,11500,0.827775,0.90055,0.9272,0.827775,0.827775,0.30018333333333325,0.90055,0.18544000000000005,0.9272,0.827775,0.8661820833333286,0.8704046329365026,0.8916719767589327,0.8726965525672397
+8.357041251778094,11750,0.827725,0.900575,0.9273,0.827725,0.827725,0.3001916666666666,0.900575,0.18546,0.9273,0.827725,0.8661516666666623,0.8703490972222169,0.8916142319118482,0.8726456646970746
+8.534850640113799,12000,0.827675,0.9006,0.9272,0.827675,0.827675,0.3001999999999999,0.9006,0.18544000000000002,0.9272,0.827675,0.8661058333333287,0.8703261011904707,0.8916124422761306,0.8726181110807445

final_metrics.json

@@ -1,16 +1,16 @@
 {
-  "val_cosine_accuracy@1": 0.82585,
-  "val_cosine_accuracy@3": 0.902175,
-  "val_cosine_accuracy@5": 0.930075,
-  "val_cosine_precision@1": 0.82585,
-  "val_cosine_precision@3": 0.30072499999999996,
-  "val_cosine_precision@5": 0.186015,
-  "val_cosine_recall@1": 0.82585,
-  "val_cosine_recall@3": 0.902175,
-  "val_cosine_recall@5": 0.930075,
-  "val_cosine_ndcg@10": 0.8922105025555344,
-  "val_cosine_mrr@1": 0.82585,
-  "val_cosine_mrr@5": 0.8661279166666617,
-  "val_cosine_mrr@10": 0.8703281448412645,
-  "val_cosine_map@100": 0.8724788643099791
+  "val_cosine_accuracy@1": 0.00065,
+  "val_cosine_accuracy@3": 0.7986,
+  "val_cosine_accuracy@5": 0.880825,
+  "val_cosine_precision@1": 0.00065,
+  "val_cosine_precision@3": 0.26619999999999994,
+  "val_cosine_precision@5": 0.17616500000000002,
+  "val_cosine_recall@1": 0.00065,
+  "val_cosine_recall@3": 0.7986,
+  "val_cosine_recall@5": 0.880825,
+  "val_cosine_ndcg@10": 0.45147470340355694,
+  "val_cosine_mrr@1": 0.00065,
+  "val_cosine_mrr@5": 0.288667083333407,
+  "val_cosine_mrr@10": 0.2951483234127803,
+  "val_cosine_map@100": 0.2980051496600344
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9f9a47024db2f08e7f0c48f2a85ed40a9741f18dc45828790d6b88202b309171
-size 1211486072
+oid sha256:362192cb39fe2be97a0273cd2772690d7be8a1fb0c2c9b2d356da5acdf07ee67
+size 90864192

modules.json

@@ -10,23 +10,5 @@
     "name": "1",
     "path": "1_Pooling",
     "type": "sentence_transformers.models.Pooling"
-  },
-  {
-    "idx": 2,
-    "name": "2",
-    "path": "2_Dense",
-    "type": "sentence_transformers.models.Dense"
-  },
-  {
-    "idx": 3,
-    "name": "3",
-    "path": "3_Dense",
-    "type": "sentence_transformers.models.Dense"
-  },
-  {
-    "idx": 4,
-    "name": "4",
-    "path": "4_Normalize",
-    "type": "sentence_transformers.models.Normalize"
   }
 ]

special_tokens_map.json

@@ -1,30 +1,34 @@
 {
-  "boi_token": "<start_of_image>",
-  "bos_token": {
-    "content": "<bos>",
+  "cls_token": {
+    "content": "[CLS]",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
-  "eoi_token": "<end_of_image>",
-  "eos_token": {
-    "content": "<eos>",
+  "mask_token": {
+    "content": "[MASK]",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
-  "image_token": "<image_soft_token>",
   "pad_token": {
-    "content": "<pad>",
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "unk_token": {
-    "content": "<unk>",
+    "content": "[UNK]",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d7ae06c2d5423018713138be1af08ead828be6d5aec3dcaf4004830d73339843
-size 33385261
+oid sha256:2fc687b11de0bc1b3d8348f92e3b49ef1089a621506c7661fbf3248fcd54947e
+size 711649

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eda78a9adeb8ee61251aa1b4dd9dd8932131463e584245b900135bee7256aae0
+oid sha256:5c95d57155864ceb9f17b5db3a6c9a6215d12f49d260b60a68d720a3bc86ddd0
 size 6161