{
  "attn_implementation": "flash_attention_2",
  "batch_sampler": "no_duplicates_fast",
  "bf16": true,
  "bf16_full_eval": true,
  "config_path": "configs/training_args/ablation/a8m-ft-prefix-none-from-full-prefix-none-a25m-ftdata.yaml",
  "data_config_path": "configs/data_args/ablation/ft-bekko-embedding-v1-a8m-prefix-none-a25m-ftdata.yaml",
  "data_summary": [
    "ruri_v3_ft-auto-wiki-qa-nemotron: train=154,956 eval=0",
    "ruri_v3_ft-jaquad: train=31,253 eval=0",
    "ruri_v3_ft-jqara: train=19,515 eval=0",
    "ruri_v3_ft-jsquad: train=61,659 eval=0",
    "ruri_v3_ft-miracl: train=25,905 eval=0",
    "ruri_v3_ft-mkqa: train=30,310 eval=0",
    "ruri_v3_ft-mr-tydi: train=18,200 eval=0",
    "ruri_v3_ft-nli: train=222,429 eval=0",
    "ruri_v3_ft-quiz-no-mori: train=37,437 eval=0",
    "ruri_v3_ft-quiz-works: train=30,631 eval=0",
    "bge_m3_ft-dureader-len-0-500+len-500-1000+len-1000-2000+len-2000-3000+len-3000-4000+len-4000-5000+len-5000-6000+len-6000-7000+len-7000-inf: train=80,416 eval=0",
    "bge_m3_ft-hotpotqa-len-0-500+len-500-1000+len-1000-2000+len-2000-3000+len-3000-4000+len-4000-5000+len-5000-6000+len-6000-7000+len-7000-inf: train=84,516 eval=0",
    "bge_m3_ft-miracl-len-0-500+len-500-1000+len-1000-2000+len-2000-3000+len-3000-4000+len-4000-5000+len-5000-6000+len-6000-7000+len-7000-inf: train=40,203 eval=0",
    "bge_m3_ft-mr-tydi-len-0-500+len-500-1000+len-1000-2000+len-2000-3000+len-3000-4000+len-4000-5000+len-5000-6000+len-6000-7000+len-7000-inf: train=48,729 eval=0",
    "bge_m3_ft-squad-len-0-500+len-500-1000+len-1000-2000+len-2000-3000+len-3000-4000+len-4000-5000+len-5000-6000+len-6000-7000+len-7000-inf: train=87,599 eval=0",
    "bge_m3_ft-other-len-0-500+len-500-1000+len-1000-2000+len-2000-3000+len-3000-4000+len-4000-5000+len-5000-6000+len-6000-7000+len-7000-inf: train=500 eval=0",
    "bge_m3_ft-msmarco-len-0-500+len-500-1000+len-1000-2000+len-2000-3000+len-3000-4000+len-4000-5000+len-5000-6000+len-6000-7000+len-7000-inf: train=200,000 eval=0",
    "bge_m3_ft-mmarco_chinese-len-0-500: train=80,000 eval=0",
    "bge_m3_ft-ATEC-len-0-500: train=11,325 eval=0",
    "bge_m3_ft-BQ-len-0-500: train=12,518 eval=0",
    "bge_m3_ft-LCQMC-len-0-500: train=10,000 eval=0",
    "bge_m3_ft-PAWSX-len-0-500: train=9,900 eval=0",
    "bekko_embedding_hard_negatives-wikipedia_hard_negatives_english: train=250,000 eval=0",
    "bekko_embedding_hard_negatives-additional-query-doc: train=80,000 eval=0",
    "bekko_embedding_hard_negatives-additional-doc-doc: train=40,000 eval=0",
    "bekko_embedding_hard_negatives-wikipedia_hard_negatives_long_docs_en: train=12,000 eval=0",
    "bekko_embedding_hard_negatives-wikipedia_hard_negatives_long_docs_ar: train=2,000 eval=0",
    "bekko_embedding_hard_negatives-wikipedia_hard_negatives_long_docs_de: train=2,000 eval=0",
    "bekko_embedding_hard_negatives-wikipedia_hard_negatives_long_docs_es: train=2,000 eval=0",
    "bekko_embedding_hard_negatives-wikipedia_hard_negatives_long_docs_fr: train=2,000 eval=0",
    "bekko_embedding_hard_negatives-wikipedia_hard_negatives_long_docs_it: train=2,000 eval=0",
    "bekko_embedding_hard_negatives-wikipedia_hard_negatives_long_docs_ja: train=2,000 eval=0",
    "bekko_embedding_hard_negatives-wikipedia_hard_negatives_long_docs_ko: train=2,000 eval=0",
    "bekko_embedding_hard_negatives-wikipedia_hard_negatives_long_docs_pt: train=2,000 eval=0",
    "bekko_embedding_hard_negatives-wikipedia_hard_negatives_long_docs_ru: train=2,000 eval=0",
    "bekko_embedding_hard_negatives-wikipedia_hard_negatives_long_docs_zh: train=2,000 eval=0",
    "bekko_embedding_hard_negatives-extra-four: train=80,000 eval=0"
  ],
  "decay_ratio": null,
  "decay_steps": null,
  "eval_steps": 400,
  "evaluation_strategy": "steps",
  "fp16": false,
  "ft_base_model": "output/models/embedding_v1/mmBERT-L4H384-pruned/a8m-pretrain-full-unir-v9-prefix-none-mrl-qat/20260704_041654/final",
  "global_rope_theta": null,
  "gradient_accumulation_steps": 1,
  "learning_rate": 1e-05,
  "local_rope_theta": null,
  "logging_steps": 10,
  "loss_mini_batch_size": 32,
  "loss_mini_batch_size_by_bs": {
    "1152": 128,
    "192": 16,
    "240": 16,
    "288": 32,
    "336": 32,
    "432": 32,
    "480": 64,
    "768": 64
  },
  "losses": {
    "doc_doc": [
      {
        "klass": "CachedMultipleNegativesMaskedBidirectionalRankingLoss",
        "kwargs": {
          "debug_mask_stats": 10,
          "gather_across_devices": false,
          "include_doc_to_query_loss": true,
          "margin": 0.01,
          "temperature": 0.03
        }
      },
      {
        "klass": "QuantizationAwareLoss",
        "kwargs": {
          "quantization_precisions": [
            "float32",
            "int8",
            "ubinary"
          ],
          "quantization_weights": [
            1.0,
            0.1,
            0.1
          ]
        }
      },
      {
        "klass": "MatryoshkaLoss",
        "kwargs": {
          "matryoshka_dims": [
            384,
            256,
            128,
            64
          ],
          "matryoshka_weights": [
            1.0,
            0.3,
            0.15,
            0.1
          ]
        }
      }
    ],
    "query_doc": [
      {
        "klass": "CachedMultipleNegativesMaskedBidirectionalRankingLoss",
        "kwargs": {
          "debug_mask_stats": 10,
          "gather_across_devices": false,
          "margin": 0.01,
          "temperature": 0.03
        }
      },
      {
        "klass": "QuantizationAwareLoss",
        "kwargs": {
          "quantization_precisions": [
            "float32",
            "int8",
            "ubinary"
          ],
          "quantization_weights": [
            1.0,
            0.1,
            0.1
          ]
        }
      },
      {
        "klass": "MatryoshkaLoss",
        "kwargs": {
          "matryoshka_dims": [
            384,
            256,
            128,
            64
          ],
          "matryoshka_weights": [
            1.0,
            0.3,
            0.15,
            0.1
          ]
        }
      }
    ]
  },
  "lr_scheduler_kwargs": null,
  "lr_scheduler_type": "cosine",
  "max_seq_length": 8192,
  "max_steps": null,
  "merged_config": {
    "_config_path": "configs/training_args/ablation/a8m-ft-prefix-none-from-full-prefix-none-a25m-ftdata.yaml",
    "_data_config_path": "configs/data_args/ablation/ft-bekko-embedding-v1-a8m-prefix-none-a25m-ftdata.yaml",
    "_training_config_path": "configs/training_args/ablation/a8m-ft-prefix-none-from-full-prefix-none-a25m-ftdata.yaml",
    "data": {
      "_config_path": "/home/hotchpotch/src/github.com/hotchpotch/mmBERT-embedding-reranker/configs/data/ft-bekko-embedding-v1-a25m-20260625.yaml",
      "common": {
        "eval_limit": 0,
        "group_small_subsets_label": "other",
        "group_small_subsets_threshold": 2000,
        "group_subsets_by_prefix": true,
        "group_subsets_prefixes": [
          "__auto__"
        ],
        "items": 8,
        "limit": null,
        "loader": "bge_m3_ft",
        "neg_sampling": {
          "strategy": "mid_tail_random"
        },
        "pair_type": "query_doc",
        "path": "output/bge-m3-data-ft-raw-dataset"
      },
      "train_datasets": [
        {
          "bs_size": 1152,
          "eval_limit": 0,
          "items": 8,
          "limit": null,
          "loader": "ruri_v3_ft",
          "max_len": 512,
          "pair_type": "query_doc",
          "subsets": [
            "*"
          ]
        },
        {
          "bs_size": 1152,
          "eval_limit": 0,
          "group_small_subsets_label": "other",
          "group_small_subsets_threshold": 2000,
          "group_subsets_by_prefix": true,
          "group_subsets_prefixes": [
            "__auto__"
          ],
          "items": 8,
          "limit": null,
          "loader": "bge_m3_ft",
          "max_len": 512,
          "neg_sampling": {
            "strategy": "mid_tail_random"
          },
          "pair_type": "query_doc",
          "path": "output/bge-m3-data-ft-raw-dataset",
          "subset_suffixes": [
            "len-0-500",
            "len-500-1000",
            "len-1000-2000",
            "len-2000-3000",
            "len-3000-4000",
            "len-4000-5000",
            "len-5000-6000",
            "len-6000-7000",
            "len-7000-inf"
          ],
          "subsets": [
            "dureader",
            "hotpotqa",
            "miracl",
            "mr-tydi",
            "pubmed_qa_labeled",
            "squad"
          ]
        },
        {
          "bs_size": 1152,
          "eval_limit": 0,
          "group_small_subsets_label": "other",
          "group_small_subsets_threshold": 2000,
          "group_subsets_by_prefix": true,
          "group_subsets_prefixes": [
            "__auto__"
          ],
          "items": 8,
          "limit": 200000,
          "loader": "bge_m3_ft",
          "max_len": 512,
          "neg_sampling": {
            "strategy": "mid_tail_random"
          },
          "pair_type": "query_doc",
          "path": "output/bge-m3-data-ft-raw-dataset",
          "subset_suffixes": [
            "len-0-500",
            "len-500-1000",
            "len-1000-2000",
            "len-2000-3000",
            "len-3000-4000",
            "len-4000-5000",
            "len-5000-6000",
            "len-6000-7000",
            "len-7000-inf"
          ],
          "subsets": [
            "msmarco"
          ]
        },
        {
          "bs_size": 1152,
          "eval_limit": 0,
          "group_small_subsets_label": "other",
          "group_small_subsets_threshold": 2000,
          "group_subsets_by_prefix": true,
          "group_subsets_prefixes": [
            "__auto__"
          ],
          "items": 8,
          "limit": 80000,
          "loader": "bge_m3_ft",
          "max_len": 512,
          "neg_sampling": {
            "strategy": "mid_tail_random"
          },
          "pair_type": "query_doc",
          "path": "output/bge-m3-data-ft-raw-dataset",
          "subset_suffixes": [
            "len-0-500"
          ],
          "subsets": [
            "mmarco_chinese"
          ]
        },
        {
          "bs_size": 1152,
          "eval_limit": 0,
          "group_small_subsets_label": "other",
          "group_small_subsets_threshold": 2000,
          "group_subsets_by_prefix": true,
          "group_subsets_prefixes": [
            "__auto__"
          ],
          "items": 8,
          "limit": null,
          "loader": "bge_m3_ft",
          "max_len": 512,
          "neg_sampling": {
            "strategy": "mid_tail_random"
          },
          "pair_type": "doc_doc",
          "pair_type_by_label": {
            "ATEC": "doc_doc",
            "BQ": "doc_doc",
            "LCQMC": "doc_doc",
            "PAWSX": "doc_doc"
          },
          "path": "output/bge-m3-data-ft-raw-dataset",
          "subset_suffixes": [
            "len-0-500"
          ],
          "subsets": [
            "ATEC",
            "BQ",
            "LCQMC",
            "PAWSX"
          ]
        },
        {
          "bs_size": 1152,
          "eval_limit": 0,
          "items": 8,
          "limit": 250000,
          "loader": "bekko_embedding_hard_negatives",
          "max_len": 512,
          "name": "hotchpotch/bekko-embedding-hard-negatives",
          "pair_type": "query_doc",
          "split": "train",
          "subset": "wikipedia-hard-negatives-english",
          "subsets": [
            "wikipedia_hard_negatives_english"
          ]
        },
        {
          "bs_size": 1152,
          "eval_limit": 0,
          "items": 8,
          "limit": null,
          "loader": "bekko_embedding_hard_negatives",
          "max_len": 512,
          "merge_subsets": true,
          "name": "hotchpotch/bekko-embedding-hard-negatives",
          "pair_type": "query_doc",
          "split": "train",
          "subset": "additional-query-doc",
          "subsets": [
            "agnews",
            "gooaq",
            "natural_questions",
            "trivia_qa"
          ]
        },
        {
          "bs_size": 1152,
          "eval_limit": 0,
          "items": 8,
          "limit": null,
          "loader": "bekko_embedding_hard_negatives",
          "max_len": 512,
          "merge_subsets": true,
          "name": "hotchpotch/bekko-embedding-hard-negatives",
          "pair_type": "doc_doc",
          "split": "train",
          "subset": "additional-doc-doc",
          "subsets": [
            "all_nli",
            "coco_captions"
          ]
        },
        {
          "bs_size": 192,
          "eval_limit": 0,
          "items": 8,
          "limit": 12000,
          "loader": "bekko_embedding_hard_negatives",
          "max_len": 8192,
          "name": "hotchpotch/bekko-embedding-hard-negatives",
          "neg_sampling": {
            "head_count": 4,
            "split_at": 7,
            "strategy": "split_random"
          },
          "pair_type": "query_doc",
          "split": "train",
          "subset": "wikipedia-hard-negatives-long-docs-en-rebalanced",
          "subsets": [
            "wikipedia_hard_negatives_long_docs_en"
          ]
        },
        {
          "bs_size": 192,
          "eval_limit": 0,
          "items": 8,
          "limit": 2000,
          "loader": "bekko_embedding_hard_negatives",
          "max_len": 8192,
          "name": "hotchpotch/bekko-embedding-hard-negatives",
          "neg_sampling": {
            "head_count": 4,
            "split_at": 7,
            "strategy": "split_random"
          },
          "pair_type": "query_doc",
          "split": "train",
          "subset": "wikipedia-hard-negatives-long-docs-other-rebalanced",
          "subsets": [
            "wikipedia_hard_negatives_long_docs_ar",
            "wikipedia_hard_negatives_long_docs_de",
            "wikipedia_hard_negatives_long_docs_es",
            "wikipedia_hard_negatives_long_docs_fr",
            "wikipedia_hard_negatives_long_docs_it",
            "wikipedia_hard_negatives_long_docs_ja",
            "wikipedia_hard_negatives_long_docs_ko",
            "wikipedia_hard_negatives_long_docs_pt",
            "wikipedia_hard_negatives_long_docs_ru",
            "wikipedia_hard_negatives_long_docs_zh"
          ]
        },
        {
          "bs_size": 1152,
          "eval_limit": 0,
          "items": 8,
          "limit": null,
          "loader": "bekko_embedding_hard_negatives",
          "max_len": 512,
          "merge_subsets": true,
          "name": "hotchpotch/bekko-embedding-hard-negatives",
          "pair_type": "query_doc",
          "split": "train",
          "subset": "extra-four",
          "subsets": [
            "arxiv",
            "codesearch",
            "fineweb",
            "pubmed"
          ]
        }
      ]
    },
    "model": {
      "doc_doc_prefix_fallback_to_query": false,
      "doc_prefix": "",
      "ft_base_model": "output/models/embedding_v1/mmBERT-L4H384-pruned/a8m-pretrain-full-unir-v9-prefix-none-mrl-qat/20260704_041654/final",
      "max_seq_length": 8192,
      "name": "hotchpotch/mmBERT-L4H384-pruned",
      "query_prefix": ""
    },
    "training": {
      "batch_sampler": "no_duplicates_fast",
      "bf16": true,
      "bf16_full_eval": true,
      "cleanup_checkpoints": false,
      "dataloader_drop_last": true,
      "dataloader_num_workers": 4,
      "dataloader_persistent_workers": false,
      "dataloader_prefetch_factor": 1,
      "eval_before_train": false,
      "eval_prefix_in_dataset": true,
      "eval_steps": 400,
      "fp16": false,
      "gradient_accumulation_steps": 1,
      "learning_rate": 1e-05,
      "logging_steps": 10,
      "loss_detail_logging": {
        "enabled": true,
        "flush_every_records": 1,
        "include_batch_events": false,
        "include_optimizer_events": true,
        "output_dir": "logs/loss_detail"
      },
      "loss_mini_batch_size": 32,
      "loss_mini_batch_size_by_bs": {
        "1152": 128,
        "192": 16,
        "240": 16,
        "288": 32,
        "336": 32,
        "432": 32,
        "480": 64,
        "768": 64
      },
      "losses": {
        "doc_doc": [
          {
            "klass": "CachedMultipleNegativesMaskedBidirectionalRankingLoss",
            "kwargs": {
              "debug_mask_stats": 10,
              "gather_across_devices": false,
              "include_doc_to_query_loss": true,
              "margin": 0.01,
              "temperature": 0.03
            }
          },
          {
            "klass": "QuantizationAwareLoss",
            "kwargs": {
              "quantization_precisions": [
                "float32",
                "int8",
                "ubinary"
              ],
              "quantization_weights": [
                1.0,
                0.1,
                0.1
              ]
            }
          },
          {
            "klass": "MatryoshkaLoss",
            "kwargs": {
              "matryoshka_dims": [
                384,
                256,
                128,
                64
              ],
              "matryoshka_weights": [
                1.0,
                0.3,
                0.15,
                0.1
              ]
            }
          }
        ],
        "query_doc": [
          {
            "klass": "CachedMultipleNegativesMaskedBidirectionalRankingLoss",
            "kwargs": {
              "debug_mask_stats": 10,
              "gather_across_devices": false,
              "margin": 0.01,
              "temperature": 0.03
            }
          },
          {
            "klass": "QuantizationAwareLoss",
            "kwargs": {
              "quantization_precisions": [
                "float32",
                "int8",
                "ubinary"
              ],
              "quantization_weights": [
                1.0,
                0.1,
                0.1
              ]
            }
          },
          {
            "klass": "MatryoshkaLoss",
            "kwargs": {
              "matryoshka_dims": [
                384,
                256,
                128,
                64
              ],
              "matryoshka_weights": [
                1.0,
                0.3,
                0.15,
                0.1
              ]
            }
          }
        ]
      },
      "lr_scheduler_type": "cosine",
      "num_train_epochs": 1,
      "optim": "adamw_torch",
      "output_root": "output/models/embedding_v1",
      "per_device_eval_batch_size": 64,
      "per_device_train_batch_size": "DS",
      "report_to": [
        "wandb"
      ],
      "save_steps": 200,
      "save_strategy": "steps",
      "save_total_limit": 5,
      "seed": 12,
      "skip_evaluation": false,
      "use_multi_dataset_sampler": true,
      "wandb_project": "mmbert-embedding",
      "warmup_ratio": 0.3,
      "weight_decay": 0.01
    }
  },
  "model_name": "ft-a8m-ft-prefix-none-from-full-prefix-none-a25m-ftdata__a8m-pretrain-full-unir-v9-prefix-none-mrl-qat",
  "monolithic_config_path": null,
  "mrl_loss": null,
  "multi_dataset_batch_sampler": null,
  "num_train_epochs": 1,
  "output_dir": "output/models/embedding_v1/ft-a8m-ft-prefix-none-from-full-prefix-none-a25m-ftdata__a8m-pretrain-full-unir-v9-prefix-none-mrl-qat/ft-a8m-prefix-none-fullpretrain-a25m-ftdata/20260707_092449",
  "override_paths": [],
  "overrides": [],
  "per_device_eval_batch_size": 64,
  "per_device_train_batch_size": "DS",
  "raw_cli_args": [
    "/home/hotchpotch/src/github.com/hotchpotch/mmBERT-embedding-reranker/.venv/bin/trainer",
    "--training_config",
    "configs/training_args/ablation/a8m-ft-prefix-none-from-full-prefix-none-a25m-ftdata.yaml",
    "--data_config",
    "configs/data_args/ablation/ft-bekko-embedding-v1-a8m-prefix-none-a25m-ftdata.yaml",
    "--run-name",
    "ft-a8m-prefix-none-fullpretrain-a25m-ftdata"
  ],
  "run_name": "ft-a8m-prefix-none-fullpretrain-a25m-ftdata-20260707_092449",
  "save_steps": 200,
  "save_strategy": "steps",
  "save_total_limit": 5,
  "seed": 12,
  "sync_once_per_batch": true,
  "train_ds_label": "ft-bekko-embedding-v1-a8m-prefix-none-a25m-ftdata",
  "training_config_path": "configs/training_args/ablation/a8m-ft-prefix-none-from-full-prefix-none-a25m-ftdata.yaml",
  "use_multi_dataset_sampler": true,
  "warmup_ratio": 0.3,
  "warmup_steps": null,
  "weight_decay": 0.01
}