| { |
| "attn_implementation": "flash_attention_2", |
| "batch_sampler": "no_duplicates_fast", |
| "bf16": true, |
| "bf16_full_eval": true, |
| "config_path": "configs/training_args/ablation/a8m-ft-prefix-none-from-full-prefix-none-a25m-ftdata.yaml", |
| "data_config_path": "configs/data_args/ablation/ft-bekko-embedding-v1-a8m-prefix-none-a25m-ftdata.yaml", |
| "data_summary": [ |
| "ruri_v3_ft-auto-wiki-qa-nemotron: train=154,956 eval=0", |
| "ruri_v3_ft-jaquad: train=31,253 eval=0", |
| "ruri_v3_ft-jqara: train=19,515 eval=0", |
| "ruri_v3_ft-jsquad: train=61,659 eval=0", |
| "ruri_v3_ft-miracl: train=25,905 eval=0", |
| "ruri_v3_ft-mkqa: train=30,310 eval=0", |
| "ruri_v3_ft-mr-tydi: train=18,200 eval=0", |
| "ruri_v3_ft-nli: train=222,429 eval=0", |
| "ruri_v3_ft-quiz-no-mori: train=37,437 eval=0", |
| "ruri_v3_ft-quiz-works: train=30,631 eval=0", |
| "bge_m3_ft-dureader-len-0-500+len-500-1000+len-1000-2000+len-2000-3000+len-3000-4000+len-4000-5000+len-5000-6000+len-6000-7000+len-7000-inf: train=80,416 eval=0", |
| "bge_m3_ft-hotpotqa-len-0-500+len-500-1000+len-1000-2000+len-2000-3000+len-3000-4000+len-4000-5000+len-5000-6000+len-6000-7000+len-7000-inf: train=84,516 eval=0", |
| "bge_m3_ft-miracl-len-0-500+len-500-1000+len-1000-2000+len-2000-3000+len-3000-4000+len-4000-5000+len-5000-6000+len-6000-7000+len-7000-inf: train=40,203 eval=0", |
| "bge_m3_ft-mr-tydi-len-0-500+len-500-1000+len-1000-2000+len-2000-3000+len-3000-4000+len-4000-5000+len-5000-6000+len-6000-7000+len-7000-inf: train=48,729 eval=0", |
| "bge_m3_ft-squad-len-0-500+len-500-1000+len-1000-2000+len-2000-3000+len-3000-4000+len-4000-5000+len-5000-6000+len-6000-7000+len-7000-inf: train=87,599 eval=0", |
| "bge_m3_ft-other-len-0-500+len-500-1000+len-1000-2000+len-2000-3000+len-3000-4000+len-4000-5000+len-5000-6000+len-6000-7000+len-7000-inf: train=500 eval=0", |
| "bge_m3_ft-msmarco-len-0-500+len-500-1000+len-1000-2000+len-2000-3000+len-3000-4000+len-4000-5000+len-5000-6000+len-6000-7000+len-7000-inf: train=200,000 eval=0", |
| "bge_m3_ft-mmarco_chinese-len-0-500: train=80,000 eval=0", |
| "bge_m3_ft-ATEC-len-0-500: train=11,325 eval=0", |
| "bge_m3_ft-BQ-len-0-500: train=12,518 eval=0", |
| "bge_m3_ft-LCQMC-len-0-500: train=10,000 eval=0", |
| "bge_m3_ft-PAWSX-len-0-500: train=9,900 eval=0", |
| "bekko_embedding_hard_negatives-wikipedia_hard_negatives_english: train=250,000 eval=0", |
| "bekko_embedding_hard_negatives-additional-query-doc: train=80,000 eval=0", |
| "bekko_embedding_hard_negatives-additional-doc-doc: train=40,000 eval=0", |
| "bekko_embedding_hard_negatives-wikipedia_hard_negatives_long_docs_en: train=12,000 eval=0", |
| "bekko_embedding_hard_negatives-wikipedia_hard_negatives_long_docs_ar: train=2,000 eval=0", |
| "bekko_embedding_hard_negatives-wikipedia_hard_negatives_long_docs_de: train=2,000 eval=0", |
| "bekko_embedding_hard_negatives-wikipedia_hard_negatives_long_docs_es: train=2,000 eval=0", |
| "bekko_embedding_hard_negatives-wikipedia_hard_negatives_long_docs_fr: train=2,000 eval=0", |
| "bekko_embedding_hard_negatives-wikipedia_hard_negatives_long_docs_it: train=2,000 eval=0", |
| "bekko_embedding_hard_negatives-wikipedia_hard_negatives_long_docs_ja: train=2,000 eval=0", |
| "bekko_embedding_hard_negatives-wikipedia_hard_negatives_long_docs_ko: train=2,000 eval=0", |
| "bekko_embedding_hard_negatives-wikipedia_hard_negatives_long_docs_pt: train=2,000 eval=0", |
| "bekko_embedding_hard_negatives-wikipedia_hard_negatives_long_docs_ru: train=2,000 eval=0", |
| "bekko_embedding_hard_negatives-wikipedia_hard_negatives_long_docs_zh: train=2,000 eval=0", |
| "bekko_embedding_hard_negatives-extra-four: train=80,000 eval=0" |
| ], |
| "decay_ratio": null, |
| "decay_steps": null, |
| "eval_steps": 400, |
| "evaluation_strategy": "steps", |
| "fp16": false, |
| "ft_base_model": "output/models/embedding_v1/mmBERT-L4H384-pruned/a8m-pretrain-full-unir-v9-prefix-none-mrl-qat/20260704_041654/final", |
| "global_rope_theta": null, |
| "gradient_accumulation_steps": 1, |
| "learning_rate": 1e-05, |
| "local_rope_theta": null, |
| "logging_steps": 10, |
| "loss_mini_batch_size": 32, |
| "loss_mini_batch_size_by_bs": { |
| "1152": 128, |
| "192": 16, |
| "240": 16, |
| "288": 32, |
| "336": 32, |
| "432": 32, |
| "480": 64, |
| "768": 64 |
| }, |
| "losses": { |
| "doc_doc": [ |
| { |
| "klass": "CachedMultipleNegativesMaskedBidirectionalRankingLoss", |
| "kwargs": { |
| "debug_mask_stats": 10, |
| "gather_across_devices": false, |
| "include_doc_to_query_loss": true, |
| "margin": 0.01, |
| "temperature": 0.03 |
| } |
| }, |
| { |
| "klass": "QuantizationAwareLoss", |
| "kwargs": { |
| "quantization_precisions": [ |
| "float32", |
| "int8", |
| "ubinary" |
| ], |
| "quantization_weights": [ |
| 1.0, |
| 0.1, |
| 0.1 |
| ] |
| } |
| }, |
| { |
| "klass": "MatryoshkaLoss", |
| "kwargs": { |
| "matryoshka_dims": [ |
| 384, |
| 256, |
| 128, |
| 64 |
| ], |
| "matryoshka_weights": [ |
| 1.0, |
| 0.3, |
| 0.15, |
| 0.1 |
| ] |
| } |
| } |
| ], |
| "query_doc": [ |
| { |
| "klass": "CachedMultipleNegativesMaskedBidirectionalRankingLoss", |
| "kwargs": { |
| "debug_mask_stats": 10, |
| "gather_across_devices": false, |
| "margin": 0.01, |
| "temperature": 0.03 |
| } |
| }, |
| { |
| "klass": "QuantizationAwareLoss", |
| "kwargs": { |
| "quantization_precisions": [ |
| "float32", |
| "int8", |
| "ubinary" |
| ], |
| "quantization_weights": [ |
| 1.0, |
| 0.1, |
| 0.1 |
| ] |
| } |
| }, |
| { |
| "klass": "MatryoshkaLoss", |
| "kwargs": { |
| "matryoshka_dims": [ |
| 384, |
| 256, |
| 128, |
| 64 |
| ], |
| "matryoshka_weights": [ |
| 1.0, |
| 0.3, |
| 0.15, |
| 0.1 |
| ] |
| } |
| } |
| ] |
| }, |
| "lr_scheduler_kwargs": null, |
| "lr_scheduler_type": "cosine", |
| "max_seq_length": 8192, |
| "max_steps": null, |
| "merged_config": { |
| "_config_path": "configs/training_args/ablation/a8m-ft-prefix-none-from-full-prefix-none-a25m-ftdata.yaml", |
| "_data_config_path": "configs/data_args/ablation/ft-bekko-embedding-v1-a8m-prefix-none-a25m-ftdata.yaml", |
| "_training_config_path": "configs/training_args/ablation/a8m-ft-prefix-none-from-full-prefix-none-a25m-ftdata.yaml", |
| "data": { |
| "_config_path": "/home/hotchpotch/src/github.com/hotchpotch/mmBERT-embedding-reranker/configs/data/ft-bekko-embedding-v1-a25m-20260625.yaml", |
| "common": { |
| "eval_limit": 0, |
| "group_small_subsets_label": "other", |
| "group_small_subsets_threshold": 2000, |
| "group_subsets_by_prefix": true, |
| "group_subsets_prefixes": [ |
| "__auto__" |
| ], |
| "items": 8, |
| "limit": null, |
| "loader": "bge_m3_ft", |
| "neg_sampling": { |
| "strategy": "mid_tail_random" |
| }, |
| "pair_type": "query_doc", |
| "path": "output/bge-m3-data-ft-raw-dataset" |
| }, |
| "train_datasets": [ |
| { |
| "bs_size": 1152, |
| "eval_limit": 0, |
| "items": 8, |
| "limit": null, |
| "loader": "ruri_v3_ft", |
| "max_len": 512, |
| "pair_type": "query_doc", |
| "subsets": [ |
| "*" |
| ] |
| }, |
| { |
| "bs_size": 1152, |
| "eval_limit": 0, |
| "group_small_subsets_label": "other", |
| "group_small_subsets_threshold": 2000, |
| "group_subsets_by_prefix": true, |
| "group_subsets_prefixes": [ |
| "__auto__" |
| ], |
| "items": 8, |
| "limit": null, |
| "loader": "bge_m3_ft", |
| "max_len": 512, |
| "neg_sampling": { |
| "strategy": "mid_tail_random" |
| }, |
| "pair_type": "query_doc", |
| "path": "output/bge-m3-data-ft-raw-dataset", |
| "subset_suffixes": [ |
| "len-0-500", |
| "len-500-1000", |
| "len-1000-2000", |
| "len-2000-3000", |
| "len-3000-4000", |
| "len-4000-5000", |
| "len-5000-6000", |
| "len-6000-7000", |
| "len-7000-inf" |
| ], |
| "subsets": [ |
| "dureader", |
| "hotpotqa", |
| "miracl", |
| "mr-tydi", |
| "pubmed_qa_labeled", |
| "squad" |
| ] |
| }, |
| { |
| "bs_size": 1152, |
| "eval_limit": 0, |
| "group_small_subsets_label": "other", |
| "group_small_subsets_threshold": 2000, |
| "group_subsets_by_prefix": true, |
| "group_subsets_prefixes": [ |
| "__auto__" |
| ], |
| "items": 8, |
| "limit": 200000, |
| "loader": "bge_m3_ft", |
| "max_len": 512, |
| "neg_sampling": { |
| "strategy": "mid_tail_random" |
| }, |
| "pair_type": "query_doc", |
| "path": "output/bge-m3-data-ft-raw-dataset", |
| "subset_suffixes": [ |
| "len-0-500", |
| "len-500-1000", |
| "len-1000-2000", |
| "len-2000-3000", |
| "len-3000-4000", |
| "len-4000-5000", |
| "len-5000-6000", |
| "len-6000-7000", |
| "len-7000-inf" |
| ], |
| "subsets": [ |
| "msmarco" |
| ] |
| }, |
| { |
| "bs_size": 1152, |
| "eval_limit": 0, |
| "group_small_subsets_label": "other", |
| "group_small_subsets_threshold": 2000, |
| "group_subsets_by_prefix": true, |
| "group_subsets_prefixes": [ |
| "__auto__" |
| ], |
| "items": 8, |
| "limit": 80000, |
| "loader": "bge_m3_ft", |
| "max_len": 512, |
| "neg_sampling": { |
| "strategy": "mid_tail_random" |
| }, |
| "pair_type": "query_doc", |
| "path": "output/bge-m3-data-ft-raw-dataset", |
| "subset_suffixes": [ |
| "len-0-500" |
| ], |
| "subsets": [ |
| "mmarco_chinese" |
| ] |
| }, |
| { |
| "bs_size": 1152, |
| "eval_limit": 0, |
| "group_small_subsets_label": "other", |
| "group_small_subsets_threshold": 2000, |
| "group_subsets_by_prefix": true, |
| "group_subsets_prefixes": [ |
| "__auto__" |
| ], |
| "items": 8, |
| "limit": null, |
| "loader": "bge_m3_ft", |
| "max_len": 512, |
| "neg_sampling": { |
| "strategy": "mid_tail_random" |
| }, |
| "pair_type": "doc_doc", |
| "pair_type_by_label": { |
| "ATEC": "doc_doc", |
| "BQ": "doc_doc", |
| "LCQMC": "doc_doc", |
| "PAWSX": "doc_doc" |
| }, |
| "path": "output/bge-m3-data-ft-raw-dataset", |
| "subset_suffixes": [ |
| "len-0-500" |
| ], |
| "subsets": [ |
| "ATEC", |
| "BQ", |
| "LCQMC", |
| "PAWSX" |
| ] |
| }, |
| { |
| "bs_size": 1152, |
| "eval_limit": 0, |
| "items": 8, |
| "limit": 250000, |
| "loader": "bekko_embedding_hard_negatives", |
| "max_len": 512, |
| "name": "hotchpotch/bekko-embedding-hard-negatives", |
| "pair_type": "query_doc", |
| "split": "train", |
| "subset": "wikipedia-hard-negatives-english", |
| "subsets": [ |
| "wikipedia_hard_negatives_english" |
| ] |
| }, |
| { |
| "bs_size": 1152, |
| "eval_limit": 0, |
| "items": 8, |
| "limit": null, |
| "loader": "bekko_embedding_hard_negatives", |
| "max_len": 512, |
| "merge_subsets": true, |
| "name": "hotchpotch/bekko-embedding-hard-negatives", |
| "pair_type": "query_doc", |
| "split": "train", |
| "subset": "additional-query-doc", |
| "subsets": [ |
| "agnews", |
| "gooaq", |
| "natural_questions", |
| "trivia_qa" |
| ] |
| }, |
| { |
| "bs_size": 1152, |
| "eval_limit": 0, |
| "items": 8, |
| "limit": null, |
| "loader": "bekko_embedding_hard_negatives", |
| "max_len": 512, |
| "merge_subsets": true, |
| "name": "hotchpotch/bekko-embedding-hard-negatives", |
| "pair_type": "doc_doc", |
| "split": "train", |
| "subset": "additional-doc-doc", |
| "subsets": [ |
| "all_nli", |
| "coco_captions" |
| ] |
| }, |
| { |
| "bs_size": 192, |
| "eval_limit": 0, |
| "items": 8, |
| "limit": 12000, |
| "loader": "bekko_embedding_hard_negatives", |
| "max_len": 8192, |
| "name": "hotchpotch/bekko-embedding-hard-negatives", |
| "neg_sampling": { |
| "head_count": 4, |
| "split_at": 7, |
| "strategy": "split_random" |
| }, |
| "pair_type": "query_doc", |
| "split": "train", |
| "subset": "wikipedia-hard-negatives-long-docs-en-rebalanced", |
| "subsets": [ |
| "wikipedia_hard_negatives_long_docs_en" |
| ] |
| }, |
| { |
| "bs_size": 192, |
| "eval_limit": 0, |
| "items": 8, |
| "limit": 2000, |
| "loader": "bekko_embedding_hard_negatives", |
| "max_len": 8192, |
| "name": "hotchpotch/bekko-embedding-hard-negatives", |
| "neg_sampling": { |
| "head_count": 4, |
| "split_at": 7, |
| "strategy": "split_random" |
| }, |
| "pair_type": "query_doc", |
| "split": "train", |
| "subset": "wikipedia-hard-negatives-long-docs-other-rebalanced", |
| "subsets": [ |
| "wikipedia_hard_negatives_long_docs_ar", |
| "wikipedia_hard_negatives_long_docs_de", |
| "wikipedia_hard_negatives_long_docs_es", |
| "wikipedia_hard_negatives_long_docs_fr", |
| "wikipedia_hard_negatives_long_docs_it", |
| "wikipedia_hard_negatives_long_docs_ja", |
| "wikipedia_hard_negatives_long_docs_ko", |
| "wikipedia_hard_negatives_long_docs_pt", |
| "wikipedia_hard_negatives_long_docs_ru", |
| "wikipedia_hard_negatives_long_docs_zh" |
| ] |
| }, |
| { |
| "bs_size": 1152, |
| "eval_limit": 0, |
| "items": 8, |
| "limit": null, |
| "loader": "bekko_embedding_hard_negatives", |
| "max_len": 512, |
| "merge_subsets": true, |
| "name": "hotchpotch/bekko-embedding-hard-negatives", |
| "pair_type": "query_doc", |
| "split": "train", |
| "subset": "extra-four", |
| "subsets": [ |
| "arxiv", |
| "codesearch", |
| "fineweb", |
| "pubmed" |
| ] |
| } |
| ] |
| }, |
| "model": { |
| "doc_doc_prefix_fallback_to_query": false, |
| "doc_prefix": "", |
| "ft_base_model": "output/models/embedding_v1/mmBERT-L4H384-pruned/a8m-pretrain-full-unir-v9-prefix-none-mrl-qat/20260704_041654/final", |
| "max_seq_length": 8192, |
| "name": "hotchpotch/mmBERT-L4H384-pruned", |
| "query_prefix": "" |
| }, |
| "training": { |
| "batch_sampler": "no_duplicates_fast", |
| "bf16": true, |
| "bf16_full_eval": true, |
| "cleanup_checkpoints": false, |
| "dataloader_drop_last": true, |
| "dataloader_num_workers": 4, |
| "dataloader_persistent_workers": false, |
| "dataloader_prefetch_factor": 1, |
| "eval_before_train": false, |
| "eval_prefix_in_dataset": true, |
| "eval_steps": 400, |
| "fp16": false, |
| "gradient_accumulation_steps": 1, |
| "learning_rate": 1e-05, |
| "logging_steps": 10, |
| "loss_detail_logging": { |
| "enabled": true, |
| "flush_every_records": 1, |
| "include_batch_events": false, |
| "include_optimizer_events": true, |
| "output_dir": "logs/loss_detail" |
| }, |
| "loss_mini_batch_size": 32, |
| "loss_mini_batch_size_by_bs": { |
| "1152": 128, |
| "192": 16, |
| "240": 16, |
| "288": 32, |
| "336": 32, |
| "432": 32, |
| "480": 64, |
| "768": 64 |
| }, |
| "losses": { |
| "doc_doc": [ |
| { |
| "klass": "CachedMultipleNegativesMaskedBidirectionalRankingLoss", |
| "kwargs": { |
| "debug_mask_stats": 10, |
| "gather_across_devices": false, |
| "include_doc_to_query_loss": true, |
| "margin": 0.01, |
| "temperature": 0.03 |
| } |
| }, |
| { |
| "klass": "QuantizationAwareLoss", |
| "kwargs": { |
| "quantization_precisions": [ |
| "float32", |
| "int8", |
| "ubinary" |
| ], |
| "quantization_weights": [ |
| 1.0, |
| 0.1, |
| 0.1 |
| ] |
| } |
| }, |
| { |
| "klass": "MatryoshkaLoss", |
| "kwargs": { |
| "matryoshka_dims": [ |
| 384, |
| 256, |
| 128, |
| 64 |
| ], |
| "matryoshka_weights": [ |
| 1.0, |
| 0.3, |
| 0.15, |
| 0.1 |
| ] |
| } |
| } |
| ], |
| "query_doc": [ |
| { |
| "klass": "CachedMultipleNegativesMaskedBidirectionalRankingLoss", |
| "kwargs": { |
| "debug_mask_stats": 10, |
| "gather_across_devices": false, |
| "margin": 0.01, |
| "temperature": 0.03 |
| } |
| }, |
| { |
| "klass": "QuantizationAwareLoss", |
| "kwargs": { |
| "quantization_precisions": [ |
| "float32", |
| "int8", |
| "ubinary" |
| ], |
| "quantization_weights": [ |
| 1.0, |
| 0.1, |
| 0.1 |
| ] |
| } |
| }, |
| { |
| "klass": "MatryoshkaLoss", |
| "kwargs": { |
| "matryoshka_dims": [ |
| 384, |
| 256, |
| 128, |
| 64 |
| ], |
| "matryoshka_weights": [ |
| 1.0, |
| 0.3, |
| 0.15, |
| 0.1 |
| ] |
| } |
| } |
| ] |
| }, |
| "lr_scheduler_type": "cosine", |
| "num_train_epochs": 1, |
| "optim": "adamw_torch", |
| "output_root": "output/models/embedding_v1", |
| "per_device_eval_batch_size": 64, |
| "per_device_train_batch_size": "DS", |
| "report_to": [ |
| "wandb" |
| ], |
| "save_steps": 200, |
| "save_strategy": "steps", |
| "save_total_limit": 5, |
| "seed": 12, |
| "skip_evaluation": false, |
| "use_multi_dataset_sampler": true, |
| "wandb_project": "mmbert-embedding", |
| "warmup_ratio": 0.3, |
| "weight_decay": 0.01 |
| } |
| }, |
| "model_name": "ft-a8m-ft-prefix-none-from-full-prefix-none-a25m-ftdata__a8m-pretrain-full-unir-v9-prefix-none-mrl-qat", |
| "monolithic_config_path": null, |
| "mrl_loss": null, |
| "multi_dataset_batch_sampler": null, |
| "num_train_epochs": 1, |
| "output_dir": "output/models/embedding_v1/ft-a8m-ft-prefix-none-from-full-prefix-none-a25m-ftdata__a8m-pretrain-full-unir-v9-prefix-none-mrl-qat/ft-a8m-prefix-none-fullpretrain-a25m-ftdata/20260707_092449", |
| "override_paths": [], |
| "overrides": [], |
| "per_device_eval_batch_size": 64, |
| "per_device_train_batch_size": "DS", |
| "raw_cli_args": [ |
| "/home/hotchpotch/src/github.com/hotchpotch/mmBERT-embedding-reranker/.venv/bin/trainer", |
| "--training_config", |
| "configs/training_args/ablation/a8m-ft-prefix-none-from-full-prefix-none-a25m-ftdata.yaml", |
| "--data_config", |
| "configs/data_args/ablation/ft-bekko-embedding-v1-a8m-prefix-none-a25m-ftdata.yaml", |
| "--run-name", |
| "ft-a8m-prefix-none-fullpretrain-a25m-ftdata" |
| ], |
| "run_name": "ft-a8m-prefix-none-fullpretrain-a25m-ftdata-20260707_092449", |
| "save_steps": 200, |
| "save_strategy": "steps", |
| "save_total_limit": 5, |
| "seed": 12, |
| "sync_once_per_batch": true, |
| "train_ds_label": "ft-bekko-embedding-v1-a8m-prefix-none-a25m-ftdata", |
| "training_config_path": "configs/training_args/ablation/a8m-ft-prefix-none-from-full-prefix-none-a25m-ftdata.yaml", |
| "use_multi_dataset_sampler": true, |
| "warmup_ratio": 0.3, |
| "warmup_steps": null, |
| "weight_decay": 0.01 |
| } |
