{ "checkpoint": "/home/zhangchao/work/minye_training/mixed_retrieval_normal_identity_balanced/retriever.pt", "protocol": { "training_stage": "multiscene_ordered_time_audio_query_v4_1", "training_sources": [ "hotpotqa" ], "hidden_size": 4096, "embedding_size": 1024, "attention_heads": 8, "max_text_tokens": 512, "max_query_tokens": 128, "dropout": 0.1, "hidden_layer_index": 19, "retrieval_lora_layers": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 ], "retrieval_lora_rank": 8, "retrieval_lora_alpha": 32.0, "retrieval_lora_dropout": 0.05, "query_context_turns": 0, "query_serialization": "current_session_audio_plus_query_audio", "validation_query_modalities": [], "training_query_modality": "audio_only", "missing_audio_policy": "current-user missing: skip; previous-user missing: text fallback", "audio_query_context_turns": 0, "audio_context_source": "current_user_audio", "conversation_memory_context_turns": 0, "conversation_memory_sources": [], "hard_negative_mining": "previous_epoch_dense_false_positives_v1", "checkpoint_selection": "last_epoch", "multi_positive_objective": "mean_positive_log_probability", "pooling": "shared_attention_then_directional_qk", "use_retrieval_rope": false, "output_normalization": "l2", "speaker_checkpoint_path": "/home/zhangchao/project/salmonn_omni_transform_v2_siyuan/siyuanTest/train_speaker_embedding/outputs/mix_5_7_8_ddp_4096_proj1024_min8/best", "speaker_window_seconds": 5.0, "time_serialization": "numeric_temporal_route_only", "memory_temporal_positions_shuffled": false, "memory_serialization": "dataset_speaker_name_plus_text", "random_history_sessions": true, "history_session_seed": 20261015, "training_speaker_names": "epoch_random_alias_all_self_no_spoken_names_v1", "cross_rank_candidates": true, "hard_negatives_per_query": 16, "hard_negative_loss_weight": 1.0, "query_counterfactuals": true, "retriever_architecture": "semantic_plus_ordered_time_qk_v4_1", "embedding_layout": "raw_concat_unit_semantic_bounded_temporal_v3", "similarity": "inner_product", "time_embedding_size": 128, "semantic_embedding_size": 1024, "retrieval_embedding_size": 1152, "time_encoder": "fixed_multiscale_features_linear_key_v4_1", "query_pool_audio_tokens": 128, "query_encoding_protocol": "incremental_sliding_kv_pinned_prefix", "query_input_truncation": null, "temporal_pooler": "retained_audio_tail_h19", "temporal_query_tokens": "last_128_audio_h19_excluding_speaker", "temporal_projection": "audio_wording_plus_reference_q_numeric_time_k_v4_1", "time_scale_seconds": 60.0, "semantic_loss_weight": 0.25, "temporal_null_loss_weight": 0.1, "temporal_relations": [ "none", "latest", "earliest", "before", "after", "session", "comparison" ], "temporal_strength": "query_vector_norm_without_classifier_gate_v3", "retrieval_rope_positions": null, "no_temporal": false, "data_jsonl": "/home/zhangchao/work/minye_training/mixed_identity_balanced_data/train_val.jsonl", "initial_checkpoint": "/home/zhangchao/work/minye_training/mixed_retrieval_normal/retriever.pt", "query_modality": "audio", "memory_modality": "text", "speaker_embedding": "frozen_4096d_prepend_to_h19", "query_speaker_name_text": false, "memory_speaker_name_text": "dataset_provided", "validation_speaker_names": "dataset_provided", "time": "session_index_and_start_sec", "cross_rank_negatives": true, "all_in_sample_memories_are_candidates": true, "candidate_memory_scope": "all_history_utterances_excluding_current_session", "temporal_wording_counts": { "train:after": 1, "train:before": 98, "train:comparison": 903, "train:earliest": 783, "train:latest": 748, "train:session": 1431, "train:unmatched": 390, "validation:before": 70, "validation:comparison": 414, "validation:earliest": 354, "validation:latest": 358, "validation:session": 789, "validation:unmatched": 161 }, "control_support_handling": "exclude_declared_current_context_supports" }, "checkpoint_selection_metric": "recall_at_k.1", "best_validation_recall_at_1": 0.7014623880386353, "best_epoch": 2, "history": [ { "epoch": 1, "checkpoint": "/home/zhangchao/work/minye_training/mixed_retrieval_normal_identity_balanced/retriever_epoch_01.pt", "train_loss": 0.9721298217773438, "train_losses": { "total": 0.9721298217773438, "global": 1.0716792345046997, "hard_negative": 0.0, "semantic": 0.7763091921806335, "temporal_null": 0.03924102708697319, "speaker_name_renamed_samples": 639, "speaker_name_skipped_mixed_or_nonself": 4108, "speaker_name_skipped_spoken_name": 267, "speaker_name_skipped_insufficient_names": 0 }, "validation": { "query_count": 3985, "recall_at_k": { "1": 0.6997685432434082, "3": 0.9176088571548462, "5": 0.9558404684066772 }, "all_support_em_at_k": { "3": 0.8727728724479675, "5": 0.9309912323951721 }, "by_speaker_query_type": { "fact_to_person": { "query_count": 786, "recall_at_k": { "1": 0.8282034397125244, "3": 0.9724811911582947, "5": 0.9802632927894592 }, "all_support_em_at_k": { "3": 0.9618320465087891, "5": 0.9669210910797119 } }, "person_to_fact": { "query_count": 969, "recall_at_k": { "1": 0.7436802983283997, "3": 0.9417711496353149, "5": 0.9750897288322449 }, "all_support_em_at_k": { "3": 0.9112487435340881, "5": 0.9638803005218506 } }, "self_to_fact": { "query_count": 509, "recall_at_k": { "1": 0.2703012526035309, "3": 0.7080877423286438, "5": 0.8973477482795715 }, "all_support_em_at_k": { "3": 0.5068762302398682, "5": 0.8192534446716309 } }, "speaker_independent": { "query_count": 1721, "recall_at_k": { "1": 0.7434049248695374, "3": 0.9409112930297852, "5": 0.9511474967002869 }, "all_support_em_at_k": { "3": 0.9186519384384155, "5": 0.9291110038757324 } } }, "by_task_and_speaker_query_type": { "change_comparison": { "fact_to_person": { "query_count": 74, "recall_at_k": { "1": 0.5, "3": 1.0, "5": 1.0 }, "all_support_em_at_k": { "3": 1.0, "5": 1.0 } }, "person_to_fact": { "query_count": 88, "recall_at_k": { "1": 0.4943181872367859, "3": 0.9602272510528564, "5": 1.0 }, "all_support_em_at_k": { "3": 0.9204545617103577, "5": 1.0 } }, "self_to_fact": { "query_count": 99, "recall_at_k": { "1": 0.13131313025951385, "3": 0.5, "5": 0.7878788113594055 }, "all_support_em_at_k": { "3": 0.1111111119389534, "5": 0.5757575631141663 } }, "speaker_independent": { "query_count": 209, "recall_at_k": { "1": 0.49920251965522766, "3": 0.9920255541801453, "5": 0.9932217597961426 }, "all_support_em_at_k": { "3": 0.9904305934906006, "5": 0.9904305934906006 } } }, "direct_recall": { "fact_to_person": { "query_count": 524, "recall_at_k": { "1": 0.9228689074516296, "3": 0.9688295125961304, "5": 0.980120837688446 }, "all_support_em_at_k": { "3": 0.9580152630805969, "5": 0.9656488299369812 } }, "person_to_fact": { "query_count": 306, "recall_at_k": { "1": 0.8218954205513, "3": 0.9327341914176941, "5": 0.9556099772453308 }, "all_support_em_at_k": { "3": 0.9215686321258545, "5": 0.9379084706306458 } }, "self_to_fact": { "query_count": 57, "recall_at_k": { "1": 0.41374269127845764, "3": 0.5891813039779663, "5": 0.6856725811958313 }, "all_support_em_at_k": { "3": 0.4035087823867798, "5": 0.5614035129547119 } }, "speaker_independent": { "query_count": 604, "recall_at_k": { "1": 0.8246412873268127, "3": 0.9071468114852905, "5": 0.9170805811882019 }, "all_support_em_at_k": { "3": 0.8807947039604187, "5": 0.8907284736633301 } } }, "historical_state": { "fact_to_person": { "query_count": 91, "recall_at_k": { "1": 0.901098906993866, "3": 1.0, "5": 1.0 }, "all_support_em_at_k": { "3": 1.0, "5": 1.0 } }, "person_to_fact": { "query_count": 125, "recall_at_k": { "1": 0.871999979019165, "3": 1.0, "5": 1.0 }, "all_support_em_at_k": { "3": 1.0, "5": 1.0 } }, "self_to_fact": { "query_count": 82, "recall_at_k": { "1": 0.31707316637039185, "3": 0.9878048896789551, "5": 1.0 }, "all_support_em_at_k": { "3": 0.9878048896789551, "5": 1.0 } }, "speaker_independent": { "query_count": 202, "recall_at_k": { "1": 0.9207921028137207, "3": 0.9975247383117676, "5": 0.9975247383117676 }, "all_support_em_at_k": { "3": 0.9950494766235352, "5": 0.9950494766235352 } } }, "latest_state": { "fact_to_person": { "query_count": 31, "recall_at_k": { "1": 0.5806451439857483, "3": 1.0, "5": 1.0 }, "all_support_em_at_k": { "3": 1.0, "5": 1.0 } }, "person_to_fact": { "query_count": 92, "recall_at_k": { "1": 0.375, "3": 0.989130437374115, "5": 1.0 }, "all_support_em_at_k": { "3": 0.989130437374115, "5": 1.0 } }, "self_to_fact": { "query_count": 82, "recall_at_k": { "1": 0.2195121943950653, "3": 0.8414633870124817, "5": 1.0 }, "all_support_em_at_k": { "3": 0.8414633870124817, "5": 1.0 } }, "speaker_independent": { "query_count": 252, "recall_at_k": { "1": 0.4682539701461792, "3": 0.9861111044883728, "5": 0.9980158805847168 }, "all_support_em_at_k": { "3": 0.9841269850730896, "5": 0.9960317611694336 } } }, "multi_hop_composition": { "fact_to_person": { "query_count": 66, "recall_at_k": { "1": 0.46037158370018005, "3": 0.9197511076927185, "5": 0.9227814078330994 }, "all_support_em_at_k": { "3": 0.8787878751754761, "5": 0.8787878751754761 } }, "person_to_fact": { "query_count": 84, "recall_at_k": { "1": 0.33185943961143494, "3": 0.6774943470954895, "5": 0.9130384922027588 }, "all_support_em_at_k": { "3": 0.4166666567325592, "5": 0.8571428656578064 } }, "self_to_fact": { "query_count": 168, "recall_at_k": { "1": 0.2738095223903656, "3": 0.6666666865348816, "5": 0.9404761791229248 }, "all_support_em_at_k": { "3": 0.369047611951828, "5": 0.886904776096344 } }, "speaker_independent": { "query_count": 30, "recall_at_k": { "1": 0.2661111056804657, "3": 0.5019444823265076, "5": 0.5530555844306946 }, "all_support_em_at_k": { "3": 0.1666666716337204, "5": 0.23333333432674408 } } }, "temporal_localization": { "person_to_fact": { "query_count": 274, "recall_at_k": { "1": 0.9279196858406067, "3": 0.9844890236854553, "5": 0.9881386756896973 }, "all_support_em_at_k": { "3": 0.9817517995834351, "5": 0.985401451587677 } }, "self_to_fact": { "query_count": 21, "recall_at_k": { "1": 0.5238094925880432, "3": 0.7301587462425232, "5": 0.8412699103355408 }, "all_support_em_at_k": { "3": 0.5714285969734192, "5": 0.7142857313156128 } }, "speaker_independent": { "query_count": 424, "recall_at_k": { "1": 0.8608490824699402, "3": 0.9410377144813538, "5": 0.9571540355682373 }, "all_support_em_at_k": { "3": 0.9150943160057068, "5": 0.9316037893295288 } } } }, "by_task": { "change_comparison": { "query_count": 470, "recall_at_k": { "1": 0.42092201113700867, "3": 0.8836879134178162, "5": 0.9523048996925354 }, "all_support_em_at_k": { "3": 0.7936170101165771, "5": 0.9063829779624939 } }, "direct_recall": { "query_count": 1491, "recall_at_k": { "1": 0.8428906202316284, "3": 0.9219203591346741, "5": 0.938296377658844 }, "all_support_em_at_k": { "3": 0.8980550169944763, "5": 0.9141515493392944 } }, "historical_state": { "query_count": 500, "recall_at_k": { "1": 0.8059999942779541, "3": 0.996999979019165, "5": 0.9990000128746033 }, "all_support_em_at_k": { "3": 0.9959999918937683, "5": 0.9980000257492065 } }, "latest_state": { "query_count": 457, "recall_at_k": { "1": 0.4124726355075836, "3": 0.9617067575454712, "5": 0.9989058971405029 }, "all_support_em_at_k": { "3": 0.9606127142906189, "5": 0.9978117942810059 } }, "multi_hop_composition": { "query_count": 348, "recall_at_k": { "1": 0.32254037261009216, "3": 0.7030788064002991, "5": 0.8970990180969238 }, "all_support_em_at_k": { "3": 0.4597701132297516, "5": 0.8218390941619873 } }, "temporal_localization": { "query_count": 719, "recall_at_k": { "1": 0.8765647411346436, "3": 0.9514371156692505, "5": 0.9655771851539612 }, "all_support_em_at_k": { "3": 0.9304589629173279, "5": 0.9457579851150513 } } }, "semantic_only": { "query_count": 3985, "recall_at_k": { "1": 0.640546441078186, "3": 0.907759428024292, "5": 0.9557926058769226 }, "all_support_em_at_k": { "3": 0.8619824051856995, "5": 0.9309912323951721 }, "by_speaker_query_type": { "fact_to_person": { "query_count": 786, "recall_at_k": { "1": 0.7862187027931213, "3": 0.971208930015564, "5": 0.9817173480987549 }, "all_support_em_at_k": { "3": 0.9605597853660583, "5": 0.9681933522224426 } }, "person_to_fact": { "query_count": 969, "recall_at_k": { "1": 0.6827927827835083, "3": 0.9407391548156738, "5": 0.9750897288322449 }, "all_support_em_at_k": { "3": 0.910216748714447, "5": 0.9659442901611328 } }, "self_to_fact": { "query_count": 509, "recall_at_k": { "1": 0.23984938859939575, "3": 0.6452193856239319, "5": 0.8904715180397034 }, "all_support_em_at_k": { "3": 0.44007858633995056, "5": 0.8113948702812195 } }, "speaker_independent": { "query_count": 1721, "recall_at_k": { "1": 0.6687390208244324, "3": 0.9378607273101807, "5": 0.9524064660072327 }, "all_support_em_at_k": { "3": 0.9145845174789429, "5": 0.9296920299530029 } } }, "by_task_and_speaker_query_type": { "change_comparison": { "fact_to_person": { "query_count": 74, "recall_at_k": { "1": 0.5, "3": 1.0, "5": 1.0 }, "all_support_em_at_k": { "3": 1.0, "5": 1.0 } }, "person_to_fact": { "query_count": 88, "recall_at_k": { "1": 0.4943181872367859, "3": 0.9602272510528564, "5": 1.0 }, "all_support_em_at_k": { "3": 0.9204545617103577, "5": 1.0 } }, "self_to_fact": { "query_count": 99, "recall_at_k": { "1": 0.1262626200914383, "3": 0.49494948983192444, "5": 0.7929292917251587 }, "all_support_em_at_k": { "3": 0.08080808073282242, "5": 0.5858585834503174 } }, "speaker_independent": { "query_count": 209, "recall_at_k": { "1": 0.48963314294815063, "3": 0.9908294081687927, "5": 0.9932217597961426 }, "all_support_em_at_k": { "3": 0.9856459498405457, "5": 0.9904305934906006 } } }, "direct_recall": { "fact_to_person": { "query_count": 524, "recall_at_k": { "1": 0.9285941123962402, "3": 0.9669210910797119, "5": 0.9820291996002197 }, "all_support_em_at_k": { "3": 0.9561068415641785, "5": 0.9675572514533997 } }, "person_to_fact": { "query_count": 306, "recall_at_k": { "1": 0.8153594732284546, "3": 0.9360021352767944, "5": 0.9539759755134583 }, "all_support_em_at_k": { "3": 0.9248365759849548, "5": 0.9379084706306458 } }, "self_to_fact": { "query_count": 57, "recall_at_k": { "1": 0.41374269127845764, "3": 0.5716374516487122, "5": 0.6856725811958313 }, "all_support_em_at_k": { "3": 0.4035087823867798, "5": 0.5614035129547119 } }, "speaker_independent": { "query_count": 604, "recall_at_k": { "1": 0.8113962411880493, "3": 0.9121137261390686, "5": 0.9201158881187439 }, "all_support_em_at_k": { "3": 0.8841059803962708, "5": 0.8923841118812561 } } }, "historical_state": { "fact_to_person": { "query_count": 91, "recall_at_k": { "1": 0.5494505167007446, "3": 1.0, "5": 1.0 }, "all_support_em_at_k": { "3": 1.0, "5": 1.0 } }, "person_to_fact": { "query_count": 125, "recall_at_k": { "1": 0.5519999861717224, "3": 0.9919999837875366, "5": 1.0 }, "all_support_em_at_k": { "3": 0.9919999837875366, "5": 1.0 } }, "self_to_fact": { "query_count": 82, "recall_at_k": { "1": 0.2195121943950653, "3": 0.7560975551605225, "5": 0.9878048896789551 }, "all_support_em_at_k": { "3": 0.7560975551605225, "5": 0.9878048896789551 } }, "speaker_independent": { "query_count": 202, "recall_at_k": { "1": 0.6188119053840637, "3": 0.9925742745399475, "5": 0.9975247383117676 }, "all_support_em_at_k": { "3": 0.9900990128517151, "5": 0.9950494766235352 } } }, "latest_state": { "fact_to_person": { "query_count": 31, "recall_at_k": { "1": 0.4516128897666931, "3": 1.0, "5": 1.0 }, "all_support_em_at_k": { "3": 1.0, "5": 1.0 } }, "person_to_fact": { "query_count": 92, "recall_at_k": { "1": 0.21195651590824127, "3": 0.97826087474823, "5": 1.0 }, "all_support_em_at_k": { "3": 0.97826087474823, "5": 1.0 } }, "self_to_fact": { "query_count": 82, "recall_at_k": { "1": 0.1463414579629898, "3": 0.6829268336296082, "5": 0.9756097197532654 }, "all_support_em_at_k": { "3": 0.6829268336296082, "5": 0.9756097197532654 } }, "speaker_independent": { "query_count": 252, "recall_at_k": { "1": 0.2460317462682724, "3": 0.9583333134651184, "5": 0.9980158805847168 }, "all_support_em_at_k": { "3": 0.9563491940498352, "5": 0.9960317611694336 } } }, "multi_hop_composition": { "fact_to_person": { "query_count": 66, "recall_at_k": { "1": 0.46037158370018005, "3": 0.9197511076927185, "5": 0.9249458909034729 }, "all_support_em_at_k": { "3": 0.8787878751754761, "5": 0.8787878751754761 } }, "person_to_fact": { "query_count": 84, "recall_at_k": { "1": 0.30209752917289734, "3": 0.6774943470954895, "5": 0.9189909100532532 }, "all_support_em_at_k": { "3": 0.4166666567325592, "5": 0.8809523582458496 } }, "self_to_fact": { "query_count": 168, "recall_at_k": { "1": 0.2678571343421936, "3": 0.675595223903656, "5": 0.9375 }, "all_support_em_at_k": { "3": 0.375, "5": 0.8809523582458496 } }, "speaker_independent": { "query_count": 30, "recall_at_k": { "1": 0.24944445490837097, "3": 0.5019444823265076, "5": 0.5641666650772095 }, "all_support_em_at_k": { "3": 0.1666666716337204, "5": 0.23333333432674408 } } }, "temporal_localization": { "person_to_fact": { "query_count": 274, "recall_at_k": { "1": 0.92974454164505, "3": 0.9844890236854553, "5": 0.9881386756896973 }, "all_support_em_at_k": { "3": 0.9817517995834351, "5": 0.985401451587677 } }, "self_to_fact": { "query_count": 21, "recall_at_k": { "1": 0.5238094925880432, "3": 0.7301587462425232, "5": 0.8174603581428528 }, "all_support_em_at_k": { "3": 0.5714285969734192, "5": 0.6666666865348816 } }, "speaker_independent": { "query_count": 424, "recall_at_k": { "1": 0.8584905862808228, "3": 0.9410377144813538, "5": 0.9571540355682373 }, "all_support_em_at_k": { "3": 0.9150943160057068, "5": 0.9316037893295288 } } } }, "by_task": { "change_comparison": { "query_count": 470, "recall_at_k": { "1": 0.4156028628349304, "3": 0.8820921778678894, "5": 0.9533687233924866 }, "all_support_em_at_k": { "3": 0.785106360912323, "5": 0.9085106253623962 } }, "direct_recall": { "query_count": 1491, "recall_at_k": { "1": 0.8381957411766052, "3": 0.9232617616653442, "5": 0.9398613572120667 }, "all_support_em_at_k": { "3": 0.8993963599205017, "5": 0.9154929518699646 } }, "historical_state": { "query_count": 500, "recall_at_k": { "1": 0.5239999890327454, "3": 0.9549999833106995, "5": 0.996999979019165 }, "all_support_em_at_k": { "3": 0.9539999961853027, "5": 0.9959999918937683 } }, "latest_state": { "query_count": 457, "recall_at_k": { "1": 0.2352297604084015, "3": 0.9157549142837524, "5": 0.9945295453071594 }, "all_support_em_at_k": { "3": 0.9146608114242554, "5": 0.9934354424476624 } }, "multi_hop_composition": { "query_count": 348, "recall_at_k": { "1": 0.31104612350463867, "3": 0.7073891758918762, "5": 0.8984674215316772 }, "all_support_em_at_k": { "3": 0.46264368295669556, "5": 0.8247126340866089 } }, "temporal_localization": { "query_count": 719, "recall_at_k": { "1": 0.8758693337440491, "3": 0.9514371156692505, "5": 0.9648817777633667 }, "all_support_em_at_k": { "3": 0.9304589629173279, "5": 0.9443671703338623 } } } }, "skipped_control_query_count": 0, "score_analysis": { "overall": { "rag_query_count": 3985, "control_query_count": 111, "gold_score": { "count": 5141, "mean": 0.7166337926745187, "std": 0.17733149515563457, "min": -0.11191169917583466, "p05": 0.3657482862472534, "p25": 0.6389928460121155, "median": 0.7480946779251099, "p75": 0.827989399433136, "p95": 0.9548864960670471, "max": 1.126090168952942 }, "best_gold_score": { "count": 3985, "mean": 0.755534760330522, "std": 0.14572272433142164, "min": 0.024165567010641098, "p05": 0.490088027715683, "p25": 0.6786190867424011, "median": 0.7751450538635254, "p75": 0.8467850089073181, "p95": 0.9710816621780396, "max": 1.126090168952942 }, "rag_max_candidate_score": { "count": 3985, "mean": 0.7640589419130699, "std": 0.13383487103235178, "min": 0.20186400413513184, "p05": 0.5207033753395081, "p25": 0.6838327050209045, "median": 0.778085470199585, "p75": 0.8501520752906799, "p95": 0.9734190702438353, "max": 1.129429817199707 }, "control_max_candidate_score": { "count": 111, "mean": 0.3709662235635626, "std": 0.15236881273689565, "min": 0.019797351211309433, "p05": 0.13829854875802994, "p25": 0.29179131984710693, "median": 0.3588598668575287, "p75": 0.46799521148204803, "p95": 0.6187787652015686, "max": 0.8129850029945374 }, "max_non_gold_score": { "count": 3985, "mean": 0.6232047809355681, "std": 0.1810787727754309, "min": 0.08055949211120605, "p05": 0.3426193714141846, "p25": 0.4937731623649597, "median": 0.6113350987434387, "p75": 0.746073842048645, "p95": 0.9490036487579345, "max": 1.129429817199707 }, "query_temporal_norm": { "count": 4096, "mean": 0.30273059851970174, "std": 0.14540258547355747, "min": 0.09085708111524582, "p05": 0.12492499314248562, "p25": 0.18279167264699936, "median": 0.24912824481725693, "p75": 0.4182180389761925, "p95": 0.5543802678585052, "max": 0.5741856694221497 }, "best_gold_margin": { "count": 3985, "mean": 0.132329979394954, "std": 0.1506653521072949, "min": -0.46813422441482544, "p05": -0.05175083875656128, "p25": 0.0041754841804504395, "median": 0.11015433073043823, "p75": 0.23661231994628906, "p95": 0.4035195529460907, "max": 0.7237380295991898 }, "weakest_gold_margin": { "count": 3985, "mean": 0.10693937787837834, "std": 0.16802301214783116, "min": -0.6715246737003326, "p05": -0.13616411089897157, "p25": -0.0015456676483154297, "median": 0.09505456686019897, "p75": 0.22502261400222778, "p95": 0.37765879631042476, "max": 0.7037184834480286 }, "top_score_best_balanced_threshold": { "threshold": 0.5420975983142853, "balanced_accuracy": 0.9165734115545909, "true_positive_rate": 0.9322459222082811, "false_positive_rate": 0.0990990990990991 }, "gold_score_best_balanced_threshold": { "threshold": 0.5420975983142853, "balanced_accuracy": 0.9085432986311279, "true_positive_rate": 0.916185696361355, "false_positive_rate": 0.0990990990990991 } }, "by_task": { "change_comparison": { "rag_query_count": 470, "control_query_count": 0, "gold_score": { "count": 942, "mean": 0.7348885870520856, "std": 0.1107366505292103, "min": 0.06459831446409225, "p05": 0.5372704863548279, "p25": 0.6722842156887054, "median": 0.7520450353622437, "p75": 0.8101696819067001, "p95": 0.8841885775327682, "max": 0.978650689125061 }, "best_gold_score": { "count": 470, "mean": 0.7707346021494967, "std": 0.09284867084255541, "min": 0.28962263464927673, "p05": 0.6213128983974456, "p25": 0.7128853797912598, "median": 0.7886177897453308, "p75": 0.83125239610672, "p95": 0.8947178870439529, "max": 0.978650689125061 }, "rag_max_candidate_score": { "count": 470, "mean": 0.7754683313217569, "std": 0.09230104027153883, "min": 0.28962263464927673, "p05": 0.6218573927879334, "p25": 0.7183136194944382, "median": 0.7937867343425751, "p75": 0.8351891040802002, "p95": 0.8965094208717346, "max": 0.978650689125061 }, "control_max_candidate_score": null, "max_non_gold_score": { "count": 470, "mean": 0.5437221028227756, "std": 0.19519320044141864, "min": 0.08743387460708618, "p05": 0.24671123027801514, "p25": 0.3817346394062042, "median": 0.5233292877674103, "p75": 0.710181474685669, "p95": 0.8506679207086563, "max": 0.9478694796562195 }, "query_temporal_norm": { "count": 470, "mean": 0.3904190681716229, "std": 0.044412626312598104, "min": 0.1604093760251999, "p05": 0.3008479714393616, "p25": 0.374271996319294, "median": 0.3975691944360733, "p75": 0.41577866673469543, "p95": 0.44142436981201166, "max": 0.5686288475990295 }, "best_gold_margin": { "count": 470, "mean": 0.22701249932672116, "std": 0.18460189504901048, "min": -0.09962069988250732, "p05": -0.03665072619915009, "p25": 0.06973769888281822, "median": 0.22310630977153778, "p75": 0.374038502573967, "p95": 0.5218270637094974, "max": 0.7237380295991898 }, "weakest_gold_margin": { "count": 470, "mean": 0.1590447726084831, "std": 0.18752205523336984, "min": -0.40735045820474625, "p05": -0.11907580196857452, "p25": 0.010347530245780945, "median": 0.15443812310695648, "p75": 0.317648746073246, "p95": 0.4485425367951393, "max": 0.6372086405754089 }, "top_score_best_balanced_threshold": null, "gold_score_best_balanced_threshold": null }, "current_only_control": { "rag_query_count": 0, "control_query_count": 110, "gold_score": null, "best_gold_score": null, "rag_max_candidate_score": null, "control_max_candidate_score": { "count": 110, "mean": 0.3699303682385521, "std": 0.15267026355881047, "min": 0.019797351211309433, "p05": 0.13738513067364694, "p25": 0.29039132595062256, "median": 0.3570753037929535, "p75": 0.46439164876937866, "p95": 0.6191292226314544, "max": 0.8129850029945374 }, "max_non_gold_score": null, "query_temporal_norm": { "count": 110, "mean": 0.2795836961404844, "std": 0.1029300128190954, "min": 0.11468714475631714, "p05": 0.12837647050619125, "p25": 0.1998436488211155, "median": 0.2662823051214218, "p75": 0.361461378633976, "p95": 0.4507403194904327, "max": 0.5361620187759399 }, "best_gold_margin": null, "weakest_gold_margin": null, "top_score_best_balanced_threshold": null, "gold_score_best_balanced_threshold": null }, "direct_recall": { "rag_query_count": 1491, "control_query_count": 0, "gold_score": { "count": 1669, "mean": 0.6795901385472988, "std": 0.2012548187301753, "min": -0.08538651466369629, "p05": 0.2746641576290131, "p25": 0.5686708092689514, "median": 0.7269337177276611, "p75": 0.8230971097946167, "p95": 0.9368460059165953, "max": 1.0985771417617798 }, "best_gold_score": { "count": 1491, "mean": 0.7201849106351934, "std": 0.1623488035209848, "min": 0.024165567010641098, "p05": 0.43137773871421814, "p25": 0.6312223970890045, "median": 0.7476219534873962, "p75": 0.8330177068710327, "p95": 0.9440494775772095, "max": 1.0985771417617798 }, "rag_max_candidate_score": { "count": 1491, "mean": 0.7322370440247553, "std": 0.14161953978055097, "min": 0.20186400413513184, "p05": 0.4857248365879059, "p25": 0.6375583410263062, "median": 0.7500801086425781, "p75": 0.8338922262191772, "p95": 0.9440494775772095, "max": 1.0985771417617798 }, "control_max_candidate_score": null, "max_non_gold_score": { "count": 1491, "mean": 0.5614989415200903, "std": 0.1312775402625397, "min": 0.08055949211120605, "p05": 0.34560634195804596, "p25": 0.47706902027130127, "median": 0.5617488026618958, "p75": 0.6446017622947693, "p95": 0.7837359011173248, "max": 0.9488265514373779 }, "query_temporal_norm": { "count": 1491, "mean": 0.2243291862024148, "std": 0.09722126786241793, "min": 0.09085708111524582, "p05": 0.11080540344119072, "p25": 0.15130296349525452, "median": 0.2039969563484192, "p75": 0.26913149654865265, "p95": 0.4267657548189163, "max": 0.5707007646560669 }, "best_gold_margin": { "count": 1491, "mean": 0.1586859691151031, "std": 0.15200214138339124, "min": -0.46813422441482544, "p05": -0.09121431410312653, "p25": 0.06090536713600159, "median": 0.1574101448059082, "p75": 0.2646420896053314, "p95": 0.4028832018375397, "max": 0.7037184834480286 }, "weakest_gold_margin": { "count": 1491, "mean": 0.1412749828050559, "std": 0.18043582198802646, "min": -0.6715246737003326, "p05": -0.19026869535446167, "p25": 0.05169305205345154, "median": 0.15304303169250488, "p75": 0.2643876075744629, "p95": 0.4028832018375397, "max": 0.7037184834480286 }, "top_score_best_balanced_threshold": null, "gold_score_best_balanced_threshold": null }, "historical_state": { "rag_query_count": 500, "control_query_count": 0, "gold_score": { "count": 502, "mean": 0.7746741928130507, "std": 0.10766275179939297, "min": 0.14238934218883514, "p05": 0.5862048655748368, "p25": 0.7167049050331116, "median": 0.793297290802002, "p75": 0.8495824038982391, "p95": 0.9157554745674134, "max": 0.9862993359565735 }, "best_gold_score": { "count": 500, "mean": 0.7764696861505509, "std": 0.1034113203801674, "min": 0.3909410536289215, "p05": 0.5930841654539108, "p25": 0.7181234359741211, "median": 0.7944096028804779, "p75": 0.8500040471553802, "p95": 0.9159746468067168, "max": 0.9862993359565735 }, "rag_max_candidate_score": { "count": 500, "mean": 0.7806070187091827, "std": 0.10195340881369842, "min": 0.3909410536289215, "p05": 0.5955297142267227, "p25": 0.722739964723587, "median": 0.7985008358955383, "p75": 0.8531655073165894, "p95": 0.9197906970977783, "max": 0.9862993359565735 }, "control_max_candidate_score": null, "max_non_gold_score": { "count": 500, "mean": 0.662049885392189, "std": 0.14984154422872092, "min": 0.15823492407798767, "p05": 0.4040971785783768, "p25": 0.5531154274940491, "median": 0.6717543303966522, "p75": 0.7823225259780884, "p95": 0.8810094028711318, "max": 0.9847855567932129 }, "query_temporal_norm": { "count": 500, "mean": 0.4881072618365288, "std": 0.10037095674200244, "min": 0.13309770822525024, "p05": 0.2568978860974312, "p25": 0.4548801779747009, "median": 0.5330244600772858, "p75": 0.5557828396558762, "p95": 0.5690453529357911, "max": 0.5741856694221497 }, "best_gold_margin": { "count": 500, "mean": 0.11441980075836182, "std": 0.12297352134656454, "min": -0.19667965173721313, "p05": -0.029210487008094786, "p25": 0.003396078944206238, "median": 0.1014467179775238, "p75": 0.19494780898094177, "p95": 0.33460708707571024, "max": 0.4947989881038666 }, "weakest_gold_margin": { "count": 500, "mean": 0.11334651419520378, "std": 0.12532201212328375, "min": -0.42393268644809723, "p05": -0.029937261343002307, "p25": 0.003252282738685608, "median": 0.1014467179775238, "p75": 0.19494780898094177, "p95": 0.33460708707571024, "max": 0.4947989881038666 }, "top_score_best_balanced_threshold": null, "gold_score_best_balanced_threshold": null }, "latest_state": { "rag_query_count": 457, "control_query_count": 0, "gold_score": { "count": 460, "mean": 0.9097354384544103, "std": 0.11280339007167291, "min": 0.2454087883234024, "p05": 0.6985023379325866, "p25": 0.8598119914531708, "median": 0.9304486811161041, "p75": 0.9877870827913284, "p95": 1.0455342650413513, "max": 1.126090168952942 }, "best_gold_score": { "count": 457, "mean": 0.9125979087219718, "std": 0.10705335656055096, "min": 0.2454087883234024, "p05": 0.7246195793151855, "p25": 0.8614522218704224, "median": 0.9308992028236389, "p75": 0.9884832501411438, "p95": 1.0457399368286133, "max": 1.126090168952942 }, "rag_max_candidate_score": { "count": 457, "mean": 0.9189921461648962, "std": 0.10519501128335003, "min": 0.3255910277366638, "p05": 0.7326323986053467, "p25": 0.867649495601654, "median": 0.9345877766609192, "p75": 0.9926048517227173, "p95": 1.0541483640670777, "max": 1.129429817199707 }, "control_max_candidate_score": null, "max_non_gold_score": { "count": 457, "mean": 0.9165826158137499, "std": 0.10844750208585242, "min": 0.3255910277366638, "p05": 0.7243556141853332, "p25": 0.8655767440795898, "median": 0.9336287379264832, "p75": 0.990559995174408, "p95": 1.0539359092712401, "max": 1.129429817199707 }, "query_temporal_norm": { "count": 457, "mean": 0.5075552688190139, "std": 0.0713984345837642, "min": 0.16853883862495422, "p05": 0.34919045567512513, "p25": 0.5019967555999756, "median": 0.5324681401252747, "p75": 0.5476772785186768, "p95": 0.5631026864051819, "max": 0.5706198215484619 }, "best_gold_margin": { "count": 457, "mean": -0.0039847070917780605, "std": 0.020039605127652755, "min": -0.1066666841506958, "p05": -0.04410884380340576, "p25": -0.005139470100402832, "median": -0.0010437369346618652, "p75": 0.0023041367530822754, "p95": 0.008276724815368652, "max": 0.21554291248321533 }, "weakest_gold_margin": { "count": 457, "mean": -0.004690558468226009, "std": 0.016042674602134533, "min": -0.1066666841506958, "p05": -0.04410884380340576, "p25": -0.005158364772796631, "median": -0.0011911392211914062, "p75": 0.0022735595703125, "p95": 0.008188319206237792, "max": 0.1005626916885376 }, "top_score_best_balanced_threshold": null, "gold_score_best_balanced_threshold": null }, "multi_hop_composition": { "rag_query_count": 348, "control_query_count": 0, "gold_score": { "count": 773, "mean": 0.6499942918171407, "std": 0.1690562525488606, "min": -0.11191169917583466, "p05": 0.2923929452896118, "p25": 0.6014822125434875, "median": 0.6969712376594543, "p75": 0.7639080286026001, "p95": 0.8181564331054687, "max": 0.8747819662094116 }, "best_gold_score": { "count": 348, "mean": 0.7210219436507116, "std": 0.09544151448010416, "min": 0.34543609619140625, "p05": 0.5357065349817276, "p25": 0.6784490048885345, "median": 0.7423729598522186, "p75": 0.7895018309354782, "p95": 0.8334888577461242, "max": 0.8747819662094116 }, "rag_max_candidate_score": { "count": 348, "mean": 0.7313055699241573, "std": 0.08715730799173863, "min": 0.4189240634441376, "p05": 0.5543014436960221, "p25": 0.6850295513868332, "median": 0.7491745948791504, "p75": 0.7964175790548325, "p95": 0.8351482093334198, "max": 0.8747819662094116 }, "control_max_candidate_score": null, "max_non_gold_score": { "count": 348, "mean": 0.6516784167495268, "std": 0.14944315060991253, "min": 0.15956036746501923, "p05": 0.3637311488389969, "p25": 0.5521483868360519, "median": 0.7000037133693695, "p75": 0.771206796169281, "p95": 0.823851415514946, "max": 0.8726747035980225 }, "query_temporal_norm": { "count": 348, "mean": 0.19508305329015885, "std": 0.03477950168836244, "min": 0.13012589514255524, "p05": 0.14681573063135148, "p25": 0.16908932477235794, "median": 0.191594660282135, "p75": 0.21390236169099808, "p95": 0.2606314122676849, "max": 0.3217720091342926 }, "best_gold_margin": { "count": 348, "mean": 0.06934352690118482, "std": 0.12927293245479557, "min": -0.18237540125846863, "p05": -0.05678632855415344, "p25": -0.011113137006759644, "median": 0.01789069175720215, "p75": 0.0949355959892273, "p95": 0.3652027100324629, "max": 0.5515869557857513 }, "weakest_gold_margin": { "count": 348, "mean": -0.01585723184655961, "std": 0.16034559704635334, "min": -0.6282003968954086, "p05": -0.2646480455994606, "p25": -0.07698842138051987, "median": -0.03306916356086731, "p75": 0.008552595973014832, "p95": 0.3040800869464872, "max": 0.5013379752635956 }, "top_score_best_balanced_threshold": null, "gold_score_best_balanced_threshold": null }, "recent_context_control": { "rag_query_count": 0, "control_query_count": 1, "gold_score": null, "best_gold_score": null, "rag_max_candidate_score": null, "control_max_candidate_score": { "count": 1, "mean": 0.4849103093147278, "std": 0.0, "min": 0.4849103093147278, "p05": 0.4849103093147278, "p25": 0.4849103093147278, "median": 0.4849103093147278, "p75": 0.4849103093147278, "p95": 0.4849103093147278, "max": 0.4849103093147278 }, "max_non_gold_score": null, "query_temporal_norm": { "count": 1, "mean": 0.17241171002388, "std": 0.0, "min": 0.17241171002388, "p05": 0.17241171002388, "p25": 0.17241171002388, "median": 0.17241171002388, "p75": 0.17241171002388, "p95": 0.17241171002388, "max": 0.17241171002388 }, "best_gold_margin": null, "weakest_gold_margin": null, "top_score_best_balanced_threshold": null, "gold_score_best_balanced_threshold": null }, "temporal_localization": { "rag_query_count": 719, "control_query_count": 0, "gold_score": { "count": 795, "mean": 0.6891861683589482, "std": 0.1710879451771218, "min": -0.10147331655025482, "p05": 0.347307488322258, "p25": 0.6123415529727936, "median": 0.7237328290939331, "p75": 0.8111500144004822, "p95": 0.8972467362880706, "max": 1.0365782976150513 }, "best_gold_score": { "count": 719, "mean": 0.7212202105676349, "std": 0.13338234880123853, "min": 0.12394365668296814, "p05": 0.4685472249984741, "p25": 0.6464675962924957, "median": 0.7408140301704407, "p75": 0.8183608055114746, "p95": 0.9048629462718963, "max": 1.0365782976150513 }, "rag_max_candidate_score": { "count": 719, "mean": 0.7284590774351765, "std": 0.12147799906182175, "min": 0.2631508409976959, "p05": 0.5057044625282288, "p25": 0.6554280817508698, "median": 0.7423368692398071, "p75": 0.818709522485733, "p95": 0.9048629462718963, "max": 1.0365782976150513 }, "control_max_candidate_score": null, "max_non_gold_score": { "count": 719, "mean": 0.5758545963049598, "std": 0.12951872399452016, "min": 0.2200310379266739, "p05": 0.3654164135456085, "p25": 0.474811851978302, "median": 0.5770975351333618, "p75": 0.6710145175457001, "p95": 0.7880617022514345, "max": 0.9568344950675964 }, "query_temporal_norm": { "count": 719, "mean": 0.20471605461645193, "std": 0.05866639582901708, "min": 0.10267208516597748, "p05": 0.12938569486141205, "p25": 0.1643894836306572, "median": 0.19598978757858276, "p75": 0.22830134630203247, "p95": 0.31625273823738104, "max": 0.5251786112785339 }, "best_gold_margin": { "count": 719, "mean": 0.14536561426267505, "std": 0.12385513267341426, "min": -0.3723912537097931, "p05": -0.04747639298439026, "p25": 0.07192777097225189, "median": 0.1291063129901886, "p75": 0.2247702181339264, "p95": 0.35997496843338017, "max": 0.5334089994430542 }, "weakest_gold_margin": { "count": 719, "mean": 0.12760782080944705, "std": 0.15015979413924416, "min": -0.5440109521150589, "p05": -0.1358781337738037, "p25": 0.057646214962005615, "median": 0.12182247638702393, "p75": 0.22186627984046936, "p95": 0.3576861590147019, "max": 0.5334089994430542 }, "top_score_best_balanced_threshold": null, "gold_score_best_balanced_threshold": null } } } } }, { "epoch": 2, "checkpoint": "/home/zhangchao/work/minye_training/mixed_retrieval_normal_identity_balanced/retriever_epoch_02.pt", "train_loss": 0.7708213925361633, "train_losses": { "total": 0.7708213925361633, "global": 1.041038990020752, "hard_negative": 0.5432254076004028, "semantic": 0.7572951316833496, "temporal_null": 0.041718825697898865, "speaker_name_renamed_samples": 639, "speaker_name_skipped_mixed_or_nonself": 4108, "speaker_name_skipped_spoken_name": 267, "speaker_name_skipped_insufficient_names": 0 }, "validation": { "query_count": 3985, "recall_at_k": { "1": 0.7014623880386353, "3": 0.9176109433174133, "5": 0.9573962092399597 }, "all_support_em_at_k": { "3": 0.8742785453796387, "5": 0.9345043897628784 }, "by_speaker_query_type": { "fact_to_person": { "query_count": 786, "recall_at_k": { "1": 0.8212059736251831, "3": 0.97481369972229, "5": 0.9802631735801697 }, "all_support_em_at_k": { "3": 0.9631043076515198, "5": 0.9656488299369812 } }, "person_to_fact": { "query_count": 969, "recall_at_k": { "1": 0.749356210231781, "3": 0.9477566480636597, "5": 0.9758120775222778 }, "all_support_em_at_k": { "3": 0.9205366373062134, "5": 0.968008279800415 } }, "self_to_fact": { "query_count": 509, "recall_at_k": { "1": 0.28176164627075195, "3": 0.6972822546958923, "5": 0.9101179242134094 }, "all_support_em_at_k": { "3": 0.5009823441505432, "5": 0.8408644199371338 } }, "speaker_independent": { "query_count": 1721, "recall_at_k": { "1": 0.7439375519752502, "3": 0.9396765232086182, "5": 0.9505664706230164 }, "all_support_em_at_k": { "3": 0.918070912361145, "5": 0.9291110038757324 } } }, "by_task_and_speaker_query_type": { "change_comparison": { "fact_to_person": { "query_count": 74, "recall_at_k": { "1": 0.5, "3": 1.0, "5": 1.0 }, "all_support_em_at_k": { "3": 1.0, "5": 1.0 } }, "person_to_fact": { "query_count": 88, "recall_at_k": { "1": 0.5, "3": 0.9602272510528564, "5": 1.0 }, "all_support_em_at_k": { "3": 0.9204545617103577, "5": 1.0 } }, "self_to_fact": { "query_count": 99, "recall_at_k": { "1": 0.13131313025951385, "3": 0.5101010203361511, "5": 0.8282828330993652 }, "all_support_em_at_k": { "3": 0.13131313025951385, "5": 0.6565656661987305 } }, "speaker_independent": { "query_count": 209, "recall_at_k": { "1": 0.49441784620285034, "3": 0.9920255541801453, "5": 0.9932217597961426 }, "all_support_em_at_k": { "3": 0.9904305934906006, "5": 0.9904305934906006 } } }, "direct_recall": { "fact_to_person": { "query_count": 524, "recall_at_k": { "1": 0.9276399612426758, "3": 0.9723281860351562, "5": 0.980120837688446 }, "all_support_em_at_k": { "3": 0.9599236249923706, "5": 0.9637404680252075 } }, "person_to_fact": { "query_count": 306, "recall_at_k": { "1": 0.813725471496582, "3": 0.9327341914176941, "5": 0.9507080316543579 }, "all_support_em_at_k": { "3": 0.9215686321258545, "5": 0.9346405267715454 } }, "self_to_fact": { "query_count": 57, "recall_at_k": { "1": 0.42836257815361023, "3": 0.6067252159118652, "5": 0.7032163739204407 }, "all_support_em_at_k": { "3": 0.4385964870452881, "5": 0.5789473652839661 } }, "speaker_independent": { "query_count": 604, "recall_at_k": { "1": 0.8134657144546509, "3": 0.9046633839607239, "5": 0.9165287613868713 }, "all_support_em_at_k": { "3": 0.8791390657424927, "5": 0.8923841118812561 } } }, "historical_state": { "fact_to_person": { "query_count": 91, "recall_at_k": { "1": 0.8351648449897766, "3": 1.0, "5": 1.0 }, "all_support_em_at_k": { "3": 1.0, "5": 1.0 } }, "person_to_fact": { "query_count": 125, "recall_at_k": { "1": 0.871999979019165, "3": 1.0, "5": 1.0 }, "all_support_em_at_k": { "3": 1.0, "5": 1.0 } }, "self_to_fact": { "query_count": 82, "recall_at_k": { "1": 0.3658536672592163, "3": 0.9512194991111755, "5": 1.0 }, "all_support_em_at_k": { "3": 0.9512194991111755, "5": 1.0 } }, "speaker_independent": { "query_count": 202, "recall_at_k": { "1": 0.9207921028137207, "3": 0.9975247383117676, "5": 0.9975247383117676 }, "all_support_em_at_k": { "3": 0.9950494766235352, "5": 0.9950494766235352 } } }, "latest_state": { "fact_to_person": { "query_count": 31, "recall_at_k": { "1": 0.5161290168762207, "3": 1.0, "5": 1.0 }, "all_support_em_at_k": { "3": 1.0, "5": 1.0 } }, "person_to_fact": { "query_count": 92, "recall_at_k": { "1": 0.5163043737411499, "3": 1.0, "5": 1.0 }, "all_support_em_at_k": { "3": 1.0, "5": 1.0 } }, "self_to_fact": { "query_count": 82, "recall_at_k": { "1": 0.24390242993831635, "3": 0.7682926654815674, "5": 1.0 }, "all_support_em_at_k": { "3": 0.7682926654815674, "5": 1.0 } }, "speaker_independent": { "query_count": 252, "recall_at_k": { "1": 0.5198412537574768, "3": 0.9900793433189392, "5": 0.9980158805847168 }, "all_support_em_at_k": { "3": 0.988095223903656, "5": 0.9960317611694336 } } }, "multi_hop_composition": { "fact_to_person": { "query_count": 66, "recall_at_k": { "1": 0.46037158370018005, "3": 0.9197511076927185, "5": 0.9227814078330994 }, "all_support_em_at_k": { "3": 0.8787878751754761, "5": 0.8787878751754761 } }, "person_to_fact": { "query_count": 84, "recall_at_k": { "1": 0.23662133514881134, "3": 0.722732424736023, "5": 0.9392289519309998 }, "all_support_em_at_k": { "3": 0.5, "5": 0.9166666865348816 } }, "self_to_fact": { "query_count": 168, "recall_at_k": { "1": 0.2678571343421936, "3": 0.675595223903656, "5": 0.9523809552192688 }, "all_support_em_at_k": { "3": 0.380952388048172, "5": 0.9047619104385376 } }, "speaker_independent": { "query_count": 30, "recall_at_k": { "1": 0.24944445490837097, "3": 0.4811111390590668, "5": 0.5475000143051147 }, "all_support_em_at_k": { "3": 0.1666666716337204, "5": 0.23333333432674408 } } }, "temporal_localization": { "person_to_fact": { "query_count": 274, "recall_at_k": { "1": 0.9370437860488892, "3": 0.9881386756896973, "5": 0.9881386756896973 }, "all_support_em_at_k": { "3": 0.985401451587677, "5": 0.985401451587677 } }, "self_to_fact": { "query_count": 21, "recall_at_k": { "1": 0.5238094925880432, "3": 0.7301587462425232, "5": 0.8174603581428528 }, "all_support_em_at_k": { "3": 0.5714285969734192, "5": 0.6666666865348816 } }, "speaker_independent": { "query_count": 424, "recall_at_k": { "1": 0.8518081307411194, "3": 0.9386793375015259, "5": 0.9559748768806458 }, "all_support_em_at_k": { "3": 0.9127358198165894, "5": 0.9292452931404114 } } } }, "by_task": { "change_comparison": { "query_count": 470, "recall_at_k": { "1": 0.4198581874370575, "3": 0.8858155608177185, "5": 0.9608155488967896 }, "all_support_em_at_k": { "3": 0.7978723645210266, "5": 0.9234042763710022 } }, "direct_recall": { "query_count": 1491, "recall_at_k": { "1": 0.838922381401062, "3": 0.9228145480155945, "5": 0.9377375245094299 }, "all_support_em_at_k": { "3": 0.8993963599205017, "5": 0.9141515493392944 } }, "historical_state": { "query_count": 500, "recall_at_k": { "1": 0.8019999861717224, "3": 0.9909999966621399, "5": 0.9990000128746033 }, "all_support_em_at_k": { "3": 0.9900000095367432, "5": 0.9980000257492065 } }, "latest_state": { "query_count": 457, "recall_at_k": { "1": 0.4693654179573059, "3": 0.9529540538787842, "5": 0.9989058971405029 }, "all_support_em_at_k": { "3": 0.9518599510192871, "5": 0.9978117942810059 } }, "multi_hop_composition": { "query_count": 348, "recall_at_k": { "1": 0.29524150490760803, "3": 0.7165126800537109, "5": 0.9086891412734985 }, "all_support_em_at_k": { "3": 0.48563218116760254, "5": 0.8448275923728943 } }, "temporal_localization": { "query_count": 719, "recall_at_k": { "1": 0.874710202217102, "3": 0.9514370560646057, "5": 0.9641863703727722 }, "all_support_em_at_k": { "3": 0.9304589629173279, "5": 0.9429763555526733 } } }, "semantic_only": { "query_count": 3985, "recall_at_k": { "1": 0.647217333316803, "3": 0.9038509726524353, "5": 0.9575216770172119 }, "all_support_em_at_k": { "3": 0.8592220544815063, "5": 0.9340025186538696 }, "by_speaker_query_type": { "fact_to_person": { "query_count": 786, "recall_at_k": { "1": 0.7639540433883667, "3": 0.9737535119056702, "5": 0.9808993339538574 }, "all_support_em_at_k": { "3": 0.9618320465087891, "5": 0.9669210910797119 } }, "person_to_fact": { "query_count": 969, "recall_at_k": { "1": 0.6972406506538391, "3": 0.9425966739654541, "5": 0.9752960801124573 }, "all_support_em_at_k": { "3": 0.9143446683883667, "5": 0.9669762849807739 } }, "self_to_fact": { "query_count": 509, "recall_at_k": { "1": 0.2503274381160736, "3": 0.6412900686264038, "5": 0.9058611989021301 }, "all_support_em_at_k": { "3": 0.4381139576435089, "5": 0.8330059051513672 } }, "speaker_independent": { "query_count": 1721, "recall_at_k": { "1": 0.6831203103065491, "3": 0.927764892578125, "5": 0.9521159529685974 }, "all_support_em_at_k": { "3": 0.9058687090873718, "5": 0.9302731156349182 } } }, "by_task_and_speaker_query_type": { "change_comparison": { "fact_to_person": { "query_count": 74, "recall_at_k": { "1": 0.5, "3": 1.0, "5": 1.0 }, "all_support_em_at_k": { "3": 1.0, "5": 1.0 } }, "person_to_fact": { "query_count": 88, "recall_at_k": { "1": 0.5, "3": 0.9545454382896423, "5": 1.0 }, "all_support_em_at_k": { "3": 0.9090908765792847, "5": 1.0 } }, "self_to_fact": { "query_count": 99, "recall_at_k": { "1": 0.1262626200914383, "3": 0.47979798913002014, "5": 0.8232322931289673 }, "all_support_em_at_k": { "3": 0.08080808073282242, "5": 0.6464646458625793 } }, "speaker_independent": { "query_count": 209, "recall_at_k": { "1": 0.4920254945755005, "3": 0.9932217597961426, "5": 0.9932217597961426 }, "all_support_em_at_k": { "3": 0.9904305934906006, "5": 0.9904305934906006 } } }, "direct_recall": { "fact_to_person": { "query_count": 524, "recall_at_k": { "1": 0.9276399612426758, "3": 0.970737874507904, "5": 0.9810749888420105 }, "all_support_em_at_k": { "3": 0.9580152630805969, "5": 0.9656488299369812 } }, "person_to_fact": { "query_count": 306, "recall_at_k": { "1": 0.7941176295280457, "3": 0.9360021352767944, "5": 0.9507080316543579 }, "all_support_em_at_k": { "3": 0.9248365759849548, "5": 0.9346405267715454 } }, "self_to_fact": { "query_count": 57, "recall_at_k": { "1": 0.42836257815361023, "3": 0.5716373920440674, "5": 0.7090643644332886 }, "all_support_em_at_k": { "3": 0.38596490025520325, "5": 0.5789473652839661 } }, "speaker_independent": { "query_count": 604, "recall_at_k": { "1": 0.8151213526725769, "3": 0.9050772786140442, "5": 0.9195640683174133 }, "all_support_em_at_k": { "3": 0.8791390657424927, "5": 0.8940397500991821 } } }, "historical_state": { "fact_to_person": { "query_count": 91, "recall_at_k": { "1": 0.4175824224948883, "3": 1.0, "5": 1.0 }, "all_support_em_at_k": { "3": 1.0, "5": 1.0 } }, "person_to_fact": { "query_count": 125, "recall_at_k": { "1": 0.5839999914169312, "3": 0.9839999675750732, "5": 1.0 }, "all_support_em_at_k": { "3": 0.9839999675750732, "5": 1.0 } }, "self_to_fact": { "query_count": 82, "recall_at_k": { "1": 0.24390242993831635, "3": 0.7682926654815674, "5": 1.0 }, "all_support_em_at_k": { "3": 0.7682926654815674, "5": 1.0 } }, "speaker_independent": { "query_count": 202, "recall_at_k": { "1": 0.6732673048973083, "3": 0.9925742745399475, "5": 0.9975247383117676 }, "all_support_em_at_k": { "3": 0.9900990128517151, "5": 0.9950494766235352 } } }, "latest_state": { "fact_to_person": { "query_count": 31, "recall_at_k": { "1": 0.29032257199287415, "3": 1.0, "5": 1.0 }, "all_support_em_at_k": { "3": 1.0, "5": 1.0 } }, "person_to_fact": { "query_count": 92, "recall_at_k": { "1": 0.39673912525177, "3": 0.967391312122345, "5": 1.0 }, "all_support_em_at_k": { "3": 0.967391312122345, "5": 1.0 } }, "self_to_fact": { "query_count": 82, "recall_at_k": { "1": 0.15853658318519592, "3": 0.6707317233085632, "5": 0.9878048896789551 }, "all_support_em_at_k": { "3": 0.6707317233085632, "5": 0.9878048896789551 } }, "speaker_independent": { "query_count": 252, "recall_at_k": { "1": 0.2976190447807312, "3": 0.9107142686843872, "5": 0.9980158805847168 }, "all_support_em_at_k": { "3": 0.908730149269104, "5": 0.9960317611694336 } } }, "multi_hop_composition": { "fact_to_person": { "query_count": 66, "recall_at_k": { "1": 0.46037158370018005, "3": 0.9197511076927185, "5": 0.9227814078330994 }, "all_support_em_at_k": { "3": 0.8787878751754761, "5": 0.8787878751754761 } }, "person_to_fact": { "query_count": 84, "recall_at_k": { "1": 0.24852609634399414, "3": 0.7167800068855286, "5": 0.9332765936851501 }, "all_support_em_at_k": { "3": 0.488095223903656, "5": 0.9047619104385376 } }, "self_to_fact": { "query_count": 168, "recall_at_k": { "1": 0.2767857015132904, "3": 0.6726190447807312, "5": 0.949404776096344 }, "all_support_em_at_k": { "3": 0.375, "5": 0.8988094925880432 } }, "speaker_independent": { "query_count": 30, "recall_at_k": { "1": 0.2661111056804657, "3": 0.4811111390590668, "5": 0.5586110949516296 }, "all_support_em_at_k": { "3": 0.1666666716337204, "5": 0.23333333432674408 } } }, "temporal_localization": { "person_to_fact": { "query_count": 274, "recall_at_k": { "1": 0.9425182342529297, "3": 0.9881386756896973, "5": 0.9881386756896973 }, "all_support_em_at_k": { "3": 0.985401451587677, "5": 0.985401451587677 } }, "self_to_fact": { "query_count": 21, "recall_at_k": { "1": 0.5238094925880432, "3": 0.7301587462425232, "5": 0.7936508655548096 }, "all_support_em_at_k": { "3": 0.5714285969734192, "5": 0.6190476417541504 } }, "speaker_independent": { "query_count": 424, "recall_at_k": { "1": 0.852594256401062, "3": 0.9386793375015259, "5": 0.9571540951728821 }, "all_support_em_at_k": { "3": 0.9127358198165894, "5": 0.9316037893295288 } } } }, "by_task": { "change_comparison": { "query_count": 470, "recall_at_k": { "1": 0.41773051023483276, "3": 0.8789006471633911, "5": 0.9597517251968384 }, "all_support_em_at_k": { "3": 0.785106360912323, "5": 0.9212765693664551 } }, "direct_recall": { "query_count": 1491, "recall_at_k": { "1": 0.835568904876709, "3": 0.9217526912689209, "5": 0.9395260214805603 }, "all_support_em_at_k": { "3": 0.8973842859268188, "5": 0.9154929518699646 } }, "historical_state": { "query_count": 500, "recall_at_k": { "1": 0.5339999794960022, "3": 0.9549999833106995, "5": 0.9990000128746033 }, "all_support_em_at_k": { "3": 0.9539999961853027, "5": 0.9980000257492065 } }, "latest_state": { "query_count": 457, "recall_at_k": { "1": 0.2921225428581238, "3": 0.8851203322410583, "5": 0.9967177510261536 }, "all_support_em_at_k": { "3": 0.8840262293815613, "5": 0.9956236481666565 } }, "multi_hop_composition": { "query_count": 348, "recall_at_k": { "1": 0.30386221408843994, "3": 0.7136391401290894, "5": 0.9067734479904175 }, "all_support_em_at_k": { "3": 0.4798850417137146, "5": 0.8390804529190063 } }, "temporal_localization": { "query_count": 719, "recall_at_k": { "1": 0.8772600889205933, "3": 0.9514370560646057, "5": 0.9641863703727722 }, "all_support_em_at_k": { "3": 0.9304589629173279, "5": 0.9429763555526733 } } } }, "skipped_control_query_count": 0, "score_analysis": { "overall": { "rag_query_count": 3985, "control_query_count": 111, "gold_score": { "count": 5141, "mean": 0.7241762260820077, "std": 0.17420755087169498, "min": -0.08927247673273087, "p05": 0.375575453042984, "p25": 0.6503528356552124, "median": 0.7556952834129333, "p75": 0.8322693705558777, "p95": 0.9593976736068726, "max": 1.1258333921432495 }, "best_gold_score": { "count": 3985, "mean": 0.7619477864668016, "std": 0.14381897115710743, "min": 0.03736429661512375, "p05": 0.4997822523117066, "p25": 0.6846296191215515, "median": 0.7807751297950745, "p75": 0.8531866669654846, "p95": 0.9743822336196898, "max": 1.1258333921432495 }, "rag_max_candidate_score": { "count": 3985, "mean": 0.7705392655083346, "std": 0.13204878015276228, "min": 0.1924441009759903, "p05": 0.5299609899520874, "p25": 0.6920320987701416, "median": 0.7846444249153137, "p75": 0.8553236126899719, "p95": 0.9765936732292174, "max": 1.1260788440704346 }, "control_max_candidate_score": { "count": 111, "mean": 0.3882304532369515, "std": 0.15184264623213228, "min": 0.03674992173910141, "p05": 0.1377359926700592, "p25": 0.2954407185316086, "median": 0.37531644105911255, "p75": 0.47543179988861084, "p95": 0.6330483555793762, "max": 0.8106650710105896 }, "max_non_gold_score": { "count": 3985, "mean": 0.630470616324633, "std": 0.17867460925989845, "min": 0.10479071736335754, "p05": 0.3529604732990265, "p25": 0.5026586651802063, "median": 0.6192934513092041, "p75": 0.7512153387069702, "p95": 0.9490407824516295, "max": 1.1260788440704346 }, "query_temporal_norm": { "count": 4096, "mean": 0.304678496593624, "std": 0.1486443855292593, "min": 0.08168468624353409, "p05": 0.11245217174291611, "p25": 0.18248226121068, "median": 0.25531914830207825, "p75": 0.42087820917367935, "p95": 0.5570324063301086, "max": 0.5792937874794006 }, "best_gold_margin": { "count": 3985, "mean": 0.13147717014216867, "std": 0.15046855171086562, "min": -0.48386645317077637, "p05": -0.05386531352996825, "p25": 0.0042411088943481445, "median": 0.10900372266769409, "p75": 0.23411086201667786, "p95": 0.4013371586799621, "max": 0.7206796705722809 }, "weakest_gold_margin": { "count": 3985, "mean": 0.10665875107915826, "std": 0.16716168516628985, "min": -0.6774372458457947, "p05": -0.13410429954528807, "p25": -0.001153707504272461, "median": 0.09585797786712646, "p75": 0.22194325923919678, "p95": 0.380974495410919, "max": 0.7206796705722809 }, "top_score_best_balanced_threshold": { "threshold": 0.619530200958252, "balanced_accuracy": 0.9084811285564109, "true_positive_rate": 0.8710163111668758, "false_positive_rate": 0.05405405405405406 }, "gold_score_best_balanced_threshold": { "threshold": 0.619530200958252, "balanced_accuracy": 0.9023330733493846, "true_positive_rate": 0.8587202007528231, "false_positive_rate": 0.05405405405405406 } }, "by_task": { "change_comparison": { "rag_query_count": 470, "control_query_count": 0, "gold_score": { "count": 942, "mean": 0.7404991486950159, "std": 0.10894139741365653, "min": 0.06327641755342484, "p05": 0.5448947280645371, "p25": 0.6833753138780594, "median": 0.7579014599323273, "p75": 0.8139380216598511, "p95": 0.8874219119548797, "max": 0.9721137285232544 }, "best_gold_score": { "count": 470, "mean": 0.7755538869411387, "std": 0.09052670544988363, "min": 0.33879566192626953, "p05": 0.6262453407049179, "p25": 0.7201669067144394, "median": 0.7918211221694946, "p75": 0.8364439904689789, "p95": 0.8994863331317902, "max": 0.9721137285232544 }, "rag_max_candidate_score": { "count": 470, "mean": 0.7802391093462071, "std": 0.08961716909016448, "min": 0.33879566192626953, "p05": 0.6289204120635986, "p25": 0.723452016711235, "median": 0.7953038215637207, "p75": 0.8430169820785522, "p95": 0.9019855648279189, "max": 0.9721137285232544 }, "control_max_candidate_score": null, "max_non_gold_score": { "count": 470, "mean": 0.548556295322611, "std": 0.19348231002676886, "min": 0.10479071736335754, "p05": 0.25167940706014635, "p25": 0.38838403671979904, "median": 0.5346508920192719, "p75": 0.7160655558109283, "p95": 0.8489899426698684, "max": 0.9398146867752075 }, "query_temporal_norm": { "count": 470, "mean": 0.38702126294374467, "std": 0.04252809385217023, "min": 0.17005102336406708, "p05": 0.30043437480926516, "p25": 0.37197282910346985, "median": 0.3936518132686615, "p75": 0.41210484504699707, "p95": 0.4362030506134033, "max": 0.568455159664154 }, "best_gold_margin": { "count": 470, "mean": 0.22699759161852776, "std": 0.1835967645486095, "min": -0.10016191005706787, "p05": -0.03408718109130859, "p25": 0.07795208692550659, "median": 0.22617237269878387, "p75": 0.37646569311618805, "p95": 0.5155857145786286, "max": 0.699258029460907 }, "weakest_gold_margin": { "count": 470, "mean": 0.16063123863745243, "std": 0.1859837532729084, "min": -0.4117041751742363, "p05": -0.11187890470027922, "p25": 0.014072544872760773, "median": 0.1509234607219696, "p75": 0.3116276040673256, "p95": 0.45314110368490207, "max": 0.6590308099985123 }, "top_score_best_balanced_threshold": null, "gold_score_best_balanced_threshold": null }, "current_only_control": { "rag_query_count": 0, "control_query_count": 110, "gold_score": null, "best_gold_score": null, "rag_max_candidate_score": null, "control_max_candidate_score": { "count": 110, "mean": 0.3872418130324645, "std": 0.1521752229685398, "min": 0.03674992173910141, "p05": 0.1367964342236519, "p25": 0.2949500158429146, "median": 0.37395188212394714, "p75": 0.4682910218834877, "p95": 0.634412682056427, "max": 0.8106650710105896 }, "max_non_gold_score": null, "query_temporal_norm": { "count": 110, "mean": 0.3163933543996377, "std": 0.12005832986975279, "min": 0.10356259346008301, "p05": 0.12996669486165047, "p25": 0.20454848557710648, "median": 0.3282453566789627, "p75": 0.413640059530735, "p95": 0.5012234553694724, "max": 0.5566943883895874 }, "best_gold_margin": null, "weakest_gold_margin": null, "top_score_best_balanced_threshold": null, "gold_score_best_balanced_threshold": null }, "direct_recall": { "rag_query_count": 1491, "control_query_count": 0, "gold_score": { "count": 1669, "mean": 0.6869575773797418, "std": 0.19855509550894992, "min": -0.056370239704847336, "p05": 0.2907854557037354, "p25": 0.5796415209770203, "median": 0.7289903163909912, "p75": 0.8238811492919922, "p95": 0.947128140926361, "max": 1.1070964336395264 }, "best_gold_score": { "count": 1491, "mean": 0.7264814051798851, "std": 0.16145032231177006, "min": 0.03736429661512375, "p05": 0.43596406280994415, "p25": 0.6369831562042236, "median": 0.752158522605896, "p75": 0.8358416557312012, "p95": 0.9545033872127533, "max": 1.1070964336395264 }, "rag_max_candidate_score": { "count": 1491, "mean": 0.7387905766366712, "std": 0.14133217273717888, "min": 0.1924441009759903, "p05": 0.4963827431201935, "p25": 0.6480166018009186, "median": 0.7563066482543945, "p75": 0.8373694121837616, "p95": 0.9545033872127533, "max": 1.1070964336395264 }, "control_max_candidate_score": null, "max_non_gold_score": { "count": 1491, "mean": 0.5692660571403907, "std": 0.1301589419087464, "min": 0.10697713494300842, "p05": 0.3561844676733017, "p25": 0.4861396551132202, "median": 0.5683011412620544, "p75": 0.6503013372421265, "p95": 0.7916618585586548, "max": 0.975655198097229 }, "query_temporal_norm": { "count": 1491, "mean": 0.22666298581061148, "std": 0.10586569224947587, "min": 0.08168468624353409, "p05": 0.0979655385017395, "p25": 0.14285876601934433, "median": 0.21143613755702972, "p75": 0.2786862254142761, "p95": 0.45315635204315186, "max": 0.572542130947113 }, "best_gold_margin": { "count": 1491, "mean": 0.15721534803949452, "std": 0.15264452667151834, "min": -0.48386645317077637, "p05": -0.09162609279155731, "p25": 0.059481263160705566, "median": 0.15777647495269775, "p75": 0.25978294014930725, "p95": 0.39778003096580505, "max": 0.7206796705722809 }, "weakest_gold_margin": { "count": 1491, "mean": 0.13992435263646585, "std": 0.18043414807290126, "min": -0.6774372458457947, "p05": -0.19127929210662842, "p25": 0.04726390540599823, "median": 0.15512782335281372, "p75": 0.25943848490715027, "p95": 0.39778003096580505, "max": 0.7206796705722809 }, "top_score_best_balanced_threshold": null, "gold_score_best_balanced_threshold": null }, "historical_state": { "rag_query_count": 500, "control_query_count": 0, "gold_score": { "count": 502, "mean": 0.7836064203564389, "std": 0.10709909743229676, "min": 0.14025598764419556, "p05": 0.5942545413970948, "p25": 0.7248825877904892, "median": 0.8013403415679932, "p75": 0.8561894297599792, "p95": 0.9279397994279861, "max": 1.012067198753357 }, "best_gold_score": { "count": 500, "mean": 0.7853946338891983, "std": 0.10275866286384987, "min": 0.37629517912864685, "p05": 0.5945305287837982, "p25": 0.7256994992494583, "median": 0.8023103773593903, "p75": 0.8565277755260468, "p95": 0.9281367629766464, "max": 1.012067198753357 }, "rag_max_candidate_score": { "count": 500, "mean": 0.7897239015698433, "std": 0.10143125862120343, "min": 0.37629517912864685, "p05": 0.6064692318439484, "p25": 0.7330148816108704, "median": 0.8051750957965851, "p75": 0.8608255237340927, "p95": 0.9311295807361601, "max": 1.012067198753357 }, "control_max_candidate_score": null, "max_non_gold_score": { "count": 500, "mean": 0.6717005554437637, "std": 0.15029697753120055, "min": 0.16829177737236023, "p05": 0.4131940007209778, "p25": 0.5654338002204895, "median": 0.6779390871524811, "p75": 0.7914849817752838, "p95": 0.8960957378149033, "max": 1.0081841945648193 }, "query_temporal_norm": { "count": 500, "mean": 0.48263769267499446, "std": 0.10428369082664105, "min": 0.090034119784832, "p05": 0.24665968716144565, "p25": 0.4482061117887497, "median": 0.5256085395812988, "p75": 0.5539183616638184, "p95": 0.5720209151506424, "max": 0.5792937874794006 }, "best_gold_margin": { "count": 500, "mean": 0.11369407844543457, "std": 0.12331652846556644, "min": -0.2083926796913147, "p05": -0.029318222403526294, "p25": 0.003192394971847534, "median": 0.10047835111618042, "p75": 0.1869528740644455, "p95": 0.33141966611146895, "max": 0.4669537842273712 }, "weakest_gold_margin": { "count": 500, "mean": 0.1126235191822052, "std": 0.12586990224972355, "min": -0.4509775638580322, "p05": -0.03355695605278015, "p25": 0.003106921911239624, "median": 0.10047835111618042, "p75": 0.1869528740644455, "p95": 0.33141966611146895, "max": 0.4669537842273712 }, "top_score_best_balanced_threshold": null, "gold_score_best_balanced_threshold": null }, "latest_state": { "rag_query_count": 457, "control_query_count": 0, "gold_score": { "count": 460, "mean": 0.9099455071532208, "std": 0.11056362986875025, "min": 0.2515583336353302, "p05": 0.7147018998861313, "p25": 0.8624780625104904, "median": 0.927989274263382, "p75": 0.9840456992387772, "p95": 1.0430482387542723, "max": 1.1258333921432495 }, "best_gold_score": { "count": 457, "mean": 0.9128042522092356, "std": 0.10462899466412111, "min": 0.2515583336353302, "p05": 0.7270167827606201, "p25": 0.8639979958534241, "median": 0.9282728433609009, "p75": 0.9842485189437866, "p95": 1.0431268453598022, "max": 1.1258333921432495 }, "rag_max_candidate_score": { "count": 457, "mean": 0.9186350223272061, "std": 0.1033423010515788, "min": 0.3498287498950958, "p05": 0.7335727095603943, "p25": 0.8663325309753418, "median": 0.9317175149917603, "p75": 0.9901189804077148, "p95": 1.0482017040252685, "max": 1.1260788440704346 }, "control_max_candidate_score": null, "max_non_gold_score": { "count": 457, "mean": 0.9157646012123617, "std": 0.1071663211990817, "min": 0.341567724943161, "p05": 0.7311487436294556, "p25": 0.863583505153656, "median": 0.9317175149917603, "p75": 0.9866815805435181, "p95": 1.0477006912231446, "max": 1.1260788440704346 }, "query_temporal_norm": { "count": 457, "mean": 0.5188334593039895, "std": 0.06393273507489536, "min": 0.12837310135364532, "p05": 0.36590389013290403, "p25": 0.5123153924942017, "median": 0.5376707315444946, "p75": 0.5540842413902283, "p95": 0.5678486704826355, "max": 0.5767868161201477 }, "best_gold_margin": { "count": 457, "mean": -0.0029603490031261154, "std": 0.020342693875631014, "min": -0.09827041625976562, "p05": -0.036279916763305664, "p25": -0.00460284948348999, "median": -0.0004582405090332031, "p75": 0.003332674503326416, "p95": 0.008798444271087644, "max": 0.23436546325683594 }, "weakest_gold_margin": { "count": 457, "mean": -0.0037261917371979427, "std": 0.01602755304203525, "min": -0.09827041625976562, "p05": -0.036279916763305664, "p25": -0.00463712215423584, "median": -0.0004839897155761719, "p75": 0.0033155083656311035, "p95": 0.00844621658325195, "max": 0.11645901203155518 }, "top_score_best_balanced_threshold": null, "gold_score_best_balanced_threshold": null }, "multi_hop_composition": { "rag_query_count": 348, "control_query_count": 0, "gold_score": { "count": 773, "mean": 0.6628726145895957, "std": 0.16474756870609927, "min": -0.05371195077896118, "p05": 0.31368120312690734, "p25": 0.6175206899642944, "median": 0.712782084941864, "p75": 0.7731857299804688, "p95": 0.8209763169288635, "max": 0.8812718987464905 }, "best_gold_score": { "count": 348, "mean": 0.7313412460444988, "std": 0.09207875076116444, "min": 0.36352241039276123, "p05": 0.5470803171396256, "p25": 0.6859159916639328, "median": 0.754956841468811, "p75": 0.7975459843873978, "p95": 0.8379994869232178, "max": 0.8812718987464905 }, "rag_max_candidate_score": { "count": 348, "mean": 0.7419123562245533, "std": 0.08379927154498036, "min": 0.43880122900009155, "p05": 0.5616730064153671, "p25": 0.6991288214921951, "median": 0.7600564062595367, "p75": 0.8044402897357941, "p95": 0.8395571231842041, "max": 0.8812718987464905 }, "control_max_candidate_score": null, "max_non_gold_score": { "count": 348, "mean": 0.663961509306883, "std": 0.14432546830207857, "min": 0.19614651799201965, "p05": 0.38625628352165225, "p25": 0.5634785890579224, "median": 0.7131754457950592, "p75": 0.7773982137441635, "p95": 0.8263130962848663, "max": 0.8776211738586426 }, "query_temporal_norm": { "count": 348, "mean": 0.19367624020696372, "std": 0.03749222859666623, "min": 0.12189259380102158, "p05": 0.13993898779153824, "p25": 0.16496126726269722, "median": 0.190658338367939, "p75": 0.21625519171357155, "p95": 0.2616093188524245, "max": 0.33960673213005066 }, "best_gold_margin": { "count": 348, "mean": 0.06737973673761576, "std": 0.1273355332284643, "min": -0.16347423195838928, "p05": -0.05428696870803833, "p25": -0.010949954390525818, "median": 0.019723743200302124, "p75": 0.0933598130941391, "p95": 0.3714251846075057, "max": 0.5118336081504822 }, "weakest_gold_margin": { "count": 348, "mean": -0.013968390235998508, "std": 0.15625439094496515, "min": -0.5777953863143921, "p05": -0.2627247408032417, "p25": -0.0745110958814621, "median": -0.02843642234802246, "p75": 0.007143333554267883, "p95": 0.30581344068050376, "max": 0.5001735091209412 }, "top_score_best_balanced_threshold": null, "gold_score_best_balanced_threshold": null }, "recent_context_control": { "rag_query_count": 0, "control_query_count": 1, "gold_score": null, "best_gold_score": null, "rag_max_candidate_score": null, "control_max_candidate_score": { "count": 1, "mean": 0.4969808757305145, "std": 0.0, "min": 0.4969808757305145, "p05": 0.4969808757305145, "p25": 0.4969808757305145, "median": 0.4969808757305145, "p75": 0.4969808757305145, "p95": 0.4969808757305145, "max": 0.4969808757305145 }, "max_non_gold_score": null, "query_temporal_norm": { "count": 1, "mean": 0.20389491319656372, "std": 0.0, "min": 0.20389491319656372, "p05": 0.20389491319656372, "p25": 0.20389491319656372, "median": 0.20389491319656372, "p75": 0.20389491319656372, "p95": 0.20389491319656372, "max": 0.20389491319656372 }, "best_gold_margin": null, "weakest_gold_margin": null, "top_score_best_balanced_threshold": null, "gold_score_best_balanced_threshold": null }, "temporal_localization": { "rag_query_count": 719, "control_query_count": 0, "gold_score": { "count": 795, "mean": 0.697561881991115, "std": 0.17009461728665182, "min": -0.08927247673273087, "p05": 0.3611560195684433, "p25": 0.6234583854675293, "median": 0.732460618019104, "p75": 0.816007524728775, "p95": 0.9051089465618133, "max": 1.0474931001663208 }, "best_gold_score": { "count": 719, "mean": 0.7292242187162102, "std": 0.13312502433949933, "min": 0.1190776377916336, "p05": 0.4726928949356079, "p25": 0.6564635336399078, "median": 0.7507467269897461, "p75": 0.8241211473941803, "p95": 0.9126609623432161, "max": 1.0474931001663208 }, "rag_max_candidate_score": { "count": 719, "mean": 0.7364202864229762, "std": 0.12132767669906487, "min": 0.24290406703948975, "p05": 0.5120499312877655, "p25": 0.6625339388847351, "median": 0.7538652420043945, "p75": 0.8249395787715912, "p95": 0.9126609623432161, "max": 1.0474931001663208 }, "control_max_candidate_score": null, "max_non_gold_score": { "count": 719, "mean": 0.5847217668163925, "std": 0.1286120778132217, "min": 0.2042156159877777, "p05": 0.37733144760131837, "p25": 0.48644280433654785, "median": 0.5866289138793945, "p75": 0.6778780221939087, "p95": 0.7978232324123383, "max": 0.9642623662948608 }, "query_temporal_norm": { "count": 719, "mean": 0.20483502627496758, "std": 0.06486274901452087, "min": 0.09465165436267853, "p05": 0.11441298574209213, "p25": 0.15930375456809998, "median": 0.20111021399497986, "p75": 0.23105795681476593, "p95": 0.3251008868217468, "max": 0.534060001373291 }, "best_gold_margin": { "count": 719, "mean": 0.14450245189981767, "std": 0.12346059833105659, "min": -0.3746306002140045, "p05": -0.04459953308105469, "p25": 0.07054683566093445, "median": 0.12600627541542053, "p75": 0.22508594393730164, "p95": 0.35579485893249513, "max": 0.5074322521686554 }, "weakest_gold_margin": { "count": 719, "mean": 0.12679185110541974, "std": 0.14944456096152975, "min": -0.5301856100559235, "p05": -0.13406084179878236, "p25": 0.06027638912200928, "median": 0.12011480331420898, "p75": 0.22024193406105042, "p95": 0.3543264567852021, "max": 0.5074322521686554 }, "top_score_best_balanced_threshold": null, "gold_score_best_balanced_threshold": null } } } } } ] }