| { |
| "checkpoint": "/home/zhangchao/work/minye_training/mixed_retrieval_normal_identity_balanced/retriever.pt", |
| "protocol": { |
| "training_stage": "multiscene_ordered_time_audio_query_v4_1", |
| "training_sources": [ |
| "hotpotqa" |
| ], |
| "hidden_size": 4096, |
| "embedding_size": 1024, |
| "attention_heads": 8, |
| "max_text_tokens": 512, |
| "max_query_tokens": 128, |
| "dropout": 0.1, |
| "hidden_layer_index": 19, |
| "retrieval_lora_layers": [ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 6, |
| 7, |
| 8, |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15, |
| 16, |
| 17, |
| 18, |
| 19 |
| ], |
| "retrieval_lora_rank": 8, |
| "retrieval_lora_alpha": 32.0, |
| "retrieval_lora_dropout": 0.05, |
| "query_context_turns": 0, |
| "query_serialization": "current_session_audio_plus_query_audio", |
| "validation_query_modalities": [], |
| "training_query_modality": "audio_only", |
| "missing_audio_policy": "current-user missing: skip; previous-user missing: text fallback", |
| "audio_query_context_turns": 0, |
| "audio_context_source": "current_user_audio", |
| "conversation_memory_context_turns": 0, |
| "conversation_memory_sources": [], |
| "hard_negative_mining": "previous_epoch_dense_false_positives_v1", |
| "checkpoint_selection": "last_epoch", |
| "multi_positive_objective": "mean_positive_log_probability", |
| "pooling": "shared_attention_then_directional_qk", |
| "use_retrieval_rope": false, |
| "output_normalization": "l2", |
| "speaker_checkpoint_path": "/home/zhangchao/project/salmonn_omni_transform_v2_siyuan/siyuanTest/train_speaker_embedding/outputs/mix_5_7_8_ddp_4096_proj1024_min8/best", |
| "speaker_window_seconds": 5.0, |
| "time_serialization": "numeric_temporal_route_only", |
| "memory_temporal_positions_shuffled": false, |
| "memory_serialization": "dataset_speaker_name_plus_text", |
| "random_history_sessions": true, |
| "history_session_seed": 20261015, |
| "training_speaker_names": "epoch_random_alias_all_self_no_spoken_names_v1", |
| "cross_rank_candidates": true, |
| "hard_negatives_per_query": 16, |
| "hard_negative_loss_weight": 1.0, |
| "query_counterfactuals": true, |
| "retriever_architecture": "semantic_plus_ordered_time_qk_v4_1", |
| "embedding_layout": "raw_concat_unit_semantic_bounded_temporal_v3", |
| "similarity": "inner_product", |
| "time_embedding_size": 128, |
| "semantic_embedding_size": 1024, |
| "retrieval_embedding_size": 1152, |
| "time_encoder": "fixed_multiscale_features_linear_key_v4_1", |
| "query_pool_audio_tokens": 128, |
| "query_encoding_protocol": "incremental_sliding_kv_pinned_prefix", |
| "query_input_truncation": null, |
| "temporal_pooler": "retained_audio_tail_h19", |
| "temporal_query_tokens": "last_128_audio_h19_excluding_speaker", |
| "temporal_projection": "audio_wording_plus_reference_q_numeric_time_k_v4_1", |
| "time_scale_seconds": 60.0, |
| "semantic_loss_weight": 0.25, |
| "temporal_null_loss_weight": 0.1, |
| "temporal_relations": [ |
| "none", |
| "latest", |
| "earliest", |
| "before", |
| "after", |
| "session", |
| "comparison" |
| ], |
| "temporal_strength": "query_vector_norm_without_classifier_gate_v3", |
| "retrieval_rope_positions": null, |
| "no_temporal": false, |
| "data_jsonl": "/home/zhangchao/work/minye_training/mixed_identity_balanced_data/train_val.jsonl", |
| "initial_checkpoint": "/home/zhangchao/work/minye_training/mixed_retrieval_normal/retriever.pt", |
| "query_modality": "audio", |
| "memory_modality": "text", |
| "speaker_embedding": "frozen_4096d_prepend_to_h19", |
| "query_speaker_name_text": false, |
| "memory_speaker_name_text": "dataset_provided", |
| "validation_speaker_names": "dataset_provided", |
| "time": "session_index_and_start_sec", |
| "cross_rank_negatives": true, |
| "all_in_sample_memories_are_candidates": true, |
| "candidate_memory_scope": "all_history_utterances_excluding_current_session", |
| "temporal_wording_counts": { |
| "train:after": 1, |
| "train:before": 98, |
| "train:comparison": 903, |
| "train:earliest": 783, |
| "train:latest": 748, |
| "train:session": 1431, |
| "train:unmatched": 390, |
| "validation:before": 70, |
| "validation:comparison": 414, |
| "validation:earliest": 354, |
| "validation:latest": 358, |
| "validation:session": 789, |
| "validation:unmatched": 161 |
| }, |
| "control_support_handling": "exclude_declared_current_context_supports" |
| }, |
| "checkpoint_selection_metric": "recall_at_k.1", |
| "best_validation_recall_at_1": 0.7014623880386353, |
| "best_epoch": 2, |
| "history": [ |
| { |
| "epoch": 1, |
| "checkpoint": "/home/zhangchao/work/minye_training/mixed_retrieval_normal_identity_balanced/retriever_epoch_01.pt", |
| "train_loss": 0.9721298217773438, |
| "train_losses": { |
| "total": 0.9721298217773438, |
| "global": 1.0716792345046997, |
| "hard_negative": 0.0, |
| "semantic": 0.7763091921806335, |
| "temporal_null": 0.03924102708697319, |
| "speaker_name_renamed_samples": 639, |
| "speaker_name_skipped_mixed_or_nonself": 4108, |
| "speaker_name_skipped_spoken_name": 267, |
| "speaker_name_skipped_insufficient_names": 0 |
| }, |
| "validation": { |
| "query_count": 3985, |
| "recall_at_k": { |
| "1": 0.6997685432434082, |
| "3": 0.9176088571548462, |
| "5": 0.9558404684066772 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.8727728724479675, |
| "5": 0.9309912323951721 |
| }, |
| "by_speaker_query_type": { |
| "fact_to_person": { |
| "query_count": 786, |
| "recall_at_k": { |
| "1": 0.8282034397125244, |
| "3": 0.9724811911582947, |
| "5": 0.9802632927894592 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9618320465087891, |
| "5": 0.9669210910797119 |
| } |
| }, |
| "person_to_fact": { |
| "query_count": 969, |
| "recall_at_k": { |
| "1": 0.7436802983283997, |
| "3": 0.9417711496353149, |
| "5": 0.9750897288322449 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9112487435340881, |
| "5": 0.9638803005218506 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 509, |
| "recall_at_k": { |
| "1": 0.2703012526035309, |
| "3": 0.7080877423286438, |
| "5": 0.8973477482795715 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.5068762302398682, |
| "5": 0.8192534446716309 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 1721, |
| "recall_at_k": { |
| "1": 0.7434049248695374, |
| "3": 0.9409112930297852, |
| "5": 0.9511474967002869 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9186519384384155, |
| "5": 0.9291110038757324 |
| } |
| } |
| }, |
| "by_task_and_speaker_query_type": { |
| "change_comparison": { |
| "fact_to_person": { |
| "query_count": 74, |
| "recall_at_k": { |
| "1": 0.5, |
| "3": 1.0, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 1.0, |
| "5": 1.0 |
| } |
| }, |
| "person_to_fact": { |
| "query_count": 88, |
| "recall_at_k": { |
| "1": 0.4943181872367859, |
| "3": 0.9602272510528564, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9204545617103577, |
| "5": 1.0 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 99, |
| "recall_at_k": { |
| "1": 0.13131313025951385, |
| "3": 0.5, |
| "5": 0.7878788113594055 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.1111111119389534, |
| "5": 0.5757575631141663 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 209, |
| "recall_at_k": { |
| "1": 0.49920251965522766, |
| "3": 0.9920255541801453, |
| "5": 0.9932217597961426 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9904305934906006, |
| "5": 0.9904305934906006 |
| } |
| } |
| }, |
| "direct_recall": { |
| "fact_to_person": { |
| "query_count": 524, |
| "recall_at_k": { |
| "1": 0.9228689074516296, |
| "3": 0.9688295125961304, |
| "5": 0.980120837688446 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9580152630805969, |
| "5": 0.9656488299369812 |
| } |
| }, |
| "person_to_fact": { |
| "query_count": 306, |
| "recall_at_k": { |
| "1": 0.8218954205513, |
| "3": 0.9327341914176941, |
| "5": 0.9556099772453308 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9215686321258545, |
| "5": 0.9379084706306458 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 57, |
| "recall_at_k": { |
| "1": 0.41374269127845764, |
| "3": 0.5891813039779663, |
| "5": 0.6856725811958313 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.4035087823867798, |
| "5": 0.5614035129547119 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 604, |
| "recall_at_k": { |
| "1": 0.8246412873268127, |
| "3": 0.9071468114852905, |
| "5": 0.9170805811882019 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.8807947039604187, |
| "5": 0.8907284736633301 |
| } |
| } |
| }, |
| "historical_state": { |
| "fact_to_person": { |
| "query_count": 91, |
| "recall_at_k": { |
| "1": 0.901098906993866, |
| "3": 1.0, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 1.0, |
| "5": 1.0 |
| } |
| }, |
| "person_to_fact": { |
| "query_count": 125, |
| "recall_at_k": { |
| "1": 0.871999979019165, |
| "3": 1.0, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 1.0, |
| "5": 1.0 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 82, |
| "recall_at_k": { |
| "1": 0.31707316637039185, |
| "3": 0.9878048896789551, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9878048896789551, |
| "5": 1.0 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 202, |
| "recall_at_k": { |
| "1": 0.9207921028137207, |
| "3": 0.9975247383117676, |
| "5": 0.9975247383117676 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9950494766235352, |
| "5": 0.9950494766235352 |
| } |
| } |
| }, |
| "latest_state": { |
| "fact_to_person": { |
| "query_count": 31, |
| "recall_at_k": { |
| "1": 0.5806451439857483, |
| "3": 1.0, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 1.0, |
| "5": 1.0 |
| } |
| }, |
| "person_to_fact": { |
| "query_count": 92, |
| "recall_at_k": { |
| "1": 0.375, |
| "3": 0.989130437374115, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.989130437374115, |
| "5": 1.0 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 82, |
| "recall_at_k": { |
| "1": 0.2195121943950653, |
| "3": 0.8414633870124817, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.8414633870124817, |
| "5": 1.0 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 252, |
| "recall_at_k": { |
| "1": 0.4682539701461792, |
| "3": 0.9861111044883728, |
| "5": 0.9980158805847168 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9841269850730896, |
| "5": 0.9960317611694336 |
| } |
| } |
| }, |
| "multi_hop_composition": { |
| "fact_to_person": { |
| "query_count": 66, |
| "recall_at_k": { |
| "1": 0.46037158370018005, |
| "3": 0.9197511076927185, |
| "5": 0.9227814078330994 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.8787878751754761, |
| "5": 0.8787878751754761 |
| } |
| }, |
| "person_to_fact": { |
| "query_count": 84, |
| "recall_at_k": { |
| "1": 0.33185943961143494, |
| "3": 0.6774943470954895, |
| "5": 0.9130384922027588 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.4166666567325592, |
| "5": 0.8571428656578064 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 168, |
| "recall_at_k": { |
| "1": 0.2738095223903656, |
| "3": 0.6666666865348816, |
| "5": 0.9404761791229248 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.369047611951828, |
| "5": 0.886904776096344 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 30, |
| "recall_at_k": { |
| "1": 0.2661111056804657, |
| "3": 0.5019444823265076, |
| "5": 0.5530555844306946 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.1666666716337204, |
| "5": 0.23333333432674408 |
| } |
| } |
| }, |
| "temporal_localization": { |
| "person_to_fact": { |
| "query_count": 274, |
| "recall_at_k": { |
| "1": 0.9279196858406067, |
| "3": 0.9844890236854553, |
| "5": 0.9881386756896973 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9817517995834351, |
| "5": 0.985401451587677 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 21, |
| "recall_at_k": { |
| "1": 0.5238094925880432, |
| "3": 0.7301587462425232, |
| "5": 0.8412699103355408 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.5714285969734192, |
| "5": 0.7142857313156128 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 424, |
| "recall_at_k": { |
| "1": 0.8608490824699402, |
| "3": 0.9410377144813538, |
| "5": 0.9571540355682373 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9150943160057068, |
| "5": 0.9316037893295288 |
| } |
| } |
| } |
| }, |
| "by_task": { |
| "change_comparison": { |
| "query_count": 470, |
| "recall_at_k": { |
| "1": 0.42092201113700867, |
| "3": 0.8836879134178162, |
| "5": 0.9523048996925354 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.7936170101165771, |
| "5": 0.9063829779624939 |
| } |
| }, |
| "direct_recall": { |
| "query_count": 1491, |
| "recall_at_k": { |
| "1": 0.8428906202316284, |
| "3": 0.9219203591346741, |
| "5": 0.938296377658844 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.8980550169944763, |
| "5": 0.9141515493392944 |
| } |
| }, |
| "historical_state": { |
| "query_count": 500, |
| "recall_at_k": { |
| "1": 0.8059999942779541, |
| "3": 0.996999979019165, |
| "5": 0.9990000128746033 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9959999918937683, |
| "5": 0.9980000257492065 |
| } |
| }, |
| "latest_state": { |
| "query_count": 457, |
| "recall_at_k": { |
| "1": 0.4124726355075836, |
| "3": 0.9617067575454712, |
| "5": 0.9989058971405029 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9606127142906189, |
| "5": 0.9978117942810059 |
| } |
| }, |
| "multi_hop_composition": { |
| "query_count": 348, |
| "recall_at_k": { |
| "1": 0.32254037261009216, |
| "3": 0.7030788064002991, |
| "5": 0.8970990180969238 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.4597701132297516, |
| "5": 0.8218390941619873 |
| } |
| }, |
| "temporal_localization": { |
| "query_count": 719, |
| "recall_at_k": { |
| "1": 0.8765647411346436, |
| "3": 0.9514371156692505, |
| "5": 0.9655771851539612 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9304589629173279, |
| "5": 0.9457579851150513 |
| } |
| } |
| }, |
| "semantic_only": { |
| "query_count": 3985, |
| "recall_at_k": { |
| "1": 0.640546441078186, |
| "3": 0.907759428024292, |
| "5": 0.9557926058769226 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.8619824051856995, |
| "5": 0.9309912323951721 |
| }, |
| "by_speaker_query_type": { |
| "fact_to_person": { |
| "query_count": 786, |
| "recall_at_k": { |
| "1": 0.7862187027931213, |
| "3": 0.971208930015564, |
| "5": 0.9817173480987549 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9605597853660583, |
| "5": 0.9681933522224426 |
| } |
| }, |
| "person_to_fact": { |
| "query_count": 969, |
| "recall_at_k": { |
| "1": 0.6827927827835083, |
| "3": 0.9407391548156738, |
| "5": 0.9750897288322449 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.910216748714447, |
| "5": 0.9659442901611328 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 509, |
| "recall_at_k": { |
| "1": 0.23984938859939575, |
| "3": 0.6452193856239319, |
| "5": 0.8904715180397034 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.44007858633995056, |
| "5": 0.8113948702812195 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 1721, |
| "recall_at_k": { |
| "1": 0.6687390208244324, |
| "3": 0.9378607273101807, |
| "5": 0.9524064660072327 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9145845174789429, |
| "5": 0.9296920299530029 |
| } |
| } |
| }, |
| "by_task_and_speaker_query_type": { |
| "change_comparison": { |
| "fact_to_person": { |
| "query_count": 74, |
| "recall_at_k": { |
| "1": 0.5, |
| "3": 1.0, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 1.0, |
| "5": 1.0 |
| } |
| }, |
| "person_to_fact": { |
| "query_count": 88, |
| "recall_at_k": { |
| "1": 0.4943181872367859, |
| "3": 0.9602272510528564, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9204545617103577, |
| "5": 1.0 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 99, |
| "recall_at_k": { |
| "1": 0.1262626200914383, |
| "3": 0.49494948983192444, |
| "5": 0.7929292917251587 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.08080808073282242, |
| "5": 0.5858585834503174 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 209, |
| "recall_at_k": { |
| "1": 0.48963314294815063, |
| "3": 0.9908294081687927, |
| "5": 0.9932217597961426 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9856459498405457, |
| "5": 0.9904305934906006 |
| } |
| } |
| }, |
| "direct_recall": { |
| "fact_to_person": { |
| "query_count": 524, |
| "recall_at_k": { |
| "1": 0.9285941123962402, |
| "3": 0.9669210910797119, |
| "5": 0.9820291996002197 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9561068415641785, |
| "5": 0.9675572514533997 |
| } |
| }, |
| "person_to_fact": { |
| "query_count": 306, |
| "recall_at_k": { |
| "1": 0.8153594732284546, |
| "3": 0.9360021352767944, |
| "5": 0.9539759755134583 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9248365759849548, |
| "5": 0.9379084706306458 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 57, |
| "recall_at_k": { |
| "1": 0.41374269127845764, |
| "3": 0.5716374516487122, |
| "5": 0.6856725811958313 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.4035087823867798, |
| "5": 0.5614035129547119 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 604, |
| "recall_at_k": { |
| "1": 0.8113962411880493, |
| "3": 0.9121137261390686, |
| "5": 0.9201158881187439 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.8841059803962708, |
| "5": 0.8923841118812561 |
| } |
| } |
| }, |
| "historical_state": { |
| "fact_to_person": { |
| "query_count": 91, |
| "recall_at_k": { |
| "1": 0.5494505167007446, |
| "3": 1.0, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 1.0, |
| "5": 1.0 |
| } |
| }, |
| "person_to_fact": { |
| "query_count": 125, |
| "recall_at_k": { |
| "1": 0.5519999861717224, |
| "3": 0.9919999837875366, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9919999837875366, |
| "5": 1.0 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 82, |
| "recall_at_k": { |
| "1": 0.2195121943950653, |
| "3": 0.7560975551605225, |
| "5": 0.9878048896789551 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.7560975551605225, |
| "5": 0.9878048896789551 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 202, |
| "recall_at_k": { |
| "1": 0.6188119053840637, |
| "3": 0.9925742745399475, |
| "5": 0.9975247383117676 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9900990128517151, |
| "5": 0.9950494766235352 |
| } |
| } |
| }, |
| "latest_state": { |
| "fact_to_person": { |
| "query_count": 31, |
| "recall_at_k": { |
| "1": 0.4516128897666931, |
| "3": 1.0, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 1.0, |
| "5": 1.0 |
| } |
| }, |
| "person_to_fact": { |
| "query_count": 92, |
| "recall_at_k": { |
| "1": 0.21195651590824127, |
| "3": 0.97826087474823, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.97826087474823, |
| "5": 1.0 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 82, |
| "recall_at_k": { |
| "1": 0.1463414579629898, |
| "3": 0.6829268336296082, |
| "5": 0.9756097197532654 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.6829268336296082, |
| "5": 0.9756097197532654 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 252, |
| "recall_at_k": { |
| "1": 0.2460317462682724, |
| "3": 0.9583333134651184, |
| "5": 0.9980158805847168 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9563491940498352, |
| "5": 0.9960317611694336 |
| } |
| } |
| }, |
| "multi_hop_composition": { |
| "fact_to_person": { |
| "query_count": 66, |
| "recall_at_k": { |
| "1": 0.46037158370018005, |
| "3": 0.9197511076927185, |
| "5": 0.9249458909034729 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.8787878751754761, |
| "5": 0.8787878751754761 |
| } |
| }, |
| "person_to_fact": { |
| "query_count": 84, |
| "recall_at_k": { |
| "1": 0.30209752917289734, |
| "3": 0.6774943470954895, |
| "5": 0.9189909100532532 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.4166666567325592, |
| "5": 0.8809523582458496 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 168, |
| "recall_at_k": { |
| "1": 0.2678571343421936, |
| "3": 0.675595223903656, |
| "5": 0.9375 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.375, |
| "5": 0.8809523582458496 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 30, |
| "recall_at_k": { |
| "1": 0.24944445490837097, |
| "3": 0.5019444823265076, |
| "5": 0.5641666650772095 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.1666666716337204, |
| "5": 0.23333333432674408 |
| } |
| } |
| }, |
| "temporal_localization": { |
| "person_to_fact": { |
| "query_count": 274, |
| "recall_at_k": { |
| "1": 0.92974454164505, |
| "3": 0.9844890236854553, |
| "5": 0.9881386756896973 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9817517995834351, |
| "5": 0.985401451587677 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 21, |
| "recall_at_k": { |
| "1": 0.5238094925880432, |
| "3": 0.7301587462425232, |
| "5": 0.8174603581428528 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.5714285969734192, |
| "5": 0.6666666865348816 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 424, |
| "recall_at_k": { |
| "1": 0.8584905862808228, |
| "3": 0.9410377144813538, |
| "5": 0.9571540355682373 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9150943160057068, |
| "5": 0.9316037893295288 |
| } |
| } |
| } |
| }, |
| "by_task": { |
| "change_comparison": { |
| "query_count": 470, |
| "recall_at_k": { |
| "1": 0.4156028628349304, |
| "3": 0.8820921778678894, |
| "5": 0.9533687233924866 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.785106360912323, |
| "5": 0.9085106253623962 |
| } |
| }, |
| "direct_recall": { |
| "query_count": 1491, |
| "recall_at_k": { |
| "1": 0.8381957411766052, |
| "3": 0.9232617616653442, |
| "5": 0.9398613572120667 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.8993963599205017, |
| "5": 0.9154929518699646 |
| } |
| }, |
| "historical_state": { |
| "query_count": 500, |
| "recall_at_k": { |
| "1": 0.5239999890327454, |
| "3": 0.9549999833106995, |
| "5": 0.996999979019165 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9539999961853027, |
| "5": 0.9959999918937683 |
| } |
| }, |
| "latest_state": { |
| "query_count": 457, |
| "recall_at_k": { |
| "1": 0.2352297604084015, |
| "3": 0.9157549142837524, |
| "5": 0.9945295453071594 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9146608114242554, |
| "5": 0.9934354424476624 |
| } |
| }, |
| "multi_hop_composition": { |
| "query_count": 348, |
| "recall_at_k": { |
| "1": 0.31104612350463867, |
| "3": 0.7073891758918762, |
| "5": 0.8984674215316772 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.46264368295669556, |
| "5": 0.8247126340866089 |
| } |
| }, |
| "temporal_localization": { |
| "query_count": 719, |
| "recall_at_k": { |
| "1": 0.8758693337440491, |
| "3": 0.9514371156692505, |
| "5": 0.9648817777633667 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9304589629173279, |
| "5": 0.9443671703338623 |
| } |
| } |
| } |
| }, |
| "skipped_control_query_count": 0, |
| "score_analysis": { |
| "overall": { |
| "rag_query_count": 3985, |
| "control_query_count": 111, |
| "gold_score": { |
| "count": 5141, |
| "mean": 0.7166337926745187, |
| "std": 0.17733149515563457, |
| "min": -0.11191169917583466, |
| "p05": 0.3657482862472534, |
| "p25": 0.6389928460121155, |
| "median": 0.7480946779251099, |
| "p75": 0.827989399433136, |
| "p95": 0.9548864960670471, |
| "max": 1.126090168952942 |
| }, |
| "best_gold_score": { |
| "count": 3985, |
| "mean": 0.755534760330522, |
| "std": 0.14572272433142164, |
| "min": 0.024165567010641098, |
| "p05": 0.490088027715683, |
| "p25": 0.6786190867424011, |
| "median": 0.7751450538635254, |
| "p75": 0.8467850089073181, |
| "p95": 0.9710816621780396, |
| "max": 1.126090168952942 |
| }, |
| "rag_max_candidate_score": { |
| "count": 3985, |
| "mean": 0.7640589419130699, |
| "std": 0.13383487103235178, |
| "min": 0.20186400413513184, |
| "p05": 0.5207033753395081, |
| "p25": 0.6838327050209045, |
| "median": 0.778085470199585, |
| "p75": 0.8501520752906799, |
| "p95": 0.9734190702438353, |
| "max": 1.129429817199707 |
| }, |
| "control_max_candidate_score": { |
| "count": 111, |
| "mean": 0.3709662235635626, |
| "std": 0.15236881273689565, |
| "min": 0.019797351211309433, |
| "p05": 0.13829854875802994, |
| "p25": 0.29179131984710693, |
| "median": 0.3588598668575287, |
| "p75": 0.46799521148204803, |
| "p95": 0.6187787652015686, |
| "max": 0.8129850029945374 |
| }, |
| "max_non_gold_score": { |
| "count": 3985, |
| "mean": 0.6232047809355681, |
| "std": 0.1810787727754309, |
| "min": 0.08055949211120605, |
| "p05": 0.3426193714141846, |
| "p25": 0.4937731623649597, |
| "median": 0.6113350987434387, |
| "p75": 0.746073842048645, |
| "p95": 0.9490036487579345, |
| "max": 1.129429817199707 |
| }, |
| "query_temporal_norm": { |
| "count": 4096, |
| "mean": 0.30273059851970174, |
| "std": 0.14540258547355747, |
| "min": 0.09085708111524582, |
| "p05": 0.12492499314248562, |
| "p25": 0.18279167264699936, |
| "median": 0.24912824481725693, |
| "p75": 0.4182180389761925, |
| "p95": 0.5543802678585052, |
| "max": 0.5741856694221497 |
| }, |
| "best_gold_margin": { |
| "count": 3985, |
| "mean": 0.132329979394954, |
| "std": 0.1506653521072949, |
| "min": -0.46813422441482544, |
| "p05": -0.05175083875656128, |
| "p25": 0.0041754841804504395, |
| "median": 0.11015433073043823, |
| "p75": 0.23661231994628906, |
| "p95": 0.4035195529460907, |
| "max": 0.7237380295991898 |
| }, |
| "weakest_gold_margin": { |
| "count": 3985, |
| "mean": 0.10693937787837834, |
| "std": 0.16802301214783116, |
| "min": -0.6715246737003326, |
| "p05": -0.13616411089897157, |
| "p25": -0.0015456676483154297, |
| "median": 0.09505456686019897, |
| "p75": 0.22502261400222778, |
| "p95": 0.37765879631042476, |
| "max": 0.7037184834480286 |
| }, |
| "top_score_best_balanced_threshold": { |
| "threshold": 0.5420975983142853, |
| "balanced_accuracy": 0.9165734115545909, |
| "true_positive_rate": 0.9322459222082811, |
| "false_positive_rate": 0.0990990990990991 |
| }, |
| "gold_score_best_balanced_threshold": { |
| "threshold": 0.5420975983142853, |
| "balanced_accuracy": 0.9085432986311279, |
| "true_positive_rate": 0.916185696361355, |
| "false_positive_rate": 0.0990990990990991 |
| } |
| }, |
| "by_task": { |
| "change_comparison": { |
| "rag_query_count": 470, |
| "control_query_count": 0, |
| "gold_score": { |
| "count": 942, |
| "mean": 0.7348885870520856, |
| "std": 0.1107366505292103, |
| "min": 0.06459831446409225, |
| "p05": 0.5372704863548279, |
| "p25": 0.6722842156887054, |
| "median": 0.7520450353622437, |
| "p75": 0.8101696819067001, |
| "p95": 0.8841885775327682, |
| "max": 0.978650689125061 |
| }, |
| "best_gold_score": { |
| "count": 470, |
| "mean": 0.7707346021494967, |
| "std": 0.09284867084255541, |
| "min": 0.28962263464927673, |
| "p05": 0.6213128983974456, |
| "p25": 0.7128853797912598, |
| "median": 0.7886177897453308, |
| "p75": 0.83125239610672, |
| "p95": 0.8947178870439529, |
| "max": 0.978650689125061 |
| }, |
| "rag_max_candidate_score": { |
| "count": 470, |
| "mean": 0.7754683313217569, |
| "std": 0.09230104027153883, |
| "min": 0.28962263464927673, |
| "p05": 0.6218573927879334, |
| "p25": 0.7183136194944382, |
| "median": 0.7937867343425751, |
| "p75": 0.8351891040802002, |
| "p95": 0.8965094208717346, |
| "max": 0.978650689125061 |
| }, |
| "control_max_candidate_score": null, |
| "max_non_gold_score": { |
| "count": 470, |
| "mean": 0.5437221028227756, |
| "std": 0.19519320044141864, |
| "min": 0.08743387460708618, |
| "p05": 0.24671123027801514, |
| "p25": 0.3817346394062042, |
| "median": 0.5233292877674103, |
| "p75": 0.710181474685669, |
| "p95": 0.8506679207086563, |
| "max": 0.9478694796562195 |
| }, |
| "query_temporal_norm": { |
| "count": 470, |
| "mean": 0.3904190681716229, |
| "std": 0.044412626312598104, |
| "min": 0.1604093760251999, |
| "p05": 0.3008479714393616, |
| "p25": 0.374271996319294, |
| "median": 0.3975691944360733, |
| "p75": 0.41577866673469543, |
| "p95": 0.44142436981201166, |
| "max": 0.5686288475990295 |
| }, |
| "best_gold_margin": { |
| "count": 470, |
| "mean": 0.22701249932672116, |
| "std": 0.18460189504901048, |
| "min": -0.09962069988250732, |
| "p05": -0.03665072619915009, |
| "p25": 0.06973769888281822, |
| "median": 0.22310630977153778, |
| "p75": 0.374038502573967, |
| "p95": 0.5218270637094974, |
| "max": 0.7237380295991898 |
| }, |
| "weakest_gold_margin": { |
| "count": 470, |
| "mean": 0.1590447726084831, |
| "std": 0.18752205523336984, |
| "min": -0.40735045820474625, |
| "p05": -0.11907580196857452, |
| "p25": 0.010347530245780945, |
| "median": 0.15443812310695648, |
| "p75": 0.317648746073246, |
| "p95": 0.4485425367951393, |
| "max": 0.6372086405754089 |
| }, |
| "top_score_best_balanced_threshold": null, |
| "gold_score_best_balanced_threshold": null |
| }, |
| "current_only_control": { |
| "rag_query_count": 0, |
| "control_query_count": 110, |
| "gold_score": null, |
| "best_gold_score": null, |
| "rag_max_candidate_score": null, |
| "control_max_candidate_score": { |
| "count": 110, |
| "mean": 0.3699303682385521, |
| "std": 0.15267026355881047, |
| "min": 0.019797351211309433, |
| "p05": 0.13738513067364694, |
| "p25": 0.29039132595062256, |
| "median": 0.3570753037929535, |
| "p75": 0.46439164876937866, |
| "p95": 0.6191292226314544, |
| "max": 0.8129850029945374 |
| }, |
| "max_non_gold_score": null, |
| "query_temporal_norm": { |
| "count": 110, |
| "mean": 0.2795836961404844, |
| "std": 0.1029300128190954, |
| "min": 0.11468714475631714, |
| "p05": 0.12837647050619125, |
| "p25": 0.1998436488211155, |
| "median": 0.2662823051214218, |
| "p75": 0.361461378633976, |
| "p95": 0.4507403194904327, |
| "max": 0.5361620187759399 |
| }, |
| "best_gold_margin": null, |
| "weakest_gold_margin": null, |
| "top_score_best_balanced_threshold": null, |
| "gold_score_best_balanced_threshold": null |
| }, |
| "direct_recall": { |
| "rag_query_count": 1491, |
| "control_query_count": 0, |
| "gold_score": { |
| "count": 1669, |
| "mean": 0.6795901385472988, |
| "std": 0.2012548187301753, |
| "min": -0.08538651466369629, |
| "p05": 0.2746641576290131, |
| "p25": 0.5686708092689514, |
| "median": 0.7269337177276611, |
| "p75": 0.8230971097946167, |
| "p95": 0.9368460059165953, |
| "max": 1.0985771417617798 |
| }, |
| "best_gold_score": { |
| "count": 1491, |
| "mean": 0.7201849106351934, |
| "std": 0.1623488035209848, |
| "min": 0.024165567010641098, |
| "p05": 0.43137773871421814, |
| "p25": 0.6312223970890045, |
| "median": 0.7476219534873962, |
| "p75": 0.8330177068710327, |
| "p95": 0.9440494775772095, |
| "max": 1.0985771417617798 |
| }, |
| "rag_max_candidate_score": { |
| "count": 1491, |
| "mean": 0.7322370440247553, |
| "std": 0.14161953978055097, |
| "min": 0.20186400413513184, |
| "p05": 0.4857248365879059, |
| "p25": 0.6375583410263062, |
| "median": 0.7500801086425781, |
| "p75": 0.8338922262191772, |
| "p95": 0.9440494775772095, |
| "max": 1.0985771417617798 |
| }, |
| "control_max_candidate_score": null, |
| "max_non_gold_score": { |
| "count": 1491, |
| "mean": 0.5614989415200903, |
| "std": 0.1312775402625397, |
| "min": 0.08055949211120605, |
| "p05": 0.34560634195804596, |
| "p25": 0.47706902027130127, |
| "median": 0.5617488026618958, |
| "p75": 0.6446017622947693, |
| "p95": 0.7837359011173248, |
| "max": 0.9488265514373779 |
| }, |
| "query_temporal_norm": { |
| "count": 1491, |
| "mean": 0.2243291862024148, |
| "std": 0.09722126786241793, |
| "min": 0.09085708111524582, |
| "p05": 0.11080540344119072, |
| "p25": 0.15130296349525452, |
| "median": 0.2039969563484192, |
| "p75": 0.26913149654865265, |
| "p95": 0.4267657548189163, |
| "max": 0.5707007646560669 |
| }, |
| "best_gold_margin": { |
| "count": 1491, |
| "mean": 0.1586859691151031, |
| "std": 0.15200214138339124, |
| "min": -0.46813422441482544, |
| "p05": -0.09121431410312653, |
| "p25": 0.06090536713600159, |
| "median": 0.1574101448059082, |
| "p75": 0.2646420896053314, |
| "p95": 0.4028832018375397, |
| "max": 0.7037184834480286 |
| }, |
| "weakest_gold_margin": { |
| "count": 1491, |
| "mean": 0.1412749828050559, |
| "std": 0.18043582198802646, |
| "min": -0.6715246737003326, |
| "p05": -0.19026869535446167, |
| "p25": 0.05169305205345154, |
| "median": 0.15304303169250488, |
| "p75": 0.2643876075744629, |
| "p95": 0.4028832018375397, |
| "max": 0.7037184834480286 |
| }, |
| "top_score_best_balanced_threshold": null, |
| "gold_score_best_balanced_threshold": null |
| }, |
| "historical_state": { |
| "rag_query_count": 500, |
| "control_query_count": 0, |
| "gold_score": { |
| "count": 502, |
| "mean": 0.7746741928130507, |
| "std": 0.10766275179939297, |
| "min": 0.14238934218883514, |
| "p05": 0.5862048655748368, |
| "p25": 0.7167049050331116, |
| "median": 0.793297290802002, |
| "p75": 0.8495824038982391, |
| "p95": 0.9157554745674134, |
| "max": 0.9862993359565735 |
| }, |
| "best_gold_score": { |
| "count": 500, |
| "mean": 0.7764696861505509, |
| "std": 0.1034113203801674, |
| "min": 0.3909410536289215, |
| "p05": 0.5930841654539108, |
| "p25": 0.7181234359741211, |
| "median": 0.7944096028804779, |
| "p75": 0.8500040471553802, |
| "p95": 0.9159746468067168, |
| "max": 0.9862993359565735 |
| }, |
| "rag_max_candidate_score": { |
| "count": 500, |
| "mean": 0.7806070187091827, |
| "std": 0.10195340881369842, |
| "min": 0.3909410536289215, |
| "p05": 0.5955297142267227, |
| "p25": 0.722739964723587, |
| "median": 0.7985008358955383, |
| "p75": 0.8531655073165894, |
| "p95": 0.9197906970977783, |
| "max": 0.9862993359565735 |
| }, |
| "control_max_candidate_score": null, |
| "max_non_gold_score": { |
| "count": 500, |
| "mean": 0.662049885392189, |
| "std": 0.14984154422872092, |
| "min": 0.15823492407798767, |
| "p05": 0.4040971785783768, |
| "p25": 0.5531154274940491, |
| "median": 0.6717543303966522, |
| "p75": 0.7823225259780884, |
| "p95": 0.8810094028711318, |
| "max": 0.9847855567932129 |
| }, |
| "query_temporal_norm": { |
| "count": 500, |
| "mean": 0.4881072618365288, |
| "std": 0.10037095674200244, |
| "min": 0.13309770822525024, |
| "p05": 0.2568978860974312, |
| "p25": 0.4548801779747009, |
| "median": 0.5330244600772858, |
| "p75": 0.5557828396558762, |
| "p95": 0.5690453529357911, |
| "max": 0.5741856694221497 |
| }, |
| "best_gold_margin": { |
| "count": 500, |
| "mean": 0.11441980075836182, |
| "std": 0.12297352134656454, |
| "min": -0.19667965173721313, |
| "p05": -0.029210487008094786, |
| "p25": 0.003396078944206238, |
| "median": 0.1014467179775238, |
| "p75": 0.19494780898094177, |
| "p95": 0.33460708707571024, |
| "max": 0.4947989881038666 |
| }, |
| "weakest_gold_margin": { |
| "count": 500, |
| "mean": 0.11334651419520378, |
| "std": 0.12532201212328375, |
| "min": -0.42393268644809723, |
| "p05": -0.029937261343002307, |
| "p25": 0.003252282738685608, |
| "median": 0.1014467179775238, |
| "p75": 0.19494780898094177, |
| "p95": 0.33460708707571024, |
| "max": 0.4947989881038666 |
| }, |
| "top_score_best_balanced_threshold": null, |
| "gold_score_best_balanced_threshold": null |
| }, |
| "latest_state": { |
| "rag_query_count": 457, |
| "control_query_count": 0, |
| "gold_score": { |
| "count": 460, |
| "mean": 0.9097354384544103, |
| "std": 0.11280339007167291, |
| "min": 0.2454087883234024, |
| "p05": 0.6985023379325866, |
| "p25": 0.8598119914531708, |
| "median": 0.9304486811161041, |
| "p75": 0.9877870827913284, |
| "p95": 1.0455342650413513, |
| "max": 1.126090168952942 |
| }, |
| "best_gold_score": { |
| "count": 457, |
| "mean": 0.9125979087219718, |
| "std": 0.10705335656055096, |
| "min": 0.2454087883234024, |
| "p05": 0.7246195793151855, |
| "p25": 0.8614522218704224, |
| "median": 0.9308992028236389, |
| "p75": 0.9884832501411438, |
| "p95": 1.0457399368286133, |
| "max": 1.126090168952942 |
| }, |
| "rag_max_candidate_score": { |
| "count": 457, |
| "mean": 0.9189921461648962, |
| "std": 0.10519501128335003, |
| "min": 0.3255910277366638, |
| "p05": 0.7326323986053467, |
| "p25": 0.867649495601654, |
| "median": 0.9345877766609192, |
| "p75": 0.9926048517227173, |
| "p95": 1.0541483640670777, |
| "max": 1.129429817199707 |
| }, |
| "control_max_candidate_score": null, |
| "max_non_gold_score": { |
| "count": 457, |
| "mean": 0.9165826158137499, |
| "std": 0.10844750208585242, |
| "min": 0.3255910277366638, |
| "p05": 0.7243556141853332, |
| "p25": 0.8655767440795898, |
| "median": 0.9336287379264832, |
| "p75": 0.990559995174408, |
| "p95": 1.0539359092712401, |
| "max": 1.129429817199707 |
| }, |
| "query_temporal_norm": { |
| "count": 457, |
| "mean": 0.5075552688190139, |
| "std": 0.0713984345837642, |
| "min": 0.16853883862495422, |
| "p05": 0.34919045567512513, |
| "p25": 0.5019967555999756, |
| "median": 0.5324681401252747, |
| "p75": 0.5476772785186768, |
| "p95": 0.5631026864051819, |
| "max": 0.5706198215484619 |
| }, |
| "best_gold_margin": { |
| "count": 457, |
| "mean": -0.0039847070917780605, |
| "std": 0.020039605127652755, |
| "min": -0.1066666841506958, |
| "p05": -0.04410884380340576, |
| "p25": -0.005139470100402832, |
| "median": -0.0010437369346618652, |
| "p75": 0.0023041367530822754, |
| "p95": 0.008276724815368652, |
| "max": 0.21554291248321533 |
| }, |
| "weakest_gold_margin": { |
| "count": 457, |
| "mean": -0.004690558468226009, |
| "std": 0.016042674602134533, |
| "min": -0.1066666841506958, |
| "p05": -0.04410884380340576, |
| "p25": -0.005158364772796631, |
| "median": -0.0011911392211914062, |
| "p75": 0.0022735595703125, |
| "p95": 0.008188319206237792, |
| "max": 0.1005626916885376 |
| }, |
| "top_score_best_balanced_threshold": null, |
| "gold_score_best_balanced_threshold": null |
| }, |
| "multi_hop_composition": { |
| "rag_query_count": 348, |
| "control_query_count": 0, |
| "gold_score": { |
| "count": 773, |
| "mean": 0.6499942918171407, |
| "std": 0.1690562525488606, |
| "min": -0.11191169917583466, |
| "p05": 0.2923929452896118, |
| "p25": 0.6014822125434875, |
| "median": 0.6969712376594543, |
| "p75": 0.7639080286026001, |
| "p95": 0.8181564331054687, |
| "max": 0.8747819662094116 |
| }, |
| "best_gold_score": { |
| "count": 348, |
| "mean": 0.7210219436507116, |
| "std": 0.09544151448010416, |
| "min": 0.34543609619140625, |
| "p05": 0.5357065349817276, |
| "p25": 0.6784490048885345, |
| "median": 0.7423729598522186, |
| "p75": 0.7895018309354782, |
| "p95": 0.8334888577461242, |
| "max": 0.8747819662094116 |
| }, |
| "rag_max_candidate_score": { |
| "count": 348, |
| "mean": 0.7313055699241573, |
| "std": 0.08715730799173863, |
| "min": 0.4189240634441376, |
| "p05": 0.5543014436960221, |
| "p25": 0.6850295513868332, |
| "median": 0.7491745948791504, |
| "p75": 0.7964175790548325, |
| "p95": 0.8351482093334198, |
| "max": 0.8747819662094116 |
| }, |
| "control_max_candidate_score": null, |
| "max_non_gold_score": { |
| "count": 348, |
| "mean": 0.6516784167495268, |
| "std": 0.14944315060991253, |
| "min": 0.15956036746501923, |
| "p05": 0.3637311488389969, |
| "p25": 0.5521483868360519, |
| "median": 0.7000037133693695, |
| "p75": 0.771206796169281, |
| "p95": 0.823851415514946, |
| "max": 0.8726747035980225 |
| }, |
| "query_temporal_norm": { |
| "count": 348, |
| "mean": 0.19508305329015885, |
| "std": 0.03477950168836244, |
| "min": 0.13012589514255524, |
| "p05": 0.14681573063135148, |
| "p25": 0.16908932477235794, |
| "median": 0.191594660282135, |
| "p75": 0.21390236169099808, |
| "p95": 0.2606314122676849, |
| "max": 0.3217720091342926 |
| }, |
| "best_gold_margin": { |
| "count": 348, |
| "mean": 0.06934352690118482, |
| "std": 0.12927293245479557, |
| "min": -0.18237540125846863, |
| "p05": -0.05678632855415344, |
| "p25": -0.011113137006759644, |
| "median": 0.01789069175720215, |
| "p75": 0.0949355959892273, |
| "p95": 0.3652027100324629, |
| "max": 0.5515869557857513 |
| }, |
| "weakest_gold_margin": { |
| "count": 348, |
| "mean": -0.01585723184655961, |
| "std": 0.16034559704635334, |
| "min": -0.6282003968954086, |
| "p05": -0.2646480455994606, |
| "p25": -0.07698842138051987, |
| "median": -0.03306916356086731, |
| "p75": 0.008552595973014832, |
| "p95": 0.3040800869464872, |
| "max": 0.5013379752635956 |
| }, |
| "top_score_best_balanced_threshold": null, |
| "gold_score_best_balanced_threshold": null |
| }, |
| "recent_context_control": { |
| "rag_query_count": 0, |
| "control_query_count": 1, |
| "gold_score": null, |
| "best_gold_score": null, |
| "rag_max_candidate_score": null, |
| "control_max_candidate_score": { |
| "count": 1, |
| "mean": 0.4849103093147278, |
| "std": 0.0, |
| "min": 0.4849103093147278, |
| "p05": 0.4849103093147278, |
| "p25": 0.4849103093147278, |
| "median": 0.4849103093147278, |
| "p75": 0.4849103093147278, |
| "p95": 0.4849103093147278, |
| "max": 0.4849103093147278 |
| }, |
| "max_non_gold_score": null, |
| "query_temporal_norm": { |
| "count": 1, |
| "mean": 0.17241171002388, |
| "std": 0.0, |
| "min": 0.17241171002388, |
| "p05": 0.17241171002388, |
| "p25": 0.17241171002388, |
| "median": 0.17241171002388, |
| "p75": 0.17241171002388, |
| "p95": 0.17241171002388, |
| "max": 0.17241171002388 |
| }, |
| "best_gold_margin": null, |
| "weakest_gold_margin": null, |
| "top_score_best_balanced_threshold": null, |
| "gold_score_best_balanced_threshold": null |
| }, |
| "temporal_localization": { |
| "rag_query_count": 719, |
| "control_query_count": 0, |
| "gold_score": { |
| "count": 795, |
| "mean": 0.6891861683589482, |
| "std": 0.1710879451771218, |
| "min": -0.10147331655025482, |
| "p05": 0.347307488322258, |
| "p25": 0.6123415529727936, |
| "median": 0.7237328290939331, |
| "p75": 0.8111500144004822, |
| "p95": 0.8972467362880706, |
| "max": 1.0365782976150513 |
| }, |
| "best_gold_score": { |
| "count": 719, |
| "mean": 0.7212202105676349, |
| "std": 0.13338234880123853, |
| "min": 0.12394365668296814, |
| "p05": 0.4685472249984741, |
| "p25": 0.6464675962924957, |
| "median": 0.7408140301704407, |
| "p75": 0.8183608055114746, |
| "p95": 0.9048629462718963, |
| "max": 1.0365782976150513 |
| }, |
| "rag_max_candidate_score": { |
| "count": 719, |
| "mean": 0.7284590774351765, |
| "std": 0.12147799906182175, |
| "min": 0.2631508409976959, |
| "p05": 0.5057044625282288, |
| "p25": 0.6554280817508698, |
| "median": 0.7423368692398071, |
| "p75": 0.818709522485733, |
| "p95": 0.9048629462718963, |
| "max": 1.0365782976150513 |
| }, |
| "control_max_candidate_score": null, |
| "max_non_gold_score": { |
| "count": 719, |
| "mean": 0.5758545963049598, |
| "std": 0.12951872399452016, |
| "min": 0.2200310379266739, |
| "p05": 0.3654164135456085, |
| "p25": 0.474811851978302, |
| "median": 0.5770975351333618, |
| "p75": 0.6710145175457001, |
| "p95": 0.7880617022514345, |
| "max": 0.9568344950675964 |
| }, |
| "query_temporal_norm": { |
| "count": 719, |
| "mean": 0.20471605461645193, |
| "std": 0.05866639582901708, |
| "min": 0.10267208516597748, |
| "p05": 0.12938569486141205, |
| "p25": 0.1643894836306572, |
| "median": 0.19598978757858276, |
| "p75": 0.22830134630203247, |
| "p95": 0.31625273823738104, |
| "max": 0.5251786112785339 |
| }, |
| "best_gold_margin": { |
| "count": 719, |
| "mean": 0.14536561426267505, |
| "std": 0.12385513267341426, |
| "min": -0.3723912537097931, |
| "p05": -0.04747639298439026, |
| "p25": 0.07192777097225189, |
| "median": 0.1291063129901886, |
| "p75": 0.2247702181339264, |
| "p95": 0.35997496843338017, |
| "max": 0.5334089994430542 |
| }, |
| "weakest_gold_margin": { |
| "count": 719, |
| "mean": 0.12760782080944705, |
| "std": 0.15015979413924416, |
| "min": -0.5440109521150589, |
| "p05": -0.1358781337738037, |
| "p25": 0.057646214962005615, |
| "median": 0.12182247638702393, |
| "p75": 0.22186627984046936, |
| "p95": 0.3576861590147019, |
| "max": 0.5334089994430542 |
| }, |
| "top_score_best_balanced_threshold": null, |
| "gold_score_best_balanced_threshold": null |
| } |
| } |
| } |
| } |
| }, |
| { |
| "epoch": 2, |
| "checkpoint": "/home/zhangchao/work/minye_training/mixed_retrieval_normal_identity_balanced/retriever_epoch_02.pt", |
| "train_loss": 0.7708213925361633, |
| "train_losses": { |
| "total": 0.7708213925361633, |
| "global": 1.041038990020752, |
| "hard_negative": 0.5432254076004028, |
| "semantic": 0.7572951316833496, |
| "temporal_null": 0.041718825697898865, |
| "speaker_name_renamed_samples": 639, |
| "speaker_name_skipped_mixed_or_nonself": 4108, |
| "speaker_name_skipped_spoken_name": 267, |
| "speaker_name_skipped_insufficient_names": 0 |
| }, |
| "validation": { |
| "query_count": 3985, |
| "recall_at_k": { |
| "1": 0.7014623880386353, |
| "3": 0.9176109433174133, |
| "5": 0.9573962092399597 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.8742785453796387, |
| "5": 0.9345043897628784 |
| }, |
| "by_speaker_query_type": { |
| "fact_to_person": { |
| "query_count": 786, |
| "recall_at_k": { |
| "1": 0.8212059736251831, |
| "3": 0.97481369972229, |
| "5": 0.9802631735801697 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9631043076515198, |
| "5": 0.9656488299369812 |
| } |
| }, |
| "person_to_fact": { |
| "query_count": 969, |
| "recall_at_k": { |
| "1": 0.749356210231781, |
| "3": 0.9477566480636597, |
| "5": 0.9758120775222778 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9205366373062134, |
| "5": 0.968008279800415 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 509, |
| "recall_at_k": { |
| "1": 0.28176164627075195, |
| "3": 0.6972822546958923, |
| "5": 0.9101179242134094 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.5009823441505432, |
| "5": 0.8408644199371338 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 1721, |
| "recall_at_k": { |
| "1": 0.7439375519752502, |
| "3": 0.9396765232086182, |
| "5": 0.9505664706230164 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.918070912361145, |
| "5": 0.9291110038757324 |
| } |
| } |
| }, |
| "by_task_and_speaker_query_type": { |
| "change_comparison": { |
| "fact_to_person": { |
| "query_count": 74, |
| "recall_at_k": { |
| "1": 0.5, |
| "3": 1.0, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 1.0, |
| "5": 1.0 |
| } |
| }, |
| "person_to_fact": { |
| "query_count": 88, |
| "recall_at_k": { |
| "1": 0.5, |
| "3": 0.9602272510528564, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9204545617103577, |
| "5": 1.0 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 99, |
| "recall_at_k": { |
| "1": 0.13131313025951385, |
| "3": 0.5101010203361511, |
| "5": 0.8282828330993652 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.13131313025951385, |
| "5": 0.6565656661987305 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 209, |
| "recall_at_k": { |
| "1": 0.49441784620285034, |
| "3": 0.9920255541801453, |
| "5": 0.9932217597961426 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9904305934906006, |
| "5": 0.9904305934906006 |
| } |
| } |
| }, |
| "direct_recall": { |
| "fact_to_person": { |
| "query_count": 524, |
| "recall_at_k": { |
| "1": 0.9276399612426758, |
| "3": 0.9723281860351562, |
| "5": 0.980120837688446 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9599236249923706, |
| "5": 0.9637404680252075 |
| } |
| }, |
| "person_to_fact": { |
| "query_count": 306, |
| "recall_at_k": { |
| "1": 0.813725471496582, |
| "3": 0.9327341914176941, |
| "5": 0.9507080316543579 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9215686321258545, |
| "5": 0.9346405267715454 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 57, |
| "recall_at_k": { |
| "1": 0.42836257815361023, |
| "3": 0.6067252159118652, |
| "5": 0.7032163739204407 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.4385964870452881, |
| "5": 0.5789473652839661 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 604, |
| "recall_at_k": { |
| "1": 0.8134657144546509, |
| "3": 0.9046633839607239, |
| "5": 0.9165287613868713 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.8791390657424927, |
| "5": 0.8923841118812561 |
| } |
| } |
| }, |
| "historical_state": { |
| "fact_to_person": { |
| "query_count": 91, |
| "recall_at_k": { |
| "1": 0.8351648449897766, |
| "3": 1.0, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 1.0, |
| "5": 1.0 |
| } |
| }, |
| "person_to_fact": { |
| "query_count": 125, |
| "recall_at_k": { |
| "1": 0.871999979019165, |
| "3": 1.0, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 1.0, |
| "5": 1.0 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 82, |
| "recall_at_k": { |
| "1": 0.3658536672592163, |
| "3": 0.9512194991111755, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9512194991111755, |
| "5": 1.0 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 202, |
| "recall_at_k": { |
| "1": 0.9207921028137207, |
| "3": 0.9975247383117676, |
| "5": 0.9975247383117676 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9950494766235352, |
| "5": 0.9950494766235352 |
| } |
| } |
| }, |
| "latest_state": { |
| "fact_to_person": { |
| "query_count": 31, |
| "recall_at_k": { |
| "1": 0.5161290168762207, |
| "3": 1.0, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 1.0, |
| "5": 1.0 |
| } |
| }, |
| "person_to_fact": { |
| "query_count": 92, |
| "recall_at_k": { |
| "1": 0.5163043737411499, |
| "3": 1.0, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 1.0, |
| "5": 1.0 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 82, |
| "recall_at_k": { |
| "1": 0.24390242993831635, |
| "3": 0.7682926654815674, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.7682926654815674, |
| "5": 1.0 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 252, |
| "recall_at_k": { |
| "1": 0.5198412537574768, |
| "3": 0.9900793433189392, |
| "5": 0.9980158805847168 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.988095223903656, |
| "5": 0.9960317611694336 |
| } |
| } |
| }, |
| "multi_hop_composition": { |
| "fact_to_person": { |
| "query_count": 66, |
| "recall_at_k": { |
| "1": 0.46037158370018005, |
| "3": 0.9197511076927185, |
| "5": 0.9227814078330994 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.8787878751754761, |
| "5": 0.8787878751754761 |
| } |
| }, |
| "person_to_fact": { |
| "query_count": 84, |
| "recall_at_k": { |
| "1": 0.23662133514881134, |
| "3": 0.722732424736023, |
| "5": 0.9392289519309998 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.5, |
| "5": 0.9166666865348816 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 168, |
| "recall_at_k": { |
| "1": 0.2678571343421936, |
| "3": 0.675595223903656, |
| "5": 0.9523809552192688 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.380952388048172, |
| "5": 0.9047619104385376 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 30, |
| "recall_at_k": { |
| "1": 0.24944445490837097, |
| "3": 0.4811111390590668, |
| "5": 0.5475000143051147 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.1666666716337204, |
| "5": 0.23333333432674408 |
| } |
| } |
| }, |
| "temporal_localization": { |
| "person_to_fact": { |
| "query_count": 274, |
| "recall_at_k": { |
| "1": 0.9370437860488892, |
| "3": 0.9881386756896973, |
| "5": 0.9881386756896973 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.985401451587677, |
| "5": 0.985401451587677 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 21, |
| "recall_at_k": { |
| "1": 0.5238094925880432, |
| "3": 0.7301587462425232, |
| "5": 0.8174603581428528 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.5714285969734192, |
| "5": 0.6666666865348816 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 424, |
| "recall_at_k": { |
| "1": 0.8518081307411194, |
| "3": 0.9386793375015259, |
| "5": 0.9559748768806458 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9127358198165894, |
| "5": 0.9292452931404114 |
| } |
| } |
| } |
| }, |
| "by_task": { |
| "change_comparison": { |
| "query_count": 470, |
| "recall_at_k": { |
| "1": 0.4198581874370575, |
| "3": 0.8858155608177185, |
| "5": 0.9608155488967896 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.7978723645210266, |
| "5": 0.9234042763710022 |
| } |
| }, |
| "direct_recall": { |
| "query_count": 1491, |
| "recall_at_k": { |
| "1": 0.838922381401062, |
| "3": 0.9228145480155945, |
| "5": 0.9377375245094299 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.8993963599205017, |
| "5": 0.9141515493392944 |
| } |
| }, |
| "historical_state": { |
| "query_count": 500, |
| "recall_at_k": { |
| "1": 0.8019999861717224, |
| "3": 0.9909999966621399, |
| "5": 0.9990000128746033 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9900000095367432, |
| "5": 0.9980000257492065 |
| } |
| }, |
| "latest_state": { |
| "query_count": 457, |
| "recall_at_k": { |
| "1": 0.4693654179573059, |
| "3": 0.9529540538787842, |
| "5": 0.9989058971405029 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9518599510192871, |
| "5": 0.9978117942810059 |
| } |
| }, |
| "multi_hop_composition": { |
| "query_count": 348, |
| "recall_at_k": { |
| "1": 0.29524150490760803, |
| "3": 0.7165126800537109, |
| "5": 0.9086891412734985 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.48563218116760254, |
| "5": 0.8448275923728943 |
| } |
| }, |
| "temporal_localization": { |
| "query_count": 719, |
| "recall_at_k": { |
| "1": 0.874710202217102, |
| "3": 0.9514370560646057, |
| "5": 0.9641863703727722 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9304589629173279, |
| "5": 0.9429763555526733 |
| } |
| } |
| }, |
| "semantic_only": { |
| "query_count": 3985, |
| "recall_at_k": { |
| "1": 0.647217333316803, |
| "3": 0.9038509726524353, |
| "5": 0.9575216770172119 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.8592220544815063, |
| "5": 0.9340025186538696 |
| }, |
| "by_speaker_query_type": { |
| "fact_to_person": { |
| "query_count": 786, |
| "recall_at_k": { |
| "1": 0.7639540433883667, |
| "3": 0.9737535119056702, |
| "5": 0.9808993339538574 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9618320465087891, |
| "5": 0.9669210910797119 |
| } |
| }, |
| "person_to_fact": { |
| "query_count": 969, |
| "recall_at_k": { |
| "1": 0.6972406506538391, |
| "3": 0.9425966739654541, |
| "5": 0.9752960801124573 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9143446683883667, |
| "5": 0.9669762849807739 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 509, |
| "recall_at_k": { |
| "1": 0.2503274381160736, |
| "3": 0.6412900686264038, |
| "5": 0.9058611989021301 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.4381139576435089, |
| "5": 0.8330059051513672 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 1721, |
| "recall_at_k": { |
| "1": 0.6831203103065491, |
| "3": 0.927764892578125, |
| "5": 0.9521159529685974 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9058687090873718, |
| "5": 0.9302731156349182 |
| } |
| } |
| }, |
| "by_task_and_speaker_query_type": { |
| "change_comparison": { |
| "fact_to_person": { |
| "query_count": 74, |
| "recall_at_k": { |
| "1": 0.5, |
| "3": 1.0, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 1.0, |
| "5": 1.0 |
| } |
| }, |
| "person_to_fact": { |
| "query_count": 88, |
| "recall_at_k": { |
| "1": 0.5, |
| "3": 0.9545454382896423, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9090908765792847, |
| "5": 1.0 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 99, |
| "recall_at_k": { |
| "1": 0.1262626200914383, |
| "3": 0.47979798913002014, |
| "5": 0.8232322931289673 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.08080808073282242, |
| "5": 0.6464646458625793 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 209, |
| "recall_at_k": { |
| "1": 0.4920254945755005, |
| "3": 0.9932217597961426, |
| "5": 0.9932217597961426 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9904305934906006, |
| "5": 0.9904305934906006 |
| } |
| } |
| }, |
| "direct_recall": { |
| "fact_to_person": { |
| "query_count": 524, |
| "recall_at_k": { |
| "1": 0.9276399612426758, |
| "3": 0.970737874507904, |
| "5": 0.9810749888420105 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9580152630805969, |
| "5": 0.9656488299369812 |
| } |
| }, |
| "person_to_fact": { |
| "query_count": 306, |
| "recall_at_k": { |
| "1": 0.7941176295280457, |
| "3": 0.9360021352767944, |
| "5": 0.9507080316543579 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9248365759849548, |
| "5": 0.9346405267715454 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 57, |
| "recall_at_k": { |
| "1": 0.42836257815361023, |
| "3": 0.5716373920440674, |
| "5": 0.7090643644332886 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.38596490025520325, |
| "5": 0.5789473652839661 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 604, |
| "recall_at_k": { |
| "1": 0.8151213526725769, |
| "3": 0.9050772786140442, |
| "5": 0.9195640683174133 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.8791390657424927, |
| "5": 0.8940397500991821 |
| } |
| } |
| }, |
| "historical_state": { |
| "fact_to_person": { |
| "query_count": 91, |
| "recall_at_k": { |
| "1": 0.4175824224948883, |
| "3": 1.0, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 1.0, |
| "5": 1.0 |
| } |
| }, |
| "person_to_fact": { |
| "query_count": 125, |
| "recall_at_k": { |
| "1": 0.5839999914169312, |
| "3": 0.9839999675750732, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9839999675750732, |
| "5": 1.0 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 82, |
| "recall_at_k": { |
| "1": 0.24390242993831635, |
| "3": 0.7682926654815674, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.7682926654815674, |
| "5": 1.0 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 202, |
| "recall_at_k": { |
| "1": 0.6732673048973083, |
| "3": 0.9925742745399475, |
| "5": 0.9975247383117676 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9900990128517151, |
| "5": 0.9950494766235352 |
| } |
| } |
| }, |
| "latest_state": { |
| "fact_to_person": { |
| "query_count": 31, |
| "recall_at_k": { |
| "1": 0.29032257199287415, |
| "3": 1.0, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 1.0, |
| "5": 1.0 |
| } |
| }, |
| "person_to_fact": { |
| "query_count": 92, |
| "recall_at_k": { |
| "1": 0.39673912525177, |
| "3": 0.967391312122345, |
| "5": 1.0 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.967391312122345, |
| "5": 1.0 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 82, |
| "recall_at_k": { |
| "1": 0.15853658318519592, |
| "3": 0.6707317233085632, |
| "5": 0.9878048896789551 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.6707317233085632, |
| "5": 0.9878048896789551 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 252, |
| "recall_at_k": { |
| "1": 0.2976190447807312, |
| "3": 0.9107142686843872, |
| "5": 0.9980158805847168 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.908730149269104, |
| "5": 0.9960317611694336 |
| } |
| } |
| }, |
| "multi_hop_composition": { |
| "fact_to_person": { |
| "query_count": 66, |
| "recall_at_k": { |
| "1": 0.46037158370018005, |
| "3": 0.9197511076927185, |
| "5": 0.9227814078330994 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.8787878751754761, |
| "5": 0.8787878751754761 |
| } |
| }, |
| "person_to_fact": { |
| "query_count": 84, |
| "recall_at_k": { |
| "1": 0.24852609634399414, |
| "3": 0.7167800068855286, |
| "5": 0.9332765936851501 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.488095223903656, |
| "5": 0.9047619104385376 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 168, |
| "recall_at_k": { |
| "1": 0.2767857015132904, |
| "3": 0.6726190447807312, |
| "5": 0.949404776096344 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.375, |
| "5": 0.8988094925880432 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 30, |
| "recall_at_k": { |
| "1": 0.2661111056804657, |
| "3": 0.4811111390590668, |
| "5": 0.5586110949516296 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.1666666716337204, |
| "5": 0.23333333432674408 |
| } |
| } |
| }, |
| "temporal_localization": { |
| "person_to_fact": { |
| "query_count": 274, |
| "recall_at_k": { |
| "1": 0.9425182342529297, |
| "3": 0.9881386756896973, |
| "5": 0.9881386756896973 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.985401451587677, |
| "5": 0.985401451587677 |
| } |
| }, |
| "self_to_fact": { |
| "query_count": 21, |
| "recall_at_k": { |
| "1": 0.5238094925880432, |
| "3": 0.7301587462425232, |
| "5": 0.7936508655548096 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.5714285969734192, |
| "5": 0.6190476417541504 |
| } |
| }, |
| "speaker_independent": { |
| "query_count": 424, |
| "recall_at_k": { |
| "1": 0.852594256401062, |
| "3": 0.9386793375015259, |
| "5": 0.9571540951728821 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9127358198165894, |
| "5": 0.9316037893295288 |
| } |
| } |
| } |
| }, |
| "by_task": { |
| "change_comparison": { |
| "query_count": 470, |
| "recall_at_k": { |
| "1": 0.41773051023483276, |
| "3": 0.8789006471633911, |
| "5": 0.9597517251968384 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.785106360912323, |
| "5": 0.9212765693664551 |
| } |
| }, |
| "direct_recall": { |
| "query_count": 1491, |
| "recall_at_k": { |
| "1": 0.835568904876709, |
| "3": 0.9217526912689209, |
| "5": 0.9395260214805603 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.8973842859268188, |
| "5": 0.9154929518699646 |
| } |
| }, |
| "historical_state": { |
| "query_count": 500, |
| "recall_at_k": { |
| "1": 0.5339999794960022, |
| "3": 0.9549999833106995, |
| "5": 0.9990000128746033 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9539999961853027, |
| "5": 0.9980000257492065 |
| } |
| }, |
| "latest_state": { |
| "query_count": 457, |
| "recall_at_k": { |
| "1": 0.2921225428581238, |
| "3": 0.8851203322410583, |
| "5": 0.9967177510261536 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.8840262293815613, |
| "5": 0.9956236481666565 |
| } |
| }, |
| "multi_hop_composition": { |
| "query_count": 348, |
| "recall_at_k": { |
| "1": 0.30386221408843994, |
| "3": 0.7136391401290894, |
| "5": 0.9067734479904175 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.4798850417137146, |
| "5": 0.8390804529190063 |
| } |
| }, |
| "temporal_localization": { |
| "query_count": 719, |
| "recall_at_k": { |
| "1": 0.8772600889205933, |
| "3": 0.9514370560646057, |
| "5": 0.9641863703727722 |
| }, |
| "all_support_em_at_k": { |
| "3": 0.9304589629173279, |
| "5": 0.9429763555526733 |
| } |
| } |
| } |
| }, |
| "skipped_control_query_count": 0, |
| "score_analysis": { |
| "overall": { |
| "rag_query_count": 3985, |
| "control_query_count": 111, |
| "gold_score": { |
| "count": 5141, |
| "mean": 0.7241762260820077, |
| "std": 0.17420755087169498, |
| "min": -0.08927247673273087, |
| "p05": 0.375575453042984, |
| "p25": 0.6503528356552124, |
| "median": 0.7556952834129333, |
| "p75": 0.8322693705558777, |
| "p95": 0.9593976736068726, |
| "max": 1.1258333921432495 |
| }, |
| "best_gold_score": { |
| "count": 3985, |
| "mean": 0.7619477864668016, |
| "std": 0.14381897115710743, |
| "min": 0.03736429661512375, |
| "p05": 0.4997822523117066, |
| "p25": 0.6846296191215515, |
| "median": 0.7807751297950745, |
| "p75": 0.8531866669654846, |
| "p95": 0.9743822336196898, |
| "max": 1.1258333921432495 |
| }, |
| "rag_max_candidate_score": { |
| "count": 3985, |
| "mean": 0.7705392655083346, |
| "std": 0.13204878015276228, |
| "min": 0.1924441009759903, |
| "p05": 0.5299609899520874, |
| "p25": 0.6920320987701416, |
| "median": 0.7846444249153137, |
| "p75": 0.8553236126899719, |
| "p95": 0.9765936732292174, |
| "max": 1.1260788440704346 |
| }, |
| "control_max_candidate_score": { |
| "count": 111, |
| "mean": 0.3882304532369515, |
| "std": 0.15184264623213228, |
| "min": 0.03674992173910141, |
| "p05": 0.1377359926700592, |
| "p25": 0.2954407185316086, |
| "median": 0.37531644105911255, |
| "p75": 0.47543179988861084, |
| "p95": 0.6330483555793762, |
| "max": 0.8106650710105896 |
| }, |
| "max_non_gold_score": { |
| "count": 3985, |
| "mean": 0.630470616324633, |
| "std": 0.17867460925989845, |
| "min": 0.10479071736335754, |
| "p05": 0.3529604732990265, |
| "p25": 0.5026586651802063, |
| "median": 0.6192934513092041, |
| "p75": 0.7512153387069702, |
| "p95": 0.9490407824516295, |
| "max": 1.1260788440704346 |
| }, |
| "query_temporal_norm": { |
| "count": 4096, |
| "mean": 0.304678496593624, |
| "std": 0.1486443855292593, |
| "min": 0.08168468624353409, |
| "p05": 0.11245217174291611, |
| "p25": 0.18248226121068, |
| "median": 0.25531914830207825, |
| "p75": 0.42087820917367935, |
| "p95": 0.5570324063301086, |
| "max": 0.5792937874794006 |
| }, |
| "best_gold_margin": { |
| "count": 3985, |
| "mean": 0.13147717014216867, |
| "std": 0.15046855171086562, |
| "min": -0.48386645317077637, |
| "p05": -0.05386531352996825, |
| "p25": 0.0042411088943481445, |
| "median": 0.10900372266769409, |
| "p75": 0.23411086201667786, |
| "p95": 0.4013371586799621, |
| "max": 0.7206796705722809 |
| }, |
| "weakest_gold_margin": { |
| "count": 3985, |
| "mean": 0.10665875107915826, |
| "std": 0.16716168516628985, |
| "min": -0.6774372458457947, |
| "p05": -0.13410429954528807, |
| "p25": -0.001153707504272461, |
| "median": 0.09585797786712646, |
| "p75": 0.22194325923919678, |
| "p95": 0.380974495410919, |
| "max": 0.7206796705722809 |
| }, |
| "top_score_best_balanced_threshold": { |
| "threshold": 0.619530200958252, |
| "balanced_accuracy": 0.9084811285564109, |
| "true_positive_rate": 0.8710163111668758, |
| "false_positive_rate": 0.05405405405405406 |
| }, |
| "gold_score_best_balanced_threshold": { |
| "threshold": 0.619530200958252, |
| "balanced_accuracy": 0.9023330733493846, |
| "true_positive_rate": 0.8587202007528231, |
| "false_positive_rate": 0.05405405405405406 |
| } |
| }, |
| "by_task": { |
| "change_comparison": { |
| "rag_query_count": 470, |
| "control_query_count": 0, |
| "gold_score": { |
| "count": 942, |
| "mean": 0.7404991486950159, |
| "std": 0.10894139741365653, |
| "min": 0.06327641755342484, |
| "p05": 0.5448947280645371, |
| "p25": 0.6833753138780594, |
| "median": 0.7579014599323273, |
| "p75": 0.8139380216598511, |
| "p95": 0.8874219119548797, |
| "max": 0.9721137285232544 |
| }, |
| "best_gold_score": { |
| "count": 470, |
| "mean": 0.7755538869411387, |
| "std": 0.09052670544988363, |
| "min": 0.33879566192626953, |
| "p05": 0.6262453407049179, |
| "p25": 0.7201669067144394, |
| "median": 0.7918211221694946, |
| "p75": 0.8364439904689789, |
| "p95": 0.8994863331317902, |
| "max": 0.9721137285232544 |
| }, |
| "rag_max_candidate_score": { |
| "count": 470, |
| "mean": 0.7802391093462071, |
| "std": 0.08961716909016448, |
| "min": 0.33879566192626953, |
| "p05": 0.6289204120635986, |
| "p25": 0.723452016711235, |
| "median": 0.7953038215637207, |
| "p75": 0.8430169820785522, |
| "p95": 0.9019855648279189, |
| "max": 0.9721137285232544 |
| }, |
| "control_max_candidate_score": null, |
| "max_non_gold_score": { |
| "count": 470, |
| "mean": 0.548556295322611, |
| "std": 0.19348231002676886, |
| "min": 0.10479071736335754, |
| "p05": 0.25167940706014635, |
| "p25": 0.38838403671979904, |
| "median": 0.5346508920192719, |
| "p75": 0.7160655558109283, |
| "p95": 0.8489899426698684, |
| "max": 0.9398146867752075 |
| }, |
| "query_temporal_norm": { |
| "count": 470, |
| "mean": 0.38702126294374467, |
| "std": 0.04252809385217023, |
| "min": 0.17005102336406708, |
| "p05": 0.30043437480926516, |
| "p25": 0.37197282910346985, |
| "median": 0.3936518132686615, |
| "p75": 0.41210484504699707, |
| "p95": 0.4362030506134033, |
| "max": 0.568455159664154 |
| }, |
| "best_gold_margin": { |
| "count": 470, |
| "mean": 0.22699759161852776, |
| "std": 0.1835967645486095, |
| "min": -0.10016191005706787, |
| "p05": -0.03408718109130859, |
| "p25": 0.07795208692550659, |
| "median": 0.22617237269878387, |
| "p75": 0.37646569311618805, |
| "p95": 0.5155857145786286, |
| "max": 0.699258029460907 |
| }, |
| "weakest_gold_margin": { |
| "count": 470, |
| "mean": 0.16063123863745243, |
| "std": 0.1859837532729084, |
| "min": -0.4117041751742363, |
| "p05": -0.11187890470027922, |
| "p25": 0.014072544872760773, |
| "median": 0.1509234607219696, |
| "p75": 0.3116276040673256, |
| "p95": 0.45314110368490207, |
| "max": 0.6590308099985123 |
| }, |
| "top_score_best_balanced_threshold": null, |
| "gold_score_best_balanced_threshold": null |
| }, |
| "current_only_control": { |
| "rag_query_count": 0, |
| "control_query_count": 110, |
| "gold_score": null, |
| "best_gold_score": null, |
| "rag_max_candidate_score": null, |
| "control_max_candidate_score": { |
| "count": 110, |
| "mean": 0.3872418130324645, |
| "std": 0.1521752229685398, |
| "min": 0.03674992173910141, |
| "p05": 0.1367964342236519, |
| "p25": 0.2949500158429146, |
| "median": 0.37395188212394714, |
| "p75": 0.4682910218834877, |
| "p95": 0.634412682056427, |
| "max": 0.8106650710105896 |
| }, |
| "max_non_gold_score": null, |
| "query_temporal_norm": { |
| "count": 110, |
| "mean": 0.3163933543996377, |
| "std": 0.12005832986975279, |
| "min": 0.10356259346008301, |
| "p05": 0.12996669486165047, |
| "p25": 0.20454848557710648, |
| "median": 0.3282453566789627, |
| "p75": 0.413640059530735, |
| "p95": 0.5012234553694724, |
| "max": 0.5566943883895874 |
| }, |
| "best_gold_margin": null, |
| "weakest_gold_margin": null, |
| "top_score_best_balanced_threshold": null, |
| "gold_score_best_balanced_threshold": null |
| }, |
| "direct_recall": { |
| "rag_query_count": 1491, |
| "control_query_count": 0, |
| "gold_score": { |
| "count": 1669, |
| "mean": 0.6869575773797418, |
| "std": 0.19855509550894992, |
| "min": -0.056370239704847336, |
| "p05": 0.2907854557037354, |
| "p25": 0.5796415209770203, |
| "median": 0.7289903163909912, |
| "p75": 0.8238811492919922, |
| "p95": 0.947128140926361, |
| "max": 1.1070964336395264 |
| }, |
| "best_gold_score": { |
| "count": 1491, |
| "mean": 0.7264814051798851, |
| "std": 0.16145032231177006, |
| "min": 0.03736429661512375, |
| "p05": 0.43596406280994415, |
| "p25": 0.6369831562042236, |
| "median": 0.752158522605896, |
| "p75": 0.8358416557312012, |
| "p95": 0.9545033872127533, |
| "max": 1.1070964336395264 |
| }, |
| "rag_max_candidate_score": { |
| "count": 1491, |
| "mean": 0.7387905766366712, |
| "std": 0.14133217273717888, |
| "min": 0.1924441009759903, |
| "p05": 0.4963827431201935, |
| "p25": 0.6480166018009186, |
| "median": 0.7563066482543945, |
| "p75": 0.8373694121837616, |
| "p95": 0.9545033872127533, |
| "max": 1.1070964336395264 |
| }, |
| "control_max_candidate_score": null, |
| "max_non_gold_score": { |
| "count": 1491, |
| "mean": 0.5692660571403907, |
| "std": 0.1301589419087464, |
| "min": 0.10697713494300842, |
| "p05": 0.3561844676733017, |
| "p25": 0.4861396551132202, |
| "median": 0.5683011412620544, |
| "p75": 0.6503013372421265, |
| "p95": 0.7916618585586548, |
| "max": 0.975655198097229 |
| }, |
| "query_temporal_norm": { |
| "count": 1491, |
| "mean": 0.22666298581061148, |
| "std": 0.10586569224947587, |
| "min": 0.08168468624353409, |
| "p05": 0.0979655385017395, |
| "p25": 0.14285876601934433, |
| "median": 0.21143613755702972, |
| "p75": 0.2786862254142761, |
| "p95": 0.45315635204315186, |
| "max": 0.572542130947113 |
| }, |
| "best_gold_margin": { |
| "count": 1491, |
| "mean": 0.15721534803949452, |
| "std": 0.15264452667151834, |
| "min": -0.48386645317077637, |
| "p05": -0.09162609279155731, |
| "p25": 0.059481263160705566, |
| "median": 0.15777647495269775, |
| "p75": 0.25978294014930725, |
| "p95": 0.39778003096580505, |
| "max": 0.7206796705722809 |
| }, |
| "weakest_gold_margin": { |
| "count": 1491, |
| "mean": 0.13992435263646585, |
| "std": 0.18043414807290126, |
| "min": -0.6774372458457947, |
| "p05": -0.19127929210662842, |
| "p25": 0.04726390540599823, |
| "median": 0.15512782335281372, |
| "p75": 0.25943848490715027, |
| "p95": 0.39778003096580505, |
| "max": 0.7206796705722809 |
| }, |
| "top_score_best_balanced_threshold": null, |
| "gold_score_best_balanced_threshold": null |
| }, |
| "historical_state": { |
| "rag_query_count": 500, |
| "control_query_count": 0, |
| "gold_score": { |
| "count": 502, |
| "mean": 0.7836064203564389, |
| "std": 0.10709909743229676, |
| "min": 0.14025598764419556, |
| "p05": 0.5942545413970948, |
| "p25": 0.7248825877904892, |
| "median": 0.8013403415679932, |
| "p75": 0.8561894297599792, |
| "p95": 0.9279397994279861, |
| "max": 1.012067198753357 |
| }, |
| "best_gold_score": { |
| "count": 500, |
| "mean": 0.7853946338891983, |
| "std": 0.10275866286384987, |
| "min": 0.37629517912864685, |
| "p05": 0.5945305287837982, |
| "p25": 0.7256994992494583, |
| "median": 0.8023103773593903, |
| "p75": 0.8565277755260468, |
| "p95": 0.9281367629766464, |
| "max": 1.012067198753357 |
| }, |
| "rag_max_candidate_score": { |
| "count": 500, |
| "mean": 0.7897239015698433, |
| "std": 0.10143125862120343, |
| "min": 0.37629517912864685, |
| "p05": 0.6064692318439484, |
| "p25": 0.7330148816108704, |
| "median": 0.8051750957965851, |
| "p75": 0.8608255237340927, |
| "p95": 0.9311295807361601, |
| "max": 1.012067198753357 |
| }, |
| "control_max_candidate_score": null, |
| "max_non_gold_score": { |
| "count": 500, |
| "mean": 0.6717005554437637, |
| "std": 0.15029697753120055, |
| "min": 0.16829177737236023, |
| "p05": 0.4131940007209778, |
| "p25": 0.5654338002204895, |
| "median": 0.6779390871524811, |
| "p75": 0.7914849817752838, |
| "p95": 0.8960957378149033, |
| "max": 1.0081841945648193 |
| }, |
| "query_temporal_norm": { |
| "count": 500, |
| "mean": 0.48263769267499446, |
| "std": 0.10428369082664105, |
| "min": 0.090034119784832, |
| "p05": 0.24665968716144565, |
| "p25": 0.4482061117887497, |
| "median": 0.5256085395812988, |
| "p75": 0.5539183616638184, |
| "p95": 0.5720209151506424, |
| "max": 0.5792937874794006 |
| }, |
| "best_gold_margin": { |
| "count": 500, |
| "mean": 0.11369407844543457, |
| "std": 0.12331652846556644, |
| "min": -0.2083926796913147, |
| "p05": -0.029318222403526294, |
| "p25": 0.003192394971847534, |
| "median": 0.10047835111618042, |
| "p75": 0.1869528740644455, |
| "p95": 0.33141966611146895, |
| "max": 0.4669537842273712 |
| }, |
| "weakest_gold_margin": { |
| "count": 500, |
| "mean": 0.1126235191822052, |
| "std": 0.12586990224972355, |
| "min": -0.4509775638580322, |
| "p05": -0.03355695605278015, |
| "p25": 0.003106921911239624, |
| "median": 0.10047835111618042, |
| "p75": 0.1869528740644455, |
| "p95": 0.33141966611146895, |
| "max": 0.4669537842273712 |
| }, |
| "top_score_best_balanced_threshold": null, |
| "gold_score_best_balanced_threshold": null |
| }, |
| "latest_state": { |
| "rag_query_count": 457, |
| "control_query_count": 0, |
| "gold_score": { |
| "count": 460, |
| "mean": 0.9099455071532208, |
| "std": 0.11056362986875025, |
| "min": 0.2515583336353302, |
| "p05": 0.7147018998861313, |
| "p25": 0.8624780625104904, |
| "median": 0.927989274263382, |
| "p75": 0.9840456992387772, |
| "p95": 1.0430482387542723, |
| "max": 1.1258333921432495 |
| }, |
| "best_gold_score": { |
| "count": 457, |
| "mean": 0.9128042522092356, |
| "std": 0.10462899466412111, |
| "min": 0.2515583336353302, |
| "p05": 0.7270167827606201, |
| "p25": 0.8639979958534241, |
| "median": 0.9282728433609009, |
| "p75": 0.9842485189437866, |
| "p95": 1.0431268453598022, |
| "max": 1.1258333921432495 |
| }, |
| "rag_max_candidate_score": { |
| "count": 457, |
| "mean": 0.9186350223272061, |
| "std": 0.1033423010515788, |
| "min": 0.3498287498950958, |
| "p05": 0.7335727095603943, |
| "p25": 0.8663325309753418, |
| "median": 0.9317175149917603, |
| "p75": 0.9901189804077148, |
| "p95": 1.0482017040252685, |
| "max": 1.1260788440704346 |
| }, |
| "control_max_candidate_score": null, |
| "max_non_gold_score": { |
| "count": 457, |
| "mean": 0.9157646012123617, |
| "std": 0.1071663211990817, |
| "min": 0.341567724943161, |
| "p05": 0.7311487436294556, |
| "p25": 0.863583505153656, |
| "median": 0.9317175149917603, |
| "p75": 0.9866815805435181, |
| "p95": 1.0477006912231446, |
| "max": 1.1260788440704346 |
| }, |
| "query_temporal_norm": { |
| "count": 457, |
| "mean": 0.5188334593039895, |
| "std": 0.06393273507489536, |
| "min": 0.12837310135364532, |
| "p05": 0.36590389013290403, |
| "p25": 0.5123153924942017, |
| "median": 0.5376707315444946, |
| "p75": 0.5540842413902283, |
| "p95": 0.5678486704826355, |
| "max": 0.5767868161201477 |
| }, |
| "best_gold_margin": { |
| "count": 457, |
| "mean": -0.0029603490031261154, |
| "std": 0.020342693875631014, |
| "min": -0.09827041625976562, |
| "p05": -0.036279916763305664, |
| "p25": -0.00460284948348999, |
| "median": -0.0004582405090332031, |
| "p75": 0.003332674503326416, |
| "p95": 0.008798444271087644, |
| "max": 0.23436546325683594 |
| }, |
| "weakest_gold_margin": { |
| "count": 457, |
| "mean": -0.0037261917371979427, |
| "std": 0.01602755304203525, |
| "min": -0.09827041625976562, |
| "p05": -0.036279916763305664, |
| "p25": -0.00463712215423584, |
| "median": -0.0004839897155761719, |
| "p75": 0.0033155083656311035, |
| "p95": 0.00844621658325195, |
| "max": 0.11645901203155518 |
| }, |
| "top_score_best_balanced_threshold": null, |
| "gold_score_best_balanced_threshold": null |
| }, |
| "multi_hop_composition": { |
| "rag_query_count": 348, |
| "control_query_count": 0, |
| "gold_score": { |
| "count": 773, |
| "mean": 0.6628726145895957, |
| "std": 0.16474756870609927, |
| "min": -0.05371195077896118, |
| "p05": 0.31368120312690734, |
| "p25": 0.6175206899642944, |
| "median": 0.712782084941864, |
| "p75": 0.7731857299804688, |
| "p95": 0.8209763169288635, |
| "max": 0.8812718987464905 |
| }, |
| "best_gold_score": { |
| "count": 348, |
| "mean": 0.7313412460444988, |
| "std": 0.09207875076116444, |
| "min": 0.36352241039276123, |
| "p05": 0.5470803171396256, |
| "p25": 0.6859159916639328, |
| "median": 0.754956841468811, |
| "p75": 0.7975459843873978, |
| "p95": 0.8379994869232178, |
| "max": 0.8812718987464905 |
| }, |
| "rag_max_candidate_score": { |
| "count": 348, |
| "mean": 0.7419123562245533, |
| "std": 0.08379927154498036, |
| "min": 0.43880122900009155, |
| "p05": 0.5616730064153671, |
| "p25": 0.6991288214921951, |
| "median": 0.7600564062595367, |
| "p75": 0.8044402897357941, |
| "p95": 0.8395571231842041, |
| "max": 0.8812718987464905 |
| }, |
| "control_max_candidate_score": null, |
| "max_non_gold_score": { |
| "count": 348, |
| "mean": 0.663961509306883, |
| "std": 0.14432546830207857, |
| "min": 0.19614651799201965, |
| "p05": 0.38625628352165225, |
| "p25": 0.5634785890579224, |
| "median": 0.7131754457950592, |
| "p75": 0.7773982137441635, |
| "p95": 0.8263130962848663, |
| "max": 0.8776211738586426 |
| }, |
| "query_temporal_norm": { |
| "count": 348, |
| "mean": 0.19367624020696372, |
| "std": 0.03749222859666623, |
| "min": 0.12189259380102158, |
| "p05": 0.13993898779153824, |
| "p25": 0.16496126726269722, |
| "median": 0.190658338367939, |
| "p75": 0.21625519171357155, |
| "p95": 0.2616093188524245, |
| "max": 0.33960673213005066 |
| }, |
| "best_gold_margin": { |
| "count": 348, |
| "mean": 0.06737973673761576, |
| "std": 0.1273355332284643, |
| "min": -0.16347423195838928, |
| "p05": -0.05428696870803833, |
| "p25": -0.010949954390525818, |
| "median": 0.019723743200302124, |
| "p75": 0.0933598130941391, |
| "p95": 0.3714251846075057, |
| "max": 0.5118336081504822 |
| }, |
| "weakest_gold_margin": { |
| "count": 348, |
| "mean": -0.013968390235998508, |
| "std": 0.15625439094496515, |
| "min": -0.5777953863143921, |
| "p05": -0.2627247408032417, |
| "p25": -0.0745110958814621, |
| "median": -0.02843642234802246, |
| "p75": 0.007143333554267883, |
| "p95": 0.30581344068050376, |
| "max": 0.5001735091209412 |
| }, |
| "top_score_best_balanced_threshold": null, |
| "gold_score_best_balanced_threshold": null |
| }, |
| "recent_context_control": { |
| "rag_query_count": 0, |
| "control_query_count": 1, |
| "gold_score": null, |
| "best_gold_score": null, |
| "rag_max_candidate_score": null, |
| "control_max_candidate_score": { |
| "count": 1, |
| "mean": 0.4969808757305145, |
| "std": 0.0, |
| "min": 0.4969808757305145, |
| "p05": 0.4969808757305145, |
| "p25": 0.4969808757305145, |
| "median": 0.4969808757305145, |
| "p75": 0.4969808757305145, |
| "p95": 0.4969808757305145, |
| "max": 0.4969808757305145 |
| }, |
| "max_non_gold_score": null, |
| "query_temporal_norm": { |
| "count": 1, |
| "mean": 0.20389491319656372, |
| "std": 0.0, |
| "min": 0.20389491319656372, |
| "p05": 0.20389491319656372, |
| "p25": 0.20389491319656372, |
| "median": 0.20389491319656372, |
| "p75": 0.20389491319656372, |
| "p95": 0.20389491319656372, |
| "max": 0.20389491319656372 |
| }, |
| "best_gold_margin": null, |
| "weakest_gold_margin": null, |
| "top_score_best_balanced_threshold": null, |
| "gold_score_best_balanced_threshold": null |
| }, |
| "temporal_localization": { |
| "rag_query_count": 719, |
| "control_query_count": 0, |
| "gold_score": { |
| "count": 795, |
| "mean": 0.697561881991115, |
| "std": 0.17009461728665182, |
| "min": -0.08927247673273087, |
| "p05": 0.3611560195684433, |
| "p25": 0.6234583854675293, |
| "median": 0.732460618019104, |
| "p75": 0.816007524728775, |
| "p95": 0.9051089465618133, |
| "max": 1.0474931001663208 |
| }, |
| "best_gold_score": { |
| "count": 719, |
| "mean": 0.7292242187162102, |
| "std": 0.13312502433949933, |
| "min": 0.1190776377916336, |
| "p05": 0.4726928949356079, |
| "p25": 0.6564635336399078, |
| "median": 0.7507467269897461, |
| "p75": 0.8241211473941803, |
| "p95": 0.9126609623432161, |
| "max": 1.0474931001663208 |
| }, |
| "rag_max_candidate_score": { |
| "count": 719, |
| "mean": 0.7364202864229762, |
| "std": 0.12132767669906487, |
| "min": 0.24290406703948975, |
| "p05": 0.5120499312877655, |
| "p25": 0.6625339388847351, |
| "median": 0.7538652420043945, |
| "p75": 0.8249395787715912, |
| "p95": 0.9126609623432161, |
| "max": 1.0474931001663208 |
| }, |
| "control_max_candidate_score": null, |
| "max_non_gold_score": { |
| "count": 719, |
| "mean": 0.5847217668163925, |
| "std": 0.1286120778132217, |
| "min": 0.2042156159877777, |
| "p05": 0.37733144760131837, |
| "p25": 0.48644280433654785, |
| "median": 0.5866289138793945, |
| "p75": 0.6778780221939087, |
| "p95": 0.7978232324123383, |
| "max": 0.9642623662948608 |
| }, |
| "query_temporal_norm": { |
| "count": 719, |
| "mean": 0.20483502627496758, |
| "std": 0.06486274901452087, |
| "min": 0.09465165436267853, |
| "p05": 0.11441298574209213, |
| "p25": 0.15930375456809998, |
| "median": 0.20111021399497986, |
| "p75": 0.23105795681476593, |
| "p95": 0.3251008868217468, |
| "max": 0.534060001373291 |
| }, |
| "best_gold_margin": { |
| "count": 719, |
| "mean": 0.14450245189981767, |
| "std": 0.12346059833105659, |
| "min": -0.3746306002140045, |
| "p05": -0.04459953308105469, |
| "p25": 0.07054683566093445, |
| "median": 0.12600627541542053, |
| "p75": 0.22508594393730164, |
| "p95": 0.35579485893249513, |
| "max": 0.5074322521686554 |
| }, |
| "weakest_gold_margin": { |
| "count": 719, |
| "mean": 0.12679185110541974, |
| "std": 0.14944456096152975, |
| "min": -0.5301856100559235, |
| "p05": -0.13406084179878236, |
| "p25": 0.06027638912200928, |
| "median": 0.12011480331420898, |
| "p75": 0.22024193406105042, |
| "p95": 0.3543264567852021, |
| "max": 0.5074322521686554 |
| }, |
| "top_score_best_balanced_threshold": null, |
| "gold_score_best_balanced_threshold": null |
| } |
| } |
| } |
| } |
| } |
| ] |
| } |
|
|
