Upload 11_12_13_lr_17's state dict

.gitattributes

@@ -39,3 +39,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 1_lr_add_bm25_10/logs/1_lr_add_bm25_10_log_plot.jpg filter=lfs diff=lfs merge=lfs -text
 10_lr_attn_chunk_pool_instead_mean_12/logs/10_lr_attn_chunk_pool_instead_mean_12_log_plot.jpg filter=lfs diff=lfs merge=lfs -text
 11_12_lr_14/logs/11_12_lr_14_log_plot.jpg filter=lfs diff=lfs merge=lfs -text
+11_12_13_lr_17/logs/11_12_13_lr_17_log_plot.jpg filter=lfs diff=lfs merge=lfs -text

11_12_13_lr_17/11_12_13_lr_17.py

@@ -0,0 +1,1593 @@
+# %% [code]
+get_ipython().system('pip install evaluate seqeval underthesea positional-encodings[pytorch]')
+
+# %% [code]
+import warnings
+warnings.filterwarnings('ignore')
+
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from torch.utils.data import Dataset, TensorDataset, DataLoader
+import torch.nn.functional as F
+import albumentations as albu
+from transformers import AutoTokenizer, AutoModel
+import torch.distributed as dist
+from torch.nn.parallel import DistributedDataParallel as DDP
+from positional_encodings.torch_encodings import PositionalEncoding1D
+
+from sklearn.metrics import f1_score
+from sklearn.preprocessing import MinMaxScaler, StandardScaler
+from scipy.spatial.transform import Rotation as R
+from sklearn.model_selection import KFold, StratifiedGroupKFold, GroupKFold, StratifiedKFold
+from sklearn.metrics import precision_recall_fscore_support
+from timm.utils import ModelEmaV3
+import timm
+
+import os
+import gc
+import json
+from pathlib import Path
+import pickle
+from tqdm.auto import tqdm
+import copy
+import numpy as np
+import pandas as pd
+import polars as pl
+from PIL import Image
+import time
+from tqdm import tqdm
+from matplotlib import pyplot as plt
+import seaborn as sns
+from multiprocessing import Manager as MemoryManager
+from functools import lru_cache
+import shutil
+import glob
+import cv2
+import random
+import re
+import joblib
+import math
+from huggingface_hub import HfApi, snapshot_download
+import evaluate
+from underthesea import word_tokenize as vi_tokenize_tool
+import spacy
+en_tokenize_tool = spacy.load("en_core_web_sm")
+from collections import defaultdict, Counter
+
+# %% [code]
+file_path = "/kaggle/input/notebooks/sambui22022517/kltn-lr-bm25/bm25_scores.npy"
+
+bm25_scores = np.load(file_path, allow_pickle=True)
+print(bm25_scores.shape)
+
+# %% [code]
+# Global config
+SEEDS = [26092004]
+topk = 1
+nfolds = 5
+only_fold_idx = None
+test_only = 0
+debug_only = 0
+
+# Config thư mục
+dataset = 'kltn/raw' # vhe, bkee, casie, kltn/only_issues, kltn/only_actions, kltn/raw
+root_dir = f'/kaggle/input/notebooks/sambui22022517/kltn-data/{dataset}' ## Thư mục chứa file train, val, test
+train_dir = f'{root_dir}'
+# val_dir = f'{root_dir}/val'
+test_dir = f'{root_dir}'
+
+# Config checkpoints
+
+# Config training
+epochs = 18 if not debug_only else 2
+batch_size = 32
+device = "cuda" if torch.cuda.is_available() else "cpu"
+# # Thêm biến toàn cục nào đó vào đây
+repo_name = 'SS3M/kltn-lr-experiments'
+state_dict_save_name = "11_12_13_lr_17"
+checkpoints_dir = state_dict_save_name
+pretrained_dir = "/kaggle/working"
+os.makedirs(f'{checkpoints_dir}', exist_ok=True)
+
+backbone_model_name = "bert-base-uncased" if dataset == "casie" else "vinai/phobert-base"
+word_tokenize = lambda text: [token.text for token in en_tokenize_tool(text)] if dataset == "casie" else vi_tokenize_tool(text)
+max_len_dict = {
+    'kltn/raw': 256, 
+    'kltn/only_issues': 52, 
+    'kltn/only_actions': 69, 
+    'vhe': 51, 
+    'bkee': 62, 
+    'casie': 40, 
+}
+zero_events_rate_dict = {
+    'kltn/raw': 1000, 
+    'kltn/only_issues': 0, 
+    'kltn/only_actions': 0.2, 
+    'vhe': 1000, # mean keep all zero-events samples
+    'bkee': 1000, 
+    'casie': 1000, 
+}
+
+max_len = max_len_dict[dataset]
+max_n_parts = 2
+max_span_len = 14
+n_negs = 5 * 20
+zero_events_rate = zero_events_rate_dict[dataset]
+
+# Trainer
+trainer_params = {
+    "training_time": "00:11:30:00",
+    "eval_mode": "max",
+    "topk": topk,
+    "save_name": state_dict_save_name,
+    "save_best": True,
+    "save_last": True,
+    "device": device, 
+    "logging": True, 
+    "logging_file": True, 
+    "checkpoints_dir": checkpoints_dir, 
+    "early_stopping": 30, 
+    "eval_from_ratio": 0.4, 
+    "eval_every": 1, 
+    "schedule_in_step": False, 
+    "use_ema": True, 
+    "ema_from_ratio": 0.3, 
+    "ema_decay": 0.9995, 
+    "max_grad_norm": 200.0, 
+    "return_best": True, 
+    "return_last": True, 
+}
+
+# Memory
+train_memory_params = {
+    'max_len': max_len, 
+    'max_n_parts': max_n_parts, 
+    'n_negs': n_negs,
+}
+val_memory_params = {
+    'max_len': max_len, 
+    'max_n_parts': max_n_parts, 
+    'n_negs': n_negs,
+}
+corpus_memory_params = {
+    'max_len': max_len, 
+    'max_n_parts': max_n_parts, 
+}
+
+# Data Loader
+def seed_worker(worker_id):
+    worker_seed = torch.initial_seed() % 2**32
+    np.random.seed(worker_seed)
+    random.seed(worker_seed)
+    
+train_loader_params = {
+    'batch_size': batch_size, 
+    'shuffle': True,
+    'pin_memory':True, 
+    'num_workers': 2, 
+    'drop_last': False, 
+    'worker_init_fn': seed_worker, 
+    'persistent_workers': False, 
+}
+val_loader_params = {
+    'batch_size': batch_size, 
+    'shuffle': False,
+    'pin_memory':True, 
+    'num_workers': 1, 
+    'drop_last': False, 
+    'worker_init_fn': seed_worker, 
+    'persistent_workers': False, 
+}
+
+# Model
+model_params = {
+    'backbone_name': backbone_model_name, 
+    'projection_dim': 256, 
+    'normalize': True, 
+}
+
+# Loss Func
+loss_func_params = {
+    'lambda_contrastive': 1.0,
+    'lambda_triplet': 0.5,
+}
+eval_func_params = {}
+
+# Optim
+optim_params = {
+    'name': 'AdamW', 
+    'lr': 1e-4,
+    'weight_decay': 1e-4,
+}
+scheduler_params = {
+    'name': 'CosineAnnealingLR',
+    'T_max': 20,  # Số epoch để hoàn thành một chu kỳ giảm LR
+    'eta_min': 1e-6  # Learning rate nhỏ nhất trong chu kỳ
+}
+
+# %% [code]
+def set_seed(seed=42):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)  # if using multi-GPU
+    torch.use_deterministic_algorithms(False)
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+    os.environ['PYTHONHASHSEED'] = str(seed)
+
+# %% [code]
+class CustomLoss(nn.Module):
+    def __init__(
+        self,
+        temperature=0.05,
+        margin=0.2,
+        lambda_contrastive=1.0,
+        lambda_triplet=0.5,
+    ):
+        super().__init__()
+
+        self.temperature = temperature
+        self.margin = margin
+
+        self.lambda_contrastive = lambda_contrastive
+        self.lambda_triplet = lambda_triplet
+
+    def forward(
+        self, 
+        encoded_text, 
+        encoded_pos, 
+        encoded_neg, 
+        pos_mask
+    ):
+        loss_contrastive = self.multi_pos_contrastive_loss(encoded_text, encoded_pos, encoded_neg, pos_mask)
+        loss_triplet = self.hardest_triplet_loss(encoded_text, encoded_pos, encoded_neg, pos_mask)
+
+        total_loss = (
+            self.lambda_contrastive * loss_contrastive + 
+            self.lambda_triplet * loss_triplet
+        )
+
+        return {
+            "total": total_loss,
+            "contrastive_loss": loss_contrastive,
+            "triplet_loss": loss_triplet,
+        }
+
+    def multi_pos_contrastive_loss(self, q, pos, neg, pos_mask):
+        B, P, D = pos.shape
+        N = neg.shape[1]
+
+        # ===== concat docs =====
+        docs = torch.cat([pos, neg], dim=1)  # [B, P+N, D]
+
+        # ===== similarity =====
+        logits = torch.matmul(q.unsqueeze(1), docs.transpose(1, 2)).squeeze(1)
+        logits = logits / self.temperature  # [B, P+N]
+
+        # ===== labels =====
+        labels = torch.zeros_like(logits)
+        labels[:, :P] = pos_mask  # chỉ pos hợp lệ
+
+        # ===== log-softmax =====
+        log_prob = logits - torch.logsumexp(logits, dim=1, keepdim=True)
+
+        # ===== normalize theo số pos thật =====
+        pos_count = pos_mask.sum(dim=1).clamp(min=1)
+
+        loss = -(labels * log_prob).sum(dim=1) / pos_count
+
+        return loss.mean()
+
+    def hardest_triplet_loss(self, q, pos, neg, pos_mask):
+        # ===== similarity =====
+        pos_sim = torch.matmul(q.unsqueeze(1), pos.transpose(1, 2)).squeeze(1)  # [B, P]
+        neg_sim = torch.matmul(q.unsqueeze(1), neg.transpose(1, 2)).squeeze(1)  # [B, N]
+
+        # ===== mask pos =====
+        pos_sim_masked = pos_sim.clone()
+        pos_sim_masked[pos_mask == 0] = float('inf')  # loại pad
+
+        # ===== hardest =====
+        hardest_pos = pos_sim_masked.min(dim=1).values
+        hardest_neg = neg_sim.max(dim=1).values
+
+        # ===== loss =====
+        loss = F.relu(self.margin + hardest_neg - hardest_pos)
+
+        return loss.mean()
+
+# %% [code]
+class CustomEvalFn(nn.Module):
+    def __init__(self):
+        super().__init__()
+
+    def forward(self, pred_topk, real_topk):
+        """
+        pred_topk: List[List[int]]  shape [B, K]
+        real_topk: List[List[int]]  shape [B, Ki]
+        """
+
+        B = len(pred_topk)
+
+        total_recall = 0.0
+        total_map = 0.0
+        total_mrp = 0.0
+
+        for i in range(B):
+            preds = pred_topk[i]
+            gts = set(real_topk[i])
+
+            # ===== Recall@K =====
+            hit = any(p in gts for p in preds)
+            total_recall += 1.0 if hit else 0.0
+
+            # ===== AP =====
+            num_hits = 0
+            ap = 0.0
+
+            for rank, p in enumerate(preds, start=1):
+                if p in gts:
+                    num_hits += 1
+                    precision_at_rank = num_hits / rank
+                    ap += precision_at_rank
+
+            if len(gts) > 0:
+                ap /= len(gts)
+
+            total_map += ap
+
+            # ===== R-Precision =====
+            r = len(gts)
+
+            if r > 0:
+                top_r = preds[:r]
+
+                tp_r = sum(p in gts for p in top_r)
+
+                rp = tp_r / r
+            else:
+                rp = 0.0
+
+            total_mrp += rp
+
+        recall = total_recall / B
+        mAP = total_map / B
+        mRP = total_mrp / B
+
+        return {
+            "recall": recall,
+            "mAP": mAP,
+            "mRP": mRP,
+        }
+
+# %% [code]
+class EncodeModel(nn.Module):
+    def __init__(
+        self,
+        backbone_name,
+        projection_dim,
+        normalize=True,
+        dropout=0.1
+    ):
+        super().__init__()
+
+        self.encoder = AutoModel.from_pretrained(backbone_name)
+
+        hidden_size = self.encoder.config.hidden_size
+
+        # mạnh hơn single linear
+        self.proj = nn.Sequential(
+            nn.Linear(hidden_size, hidden_size),
+            nn.GELU(),
+            nn.Dropout(dropout),
+            nn.Linear(hidden_size, projection_dim)
+        )
+
+        # residual projection
+        self.residual_proj = (
+            nn.Identity()
+            if hidden_size == projection_dim
+            else nn.Linear(hidden_size, projection_dim)
+        )
+        self.chunk_attn = nn.Linear(hidden_size, 1)
+
+        self.normalize = normalize
+
+    def mean_pooling(self, hidden, attention_mask):
+        mask = attention_mask.unsqueeze(-1).float()  # [N, L, 1]
+        pooled = (hidden * mask).sum(dim=1) / mask.sum(dim=1).clamp(min=1e-6)
+
+        return pooled
+
+    def chunk_attention_pooling(self, chunk_repr, dim):
+        scores = self.chunk_attn(chunk_repr)
+
+        weights = torch.softmax(scores, dim=dim)
+
+        pooled = (chunk_repr * weights).sum(dim=dim)
+
+        return pooled
+
+    def forward(self, input_ids, attention_mask, is_query=True):
+
+        if is_query:
+            # [B, n_parts, L]
+            B, n_parts, L = input_ids.shape
+
+            input_ids = input_ids.view(-1, L)
+            attention_mask = attention_mask.view(-1, L)
+
+            outputs = self.encoder(
+                input_ids=input_ids,
+                attention_mask=attention_mask
+            )
+
+            hidden = outputs.last_hidden_state # [B*n_parts, L, H]
+
+            chunk_repr = self.mean_pooling(hidden, attention_mask) # [B*n_parts, H]
+            chunk_repr = chunk_repr.view(B, n_parts, -1) # [B, n_parts, H]
+            final_repr = self.chunk_attention_pooling(chunk_repr, dim=1) # [B, H]
+            # [B, H]
+
+        else:
+            # [B, K, n_parts, L]
+            B, K, n_parts, L = input_ids.shape
+
+            input_ids = input_ids.view(-1, L)
+            attention_mask = attention_mask.view(-1, L)
+
+            outputs = self.encoder(
+                input_ids=input_ids,
+                attention_mask=attention_mask
+            )
+
+            hidden = outputs.last_hidden_state
+
+            chunk_repr = self.mean_pooling(hidden, attention_mask) # [B*K*n_parts, H]
+            chunk_repr = chunk_repr.view(B, K, n_parts, -1) # [B, K, n_parts, H]
+            final_repr = self.chunk_attention_pooling(chunk_repr, dim=2) # [B, K, H]
+
+        # residual MLP projection
+        emb = self.proj(final_repr) + self.residual_proj(final_repr)
+
+        if self.normalize:
+            emb = F.normalize(emb, dim=-1)
+
+        return emb
+
+def test_model():
+    model = nn.DataParallel(EncodeModel('vinai/phobert-base', 256, True)).to(device)
+    model.eval()
+
+    bz = 32
+    vocab_size = 1000
+    qi = torch.randint(0, vocab_size, (bz, 1, 256)).to(device)
+    qa = torch.ones(bz, 1, 256).to(device)
+    di = torch.randint(0, vocab_size, (bz, 5, 2, 256)).to(device)
+    da = torch.ones(bz, 5, 2, 256).to(device)
+
+    st = time.time()
+    with torch.no_grad():
+        encoded_text = model(qi, qa, is_query=True)
+        encoded_pos = model(di, da, is_query=False)
+        encoded_neg = model(di, da, is_query=False)
+        print(encoded_text.shape, encoded_pos.shape, encoded_neg.shape)
+    print(time.time() - st)
+
+    del model, qi, qa, di, da, encoded_text, encoded_pos, encoded_neg
+    torch.cuda.empty_cache()
+    gc.collect()
+test_model()
+
+# %% [code]
+def configure_optimizers(network, optim_params, scheduler_params):
+    try:
+        optim_params = copy.copy(optim_params)
+        scheduler_params = copy.copy(scheduler_params)
+        
+        optim_name = optim_params.pop('name')
+        scheduler_name = scheduler_params.pop('name')
+    
+        optimizer_cls = globals().get(optim_name) or getattr(optim, optim_name, None)
+        scheduler_cls = globals().get(scheduler_name) or getattr(optim.lr_scheduler, scheduler_name, None)
+    
+        if optimizer_cls is None:
+            raise ValueError(f"Optimizer '{optim_name}' is not available!")
+    
+        optimizer = optimizer_cls(network.parameters(), **optim_params)
+        
+        scheduler = None
+        if scheduler_params and scheduler_cls:  # Chỉ tạo scheduler nếu có tham số
+            scheduler = scheduler_cls(optimizer, **scheduler_params)
+    
+        return optimizer, scheduler
+
+    except KeyError as e:
+        raise ValueError(f"Missing {e} in config!!")
+            
+def freeze(self, model):
+    model.eval()
+    for param in model.parameters():
+        param.requires_grad = False
+
+def unfreeze(self, model):
+    model.train()
+    for param in model.parameters():
+        param.requires_grad = True
+
+def reduce_batch_size(loader, ratio=0.5):
+    new_bs = max(1, int(loader.batch_size * ratio))
+
+    shuffle = isinstance(loader.sampler, RandomSampler)
+
+    new_loader = DataLoader(
+        dataset=loader.dataset,
+        batch_size=new_bs,
+        shuffle=shuffle,
+        sampler=None if shuffle else loader.sampler,
+        num_workers=loader.num_workers,
+        collate_fn=loader.collate_fn,
+        pin_memory=loader.pin_memory,
+        drop_last=loader.drop_last,
+        timeout=loader.timeout,
+        worker_init_fn=loader.worker_init_fn,
+        multiprocessing_context=loader.multiprocessing_context,
+        generator=loader.generator,
+        prefetch_factor=loader.prefetch_factor if loader.num_workers > 0 else None,
+        persistent_workers=loader.persistent_workers,
+        pin_memory_device=loader.pin_memory_device
+    )
+
+    return new_loader
+
+def list_to_tuple(x):
+    if isinstance(x, (list, tuple)):
+        return tuple(list_to_tuple(i) for i in x)
+    return x
+
+def fmt(x):
+    if isinstance(x, float):
+        return round(x, 5)
+    if isinstance(x, dict):
+        return {k: fmt(v) for k, v in x.items()}
+    if isinstance(x, list):
+        return [fmt(v) for v in x]
+    return x
+    
+class ModelEmaV3Proxy(ModelEmaV3):
+    def __getattr__(self, name):
+        try:
+            return super().__getattr__(name)
+        except AttributeError:
+            return getattr(self.module, name)
+            
+class DataParallelProxy(nn.DataParallel):
+    def __getattr__(self, name):
+        try:
+            return super().__getattr__(name)
+        except AttributeError:
+            attr = getattr(self.module, name)
+
+            if callable(attr):
+                def wrapper(*args, **kwargs):
+                    return self._parallel_apply_method(name, *args, **kwargs)
+                return wrapper
+
+            return attr
+
+    def _parallel_apply_method(self, method_name, *inputs, **kwargs):
+        if not self.device_ids:
+            return getattr(self.module, method_name)(*inputs, **kwargs)
+
+        inputs_scattered, kwargs_scattered = self.scatter(inputs, kwargs, self.device_ids)
+
+        replicas = self.replicate(self.module, self.device_ids)
+
+        outputs = self.parallel_apply(
+            [getattr(replica, method_name) for replica in replicas],
+            inputs_scattered,
+            kwargs_scattered
+        )
+
+        return self.gather(outputs, self.output_device)
+    
+class Trainer:
+    def __init__(
+        self, training_time="00:11:30:00", eval_mode="max", topk=1, save_name="network", save_best=True, save_last=False, max_grad_norm=200.0, 
+        logging=0, logging_file=False, checkpoints_dir="", early_stopping=False, eval_from_ratio=-1, eval_every=1, device='cpu', 
+        schedule_in_step=True, use_ema=True, ema_from_ratio=-1, ema_decay=0.999, return_best=True, return_last=True
+    ):
+        self.ema_net = None
+        
+        self.training_time = self._time_str_to_seconds(training_time)
+        self.mode = eval_mode
+        self.topk = topk
+        self.device = device
+        self.logging = logging if logging < epochs else 1
+        self.logging_file = logging_file
+        self.checkpoints_dir = checkpoints_dir
+        self.early_stopping = early_stopping
+        self.eval_from_ratio = eval_from_ratio
+        self.eval_every = eval_every
+        self.save_name = save_name
+        self.save_best = save_best
+        self.save_last = save_last
+        self.return_best = return_best
+        self.return_last = return_last
+        self.max_grad_norm = max_grad_norm
+        self.schedule_in_step = schedule_in_step
+        self.use_ema = use_ema
+        self.ema_from_ratio = ema_from_ratio
+        self.ema_decay = ema_decay
+        
+        self.best_stage = [[float('-inf') if self.mode == 'max' else float('inf'), None, None]]
+        self.grad_scaler = torch.amp.GradScaler(self.device, init_scale=1024.0)
+
+    def fit(self, network, optimizer, scheduler, loss_fn, epochs, train_loader, val_loader=None, corpus_loader=None, eval_fn=None, start_epoch=1, start_training_time=None, refresh_every=3):
+        if eval_fn is None:
+            if self.mode == "max":
+                eval_fn = lambda *x: -loss_fn(*x)
+            else:
+                eval_fn = lambda *x: loss_fn(*x)
+
+        if torch.cuda.device_count() > 1:
+            network = DataParallelProxy(network)
+        network = network.to(self.device)
+            
+        if not start_training_time:
+            start_training_time = time.time()
+            
+        start_ema = int(epochs * self.ema_from_ratio)
+        start_eval = int(epochs * self.eval_from_ratio)
+        
+        if val_loader is None:
+            print(f'[Trainer CallBack] 📢 Không có Val Set, không thể đánh giá và Early Stopping!')
+        else:
+            model_to_use_str = 'mô hình EMA' if self.use_ema else 'mô hình gốc'
+            start_model_update_str = f'Bắt đầu cập nhật EMA từ epoch {start_epoch + start_ema}!' if self.use_ema else ''
+            print(f'[Trainer CallBack] 📢 Đánh giá bằng {model_to_use_str} từ epoch {start_epoch + start_eval}!', start_model_update_str)
+
+        training_log = {}
+        for epoch in range(start_epoch, epochs+start_epoch):
+            if self.use_ema and self.ema_net is None and epoch - start_epoch >= start_ema:
+                self.ema_net = ModelEmaV3Proxy(network, self.ema_decay, device=self.device)
+                
+            try:
+                eval_net = self.ema_net if (self.use_ema and self.ema_net is not None) else network
+                if (epoch - start_epoch) % refresh_every == 0:
+                    encoded_docs = self._get_encoded_docs(eval_net, corpus_loader)
+                    print(f"[Trainer CallBack] 📢 Epoch {epoch}/{epochs}: Refresh Encoded Doc (refresh_every={refresh_every})!")
+                    
+                train_loss_epoch, train_loss_epoch_dict = self._train_epoch(network, train_loader, optimizer, scheduler, loss_fn, encoded_docs)
+                logging_dict = {'lr': [group['lr'] for group in optimizer.param_groups], 'train_loss': train_loss_epoch}
+                logging_dict.update(train_loss_epoch_dict)
+                    
+                if val_loader is not None and epoch - start_epoch >= start_eval and (epoch - start_epoch - start_eval) % self.eval_every == 0:                    
+                    val_score, val_score_dict, _ = self._eval_epoch(eval_net, val_loader, eval_fn, encoded_docs)
+                    update = self._update_best_network(eval_net, val_score, epoch)
+                    logging_dict.update({'val_score': val_score, 'best_score': self.best_stage[0][0], 'new_best_model': update})
+                    logging_dict.update(val_score_dict)
+                if not self.schedule_in_step and scheduler:
+                    scheduler.step()
+                    
+            except RuntimeError as e:
+                if "out of memory" in str(e).lower():
+                    print(f"[Trainer CallBack] ⚠️ Epoch {epoch}/{epochs}: CUDA Out of Memory! Clearing GPU cache...")
+                    torch.cuda.empty_cache()
+                    gc.collect()
+                    if torch.cuda.is_available():
+                        torch.cuda.synchronize()
+                    print(f"[Trainer CallBack] ✅ Epoch {epoch}/{epochs}: GPU memory cleared")
+                    
+                    train_loader = reduce_batch_size(train_loader, ratio=0.5)
+                    if val_loader is not None:
+                        val_loader = reduce_batch_size(val_loader, ratio=0.5)
+                        
+                    logging_dict = {'lr': [group['lr'] for group in optimizer.param_groups], 'train_loss': float('inf')}
+                else:
+                    raise
+                    
+            training_log[epoch] = logging_dict                    
+            if self.is_early_stopping(epoch):
+                print(f'[Trainer CallBack] 📢 Epoch {epoch}/{epochs}: Detect Overfitting! Breaking Training Process...')
+                break
+            if self.logging:
+                if epoch % self.logging == 0:
+                    print(f'[Trainer CallBack] 📢 Epoch {epoch}/{epochs}:', fmt(logging_dict))
+            else:
+                print(f'{epoch}...', end=' ')
+                    
+            if self._at_time_limit(start_training_time):
+                print(f'[Trainer CallBack] ⚠️ Epoch {epoch}/{epochs}: Thời gian training giới hạn là {self.training_time}, hết giờ tại epoch {epoch}/{epochs}')
+                break
+                
+        if self.logging_file:
+            os.makedirs(f'{self.checkpoints_dir}/logs', exist_ok=True)
+            with open(f"{self.checkpoints_dir}/logs/{self.save_name}_logging.json", "a", encoding="utf-8") as f:
+                f.write(json.dumps(training_log))
+                
+        if self.use_ema and self.ema_net is not None:
+            self._save_state_dict(self.ema_net.module)
+        else:
+            self._save_state_dict(network)
+        print(f'[Trainer CallBack] 📢 Kết thúc training.\n')
+
+        best_model, last_model = None, None
+        eval_net = self.ema_net.module if (self.use_ema and self.ema_net is not None) else network
+        if self.return_best :
+            best_model = self.best_stage[0][2] if self.best_stage[0][2] is not None else eval_net.state_dict()
+            best_model = {k.replace("module.", ""): v.detach().cpu().clone() for k, v in best_model.items()}
+        if self.return_last:
+            last_model = eval_net.state_dict()
+            last_model = {k.replace("module.", ""): v.detach().cpu().clone() for k, v in last_model.items()}
+
+        del network
+        torch.cuda.empty_cache()
+        gc.collect()
+        return training_log, best_model, last_model
+
+    def _time_str_to_seconds(self, time_str):
+        days, hours, minutes, seconds = map(int, time_str.split(":"))
+        return days * 86400 + hours * 3600 + minutes * 60 + seconds
+
+    def _update_best_network(self, network, val_score, epoch):
+        topk = max(1, self.topk)
+        self.best_stage.append([val_score, epoch, {k: v.detach().cpu().clone() for k, v in network.state_dict().items()}])
+        self.best_stage = sorted(self.best_stage, reverse=(self.mode == 'max'), key=lambda x: x[0])[:topk]
+        if val_score in [x[0] for x in self.best_stage]:
+            return True
+        return False
+
+    def is_early_stopping(self, epoch):
+        if self.best_stage[0][1] is None:
+            return False
+        if not self.early_stopping:
+            return False
+        return epoch - self.best_stage[0][1] >= self.early_stopping
+
+    def _at_time_limit(self, start_training_time):
+        return time.time() - start_training_time >= self.training_time
+
+    def _save_state_dict(self, network):
+        if self.topk <= 0:
+            return
+
+        if self.save_best:
+            for r in range(self.topk):
+                os.makedirs(f'{self.checkpoints_dir}/r{r+1}s', exist_ok=True)
+                
+            for rank, (score, epoch, state_dict) in enumerate(self.best_stage):
+                if state_dict is None:
+                    continue
+                state_dict = {k.replace("module.", ""): v.detach().cpu().clone() for k, v in state_dict.items()}
+                torch.save(state_dict, f'{self.checkpoints_dir}/r{rank+1}s/{self.save_name}_r{rank+1}_vs{score:.5f}_{"ema" if self.ema_net is not None else ""}.pth')
+        if self.save_last:
+            os.makedirs(f'{self.checkpoints_dir}/lasts', exist_ok=True)
+            state_dict = {k.replace("module.", ""): v.detach().cpu().clone() for k, v in network.state_dict().items()}
+            torch.save(state_dict, f'{self.checkpoints_dir}/lasts/{self.save_name}_last_{"ema" if self.ema_net is not None else ""}.pth')
+        
+    def _train_epoch(self, network, train_loader, optimizer, scheduler, loss_fn, encoded_docs):
+        network.train()
+        total_loss = 0
+        total_loss_dict = {}
+        for batch_idx, batch in enumerate(train_loader):
+            optimizer.zero_grad()
+            with torch.autocast(device_type=self.device, dtype=torch.float16):
+                loss, loss_dict = self._cal_loss(network, batch, batch_idx, loss_fn, encoded_docs)
+
+                for k, v in loss_dict.items():
+                    t = total_loss_dict.get(k, 0)
+                    total_loss_dict[k] = t + v
+            self.grad_scaler.scale(loss).backward()
+            self.grad_scaler.unscale_(optimizer)
+            grad_norm = nn.utils.clip_grad_norm_(network.parameters(), self.max_grad_norm)
+            # print(grad_norm) # Bỏ cmt dòng này để biết nên chọn max_grad_norm bằng bao nhiêu...
+            self.grad_scaler.step(optimizer)
+            self.grad_scaler.update()
+            if self.schedule_in_step and scheduler:
+                scheduler.step()
+            if self.use_ema and self.ema_net is not None:
+                self.ema_net.update(network)
+            total_loss += loss
+        return (total_loss / len(train_loader)).item(), {k: v.item() / len(train_loader) for k, v in total_loss_dict.items()}
+
+    def _eval_epoch(self, network, val_loader, eval_fn, encoded_docs):
+        network.eval()
+        total_score = 0.0
+        total_score_dict = {}
+        object_lists = None  # sẽ init sau
+    
+        with torch.no_grad():
+            for batch_idx, batch in enumerate(val_loader):
+                score, score_dict, objects = self._cal_val_score(network, batch, batch_idx, eval_fn, encoded_docs)
+                total_score += score
+
+                for k, v in score_dict.items():
+                    t = total_score_dict.get(k, 0)
+                    total_score_dict[k] = t + v
+    
+                if objects:
+                    if object_lists is None:
+                        object_lists = [[] for _ in range(len(objects))]
+    
+                    for i, obj in enumerate(objects):
+                        object_lists[i].append(obj.detach())
+    
+        if object_lists is not None:
+            object_arrays = [
+                torch.concat(obj_list, dim=0).cpu().numpy()
+                for obj_list in object_lists
+            ]
+        else:
+            object_arrays = []
+    
+        return total_score / len(val_loader), {k: v / len(val_loader) for k, v in total_score_dict.items()}, object_arrays
+
+    def _get_encoded_docs(self, network, corpus_loader):
+        network.eval()
+        with torch.no_grad():
+            encoded_docs = []
+            for batch_idx, batch in enumerate(corpus_loader):
+                input_ids = batch['input_ids'].to(self.device)
+                attn_mask = batch['attn_mask'].to(self.device)
+                encoded_doc = network(input_ids, attn_mask, is_query=False)
+                encoded_docs.append(encoded_doc)
+            encoded_docs = torch.concat(encoded_docs, dim=0).squeeze(1)
+        return encoded_docs
+
+    def _cal_loss(self, network, batch, batch_idx, loss_fn, encoded_docs):
+        # Bạn cần override _cal_loss để tính loss
+        text_input_ids = batch['text_input_ids'].to(self.device)
+        text_attn_mask = batch['text_attn_mask'].to(self.device)
+        pos_idxes = batch['pos_idxes'].to(self.device)
+        pos_mask = batch['pos_mask'].to(self.device)
+        neg_idxes = batch['neg_idxes'].to(self.device)
+
+        encoded_text = network(text_input_ids, text_attn_mask, is_query=True)
+        encoded_pos = encoded_docs[pos_idxes]
+        encoded_neg = encoded_docs[neg_idxes]
+
+        loss_dict = loss_fn(encoded_text, encoded_pos, encoded_neg, pos_mask)
+        return loss_dict['total'], loss_dict
+
+    def _cal_val_score(self, network, batch, batch_idx, eval_fn, encoded_docs):
+        # Bạn cần override _cal_val_score để tính val score, list bên cạnh là để trả về y hay pred gì đó (nếu cần)
+        text_input_ids = batch['text_input_ids'].to(self.device)
+        text_attn_mask = batch['text_attn_mask'].to(self.device)
+        gt_pos_idxes = batch['gt_pos_idxes']
+        encoded_text = network(text_input_ids, text_attn_mask, is_query=True)
+
+        scores = torch.matmul(encoded_text, encoded_docs.T)
+        topk_scores, topk_indices = torch.topk(scores, k=10)
+        pred_topk = [
+            idx[score > 0].tolist()
+            for score, idx in zip(topk_scores, topk_indices)
+        ]
+
+        pred_topk = list_to_tuple(pred_topk)
+        gt_pos_idxes = list_to_tuple(gt_pos_idxes)
+        score_dict = eval_fn(pred_topk, gt_pos_idxes)
+        return score_dict['recall'], score_dict, []
+
+# %% [code]
+def tokenize_to_parts(text, tokenizer, max_len, max_n_parts):
+    # Tokenize với overflow để chia thành nhiều đoạn
+    enc = tokenizer(
+        text,
+        max_length=max_len*max_n_parts,
+        truncation=True,
+        padding="max_length",
+        return_overflowing_tokens=True,
+        return_tensors="pt"
+    )
+
+    input_ids = enc["input_ids"].reshape(max_n_parts, max_len)          # (n_parts, max_len)
+    attn_mask = enc["attention_mask"].reshape(max_n_parts, max_len)     # (n_parts, max_len)
+
+    return input_ids, attn_mask
+
+class LawRetrievalDataset(Dataset):
+    def __init__(self, all_data, using_idxes, corpus_dict, tokenizer, max_len, max_n_parts, n_negs):
+        super().__init__()
+
+        self.all_data = all_data
+        self.using_idxes = using_idxes
+        self.tokenizer = tokenizer
+        self.max_len = max_len
+        self.max_n_parts = max_n_parts
+        self.n_negs = n_negs
+
+        # ===== BUILD CORPUS =====
+        idx = 0
+        self.corpus_list = []
+        self.corpus_dict = {}
+
+        for doc_name, articles_dict in corpus_dict.items():
+            self.corpus_dict[doc_name] = {}
+            for article_idx, content in articles_dict.items():
+                self.corpus_list.append([doc_name, article_idx, content])
+                self.corpus_dict[doc_name][article_idx] = {
+                    'content': content,
+                    'idx': idx
+                }
+                idx += 1
+
+    def __len__(self):
+        return len(self.using_idxes)
+
+    # ===== ENCODE DOC =====
+    def _encode_contexts(self, idxes):
+        all_input_ids, all_attn_mask = [], []
+
+        for idx in idxes:
+            name, art, _ = self.corpus_list[idx]
+            corpus = self.corpus_dict[name][art]
+
+            if 'content_input_ids' in corpus:
+                content_input_ids = corpus['content_input_ids']
+                content_attn_mask = corpus['content_attn_mask']
+            else:
+                content = corpus['content']
+                content_input_ids, content_attn_mask = tokenize_to_parts(
+                    content, self.tokenizer, self.max_len, self.max_n_parts
+                )
+                corpus['content_input_ids'] = content_input_ids
+                corpus['content_attn_mask'] = content_attn_mask
+
+            all_input_ids.append(content_input_ids)
+            all_attn_mask.append(content_attn_mask)
+
+        all_input_ids = torch.stack(all_input_ids)
+        all_attn_mask = torch.stack(all_attn_mask)
+
+        return all_input_ids, all_attn_mask
+
+    def __getitem__(self, idx):
+        ridx = self.using_idxes[idx]
+        data = self.all_data[ridx]
+        
+        query_text = data['text']
+
+        text_input_ids, text_attn_mask = tokenize_to_parts(
+            query_text, self.tokenizer, self.max_len, 1
+        )
+
+        # ===== POS =====
+        gt_pos_idxes = []
+        hard_names = []
+        for law in data['relevant_law']:
+            name = law['doc']
+            art = law['art']
+            gt_pos_idxes.append(self.corpus_dict[name][art]['idx'])
+            if name not in hard_names:
+                hard_names.append(name)
+
+        pos_idxes = torch.tensor(gt_pos_idxes, dtype=torch.long)
+        pos_mask = torch.ones(len(pos_idxes))
+
+        # ===== NEG =====
+        hard_neg_idxes = []
+        for name in hard_names:
+            for content in self.corpus_dict[name].values():
+                if content['idx'] in gt_pos_idxes:
+                    continue
+                hard_neg_idxes.append(content['idx'])
+
+        easy_neg_idxes = list(range(len(self.corpus_list)))
+        for i in gt_pos_idxes + hard_neg_idxes:
+            if i in easy_neg_idxes:
+                easy_neg_idxes.remove(i)
+
+        n_hards = min(len(hard_neg_idxes), self.n_negs // 2)
+        neg_idxes = random.sample(hard_neg_idxes, n_hards) + random.sample(easy_neg_idxes, self.n_negs - n_hards)
+        neg_idxes = torch.tensor(neg_idxes, dtype=torch.long)
+
+        return {
+            'text_input_ids': text_input_ids,
+            'text_attn_mask': text_attn_mask,
+            'gt_pos_idxes': gt_pos_idxes,
+            'pos_idxes': pos_idxes,
+            'pos_mask': pos_mask,
+            'neg_idxes': neg_idxes,
+        }
+
+class CorpusDataset(Dataset):
+    def __init__(self, corpus_dict, tokenizer, max_len, max_n_parts):
+        super().__init__()
+        self.tokenizer = tokenizer
+        self.max_len = max_len
+        self.max_n_parts = max_n_parts
+
+        idx = 0
+        self.corpus_list = []
+        self.corpus_dict = {}
+        for doc_name, articles_dict in corpus_dict.items():
+            self.corpus_dict[doc_name] = {}
+            for article_idx, content in articles_dict.items():
+                self.corpus_list.append([doc_name, article_idx, content])
+                self.corpus_dict[doc_name][article_idx] = {'content': content, 'idx': idx}
+                idx += 1
+
+    def __len__(self):
+        return len(self.corpus_list)
+        
+    def _encode_contexts(self, idxes):
+        all_input_ids, all_attn_mask = [], []
+        for idx in idxes:
+            name = self.corpus_list[idx][0]
+            art  = self.corpus_list[idx][1]
+            corpus = self.corpus_dict[name][art]
+            if 'content_input_ids' in corpus and 'content_attn_mask' in corpus:
+                content_input_ids = corpus['content_input_ids']
+                content_attn_mask = corpus['content_attn_mask']
+            else:
+                content = corpus['content']
+                content_input_ids, content_attn_mask = tokenize_to_parts(content, self.tokenizer, self.max_len, self.max_n_parts)
+                corpus['content_input_ids'] = content_input_ids
+                corpus['content_attn_mask'] = content_attn_mask
+                
+            all_input_ids.append(content_input_ids)
+            all_attn_mask.append(content_attn_mask)
+            
+        all_input_ids = torch.stack(all_input_ids)
+        all_attn_mask = torch.stack(all_attn_mask)
+        return all_input_ids, all_attn_mask
+
+    def __getitem__(self, idx):
+        input_ids, attn_mask = self._encode_contexts([idx])
+        
+        return {
+            'input_ids': input_ids, 
+            'attn_mask': attn_mask, 
+        }    
+
+def _pad_batch(tensor_list, pad_value=0):
+    """
+    tensor_list: list of tensors, mỗi tensor shape (Nk, max_n_parts, max_len)
+    return: tensor shape (B, max_Nk, max_n_parts, max_len)
+    """
+    max_Nk = max(t.size(0) for t in tensor_list)
+
+    padded = []
+    for t in tensor_list:
+        Nk = t.size(0)
+
+        if Nk < max_Nk:
+            pad_shape = (max_Nk - Nk, *t.shape[1:])
+            pad_tensor = t.new_full(pad_shape, pad_value)
+            t = torch.cat([t, pad_tensor], dim=0)
+
+        padded.append(t)
+
+    return torch.stack(padded)  # (B, max_Nk, max_n_parts, max_len)
+    
+def collate_fn(batch):
+    text_input_ids = torch.stack([b["text_input_ids"] for b in batch])
+    text_attn_mask = torch.stack([b["text_attn_mask"] for b in batch])
+    gt_pos_idxes = [b["gt_pos_idxes"] for b in batch]
+    neg_idxes = torch.stack([b["neg_idxes"] for b in batch])
+
+    pos_idxes = [b["pos_idxes"].unsqueeze(-1).unsqueeze(-1) for b in batch]
+    pos_mask = [b["pos_mask"].unsqueeze(-1).unsqueeze(-1) for b in batch]
+
+    # pad theo Nk
+    pos_idxes = _pad_batch(pos_idxes, pad_value=0).squeeze(-1).squeeze(-1)
+    pos_mask = _pad_batch(pos_mask, pad_value=0).squeeze(-1).squeeze(-1)
+
+    return {
+        'text_input_ids': text_input_ids,
+        'text_attn_mask': text_attn_mask,
+        'gt_pos_idxes': gt_pos_idxes,
+        'pos_idxes': pos_idxes,
+        'pos_mask': pos_mask,
+        'neg_idxes': neg_idxes,
+    }
+
+# %% [code]
+def encode_corpus(state_dicts, network, corpus_loader, device):
+    if torch.cuda.device_count() > 1:
+        network = nn.DataParallel(network)
+    network.to(device)
+    network.eval()
+
+    all_model_embs = []
+    for i, state_dict in enumerate(state_dicts):
+        # ===== load model =====
+        if torch.cuda.device_count() > 1:
+            network.module.load_state_dict(state_dict)
+        else:
+            network.load_state_dict(state_dict)
+
+        encoded_docs = []
+
+        with torch.no_grad():
+            for batch in corpus_loader:
+                input_ids = batch['input_ids'].to(device)
+                attn_mask = batch['attn_mask'].to(device)
+
+                emb = network(input_ids, attn_mask, is_query=False)  # [B, 1, D] hoặc [B, D]
+
+                encoded_docs.append(emb)
+
+        encoded_docs = torch.concat(encoded_docs, dim=0).squeeze(1)  # [N, D]
+        all_model_embs.append(encoded_docs)
+
+    # ===== ensemble =====
+    # stack → [M, N, D]
+    all_model_embs = torch.stack(all_model_embs, dim=0)
+    final_embs = all_model_embs.mean(dim=0)  # [N, D]
+
+    return final_embs
+
+def test(state_dicts, network, test_loader, device, eval_fn, encoded_docs, bm25_scores, topks=[5, 10, 15]):
+    if torch.cuda.device_count() > 1:
+        network = nn.DataParallel(network)
+    network.to(device)
+    network.eval()
+
+    per_model_scores = []
+    max_k = max(topks)
+    
+    all_scores = []
+    all_gt_pos_idxes = []
+    with torch.no_grad():
+        for batch in test_loader:
+            text_input_ids = batch['text_input_ids'].to(device)
+            text_attn_mask = batch['text_attn_mask'].to(device)
+            gt_pos_idxes = batch['gt_pos_idxes']
+            all_gt_pos_idxes.extend(gt_pos_idxes)
+
+            list_encoded_texts = []
+
+            for state_dict in state_dicts:
+                # ===== load model =====
+                if torch.cuda.device_count() > 1:
+                    network.module.load_state_dict(state_dict)
+                else:
+                    network.load_state_dict(state_dict)
+
+                encoded_text = network(text_input_ids, text_attn_mask, is_query=True)
+                list_encoded_texts.append(encoded_text)
+                
+            ensemble_encoded_text = torch.stack(list_encoded_texts, dim=0).mean(dim=0)
+            scores = torch.matmul(ensemble_encoded_text, encoded_docs.T) # B, M
+            all_scores.append(scores)
+
+    all_scores = torch.concat(all_scores, dim=0)          # (N, M)
+    
+    bm25_scores = torch.tensor(bm25_scores).to(all_scores.device)  # (N, M)    
+    bm25_scores = bm25_scores.float()
+    
+    # min-max về [0, 1]
+    bm25_scores = (
+        bm25_scores - bm25_scores.min(dim=-1, keepdim=True).values
+    ) / (
+        bm25_scores.max(dim=-1, keepdim=True).values
+        - bm25_scores.min(dim=-1, keepdim=True).values
+        + 1e-8
+    )
+    # đổi sang [-1, 1]
+    bm25_scores = bm25_scores * 2 - 1
+    
+    all_gt_pos_idxes = list_to_tuple(all_gt_pos_idxes)
+    
+    final_score = {}
+    for weight in [0, 0.25, 0.5, 0.75, 1]:
+        
+        # score cuối
+        final_scores = all_scores + weight * bm25_scores
+    
+        # topk lớn nhất
+        topk_scores, topk_indices = torch.topk(final_scores, k=max_k)
+    
+        pred_topk_full = [
+            idx[score > 0].tolist()
+            for score, idx in zip(topk_scores, topk_indices)
+        ]
+    
+        pred_topk_full = list_to_tuple(pred_topk_full)
+    
+        final_score[weight] = {}
+    
+        for k in topks:
+            pred_topk_k = [p[:k] for p in pred_topk_full]
+            final_score[weight][k] = eval_fn(pred_topk_k, all_gt_pos_idxes)
+    
+    return final_score
+
+# %% [code]
+with open(f'{train_dir}/train.json', "r", encoding="utf-8") as f:
+    data_train = json.load(f)
+    
+with open(f'{test_dir}/test.json', "r", encoding="utf-8") as f:
+    data_test = json.load(f)
+    
+with open(f'{test_dir}/corpus.json', "r", encoding="utf-8") as f:
+    data_corpus = json.load(f)
+
+print('Train:', len(data_train))
+print('Test:', len(data_test))
+print('Corpus:', len(data_corpus))
+
+# %% [code]
+# trigger_types = sorted(list(set([e['label'] for d in data_train + data_test for e in d['issues']]))) # NBR : Neighbor relation
+# bio_trigger_types = ['O'] + [f'{prefix}-{trg}' for trg in trigger_types for prefix in ['B', 'I']]
+# trigger_label2id = {l: i for i, l in enumerate(bio_trigger_types)}
+# trigger_id2label = {i: l for l, i in trigger_label2id.items()}
+
+# argument_types = sorted(list(set([a['role'] for d in data_train + data_test for e in d['issues'] for a in e['arguments']])))
+# bio_argument_types = ['O'] + [f'{prefix}-{arg}' for arg in argument_types for prefix in ['B', 'I']]
+# argument_label2id = {l: i for i, l in enumerate(bio_argument_types)}
+# argument_id2label = {i: l for l, i in argument_label2id.items()}
+
+# label2id = {
+#     'Trg': trigger_label2id, 
+#     'Arg': argument_label2id, 
+# }
+
+# id2label = {
+#     'Trg': trigger_id2label, 
+#     'Arg': argument_id2label, 
+# }
+
+# %% [code]
+# zero_events_idxes = []
+# for idx, d in enumerate(data_train):
+#     if len(d['issues']) == 0:
+#         zero_events_idxes.append(idx)
+
+# n_zero_events_samples = len(zero_events_idxes)
+# n_has_events_samples = len(data_train) - n_zero_events_samples
+
+# random.seed(42)
+# k = min(int(n_has_events_samples * zero_events_rate), len(zero_events_idxes))
+# sampled_zero_events_idxes = random.sample(zero_events_idxes, k)
+
+# new_data_train = []
+# for idx, d in enumerate(data_train):
+#     if len(d['issues']) == 0:
+#         if idx in sampled_zero_events_idxes:
+#             new_data_train.append(d)
+#     else:
+#         new_data_train.append(d)
+# data_train = new_data_train
+
+# print('Train:', len(data_train))
+
+# %% [code]
+if debug_only:
+    data_train = data_train[:20]
+    data_test = data_test[:20]
+
+    print('Train:', len(data_train))
+    print('Test:', len(data_test))
+
+# %% [code]
+tokenizer = AutoTokenizer.from_pretrained(backbone_model_name)
+
+# %% [code]
+print('Experiment name:', state_dict_save_name)
+
+# %% [code]
+if not test_only:
+    full_idxes = np.array(range(len(data_train)))
+    training_logs, best_models, last_models = [], [], []
+    start_training_time = time.time()
+    for seed in SEEDS:
+        kf = KFold(n_splits=nfolds, shuffle=True, random_state=seed)
+        generator = torch.Generator()
+        generator.manual_seed(seed)
+    
+        corpusset = CorpusDataset(data_corpus, tokenizer, **corpus_memory_params)
+        corpus_loader = DataLoader(corpusset, generator=generator, **val_loader_params)
+        for fold_idx, (tr_idx, va_idx) in enumerate(kf.split(full_idxes)):
+            if only_fold_idx is not None and only_fold_idx >= 0 and only_fold_idx != fold_idx:
+                continue
+            set_seed(seed)
+    
+            train_idxes, val_idxes = full_idxes[tr_idx], full_idxes[va_idx]
+
+            trainset = LawRetrievalDataset(data_train, train_idxes, data_corpus, tokenizer, **train_memory_params)
+            valset = LawRetrievalDataset(data_train, val_idxes, data_corpus, tokenizer, **val_memory_params)
+            
+            train_loader = DataLoader(trainset, generator=generator, collate_fn=collate_fn, **train_loader_params)
+            val_loader = DataLoader(valset, generator=generator, collate_fn=collate_fn, **val_loader_params)
+    
+            my_model = EncodeModel(
+                **model_params
+            )
+            total_params = sum(p.numel() for p in my_model.parameters())
+            print(f"Total params: {total_params:,}")
+            
+            # optimizer, scheduler = configure_optimizers(my_model, optim_params, scheduler_params)
+            encoder_params = set(map(id, my_model.encoder.parameters()))
+            other_params = [
+                p for p in my_model.parameters()
+                if id(p) not in encoder_params
+            ]        
+            optimizer = optim.AdamW([
+                {"params": my_model.encoder.parameters(), "lr": 2e-5},
+                {"params": other_params}
+            ], lr=5e-4)
+            scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=20, eta_min=1e-6)
+            
+            loss_fn = CustomLoss(
+                **loss_func_params
+            )
+            eval_fn = CustomEvalFn(**eval_func_params)
+            trainer_params['save_name'] = f'{state_dict_save_name}_s{seed}_f{fold_idx}'
+            trainer = Trainer(**trainer_params)
+    
+            print(f'Start Training Fold {fold_idx}...')
+            training_log, best_model, last_model = trainer.fit(
+                my_model, optimizer, scheduler, loss_fn, epochs, train_loader, val_loader, corpus_loader, eval_fn, 
+                start_epoch=1, start_training_time=start_training_time, refresh_every=2, 
+            )
+            
+            training_logs.append(training_log)
+            best_models.append(best_model)
+            last_models.append(last_model)
+
+# %% [code]
+def load_all_state_dicts(folder):
+    files = []
+    
+    for file in os.listdir(folder):
+        if file.endswith(".pt") or file.endswith(".pth"):
+            m = re.search(r"f(\d+)", file)   # tìm f<số>
+            if m:
+                fold = int(m.group(1))
+                files.append((fold, file))
+
+    # sort theo fold
+    files.sort(key=lambda x: x[0])
+
+    state_dicts = []
+    for fold, file in files:
+        path = os.path.join(folder, file)
+        print(f"Loading fold {fold}: {file}")
+        state_dict = torch.load(path, map_location="cpu")
+        state_dicts.append(state_dict)
+
+    return state_dicts
+
+if test_only:
+    snapshot_download(repo_id=repo_name, local_dir="", repo_type="model", allow_patterns=[f"{state_dict_save_name}/**"])
+    get_ipython().system('rm -rf .cache .gitattributes')
+    
+    best_models = load_all_state_dicts(f"{state_dict_save_name}/r1s")
+    last_models = load_all_state_dicts(f"{state_dict_save_name}/lasts")
+
+# %% [code]
+os.makedirs(f'{checkpoints_dir}/results', exist_ok=True)
+testset = LawRetrievalDataset(data_test, range(len(data_test)), data_corpus, tokenizer, **val_memory_params)
+generator = torch.Generator()
+test_loader = DataLoader(testset, generator=generator, collate_fn=collate_fn, **val_loader_params)
+eval_fn = CustomEvalFn(**eval_func_params)
+my_model = EncodeModel(
+    **model_params
+)
+total_params = sum(p.numel() for p in my_model.parameters())
+print(f"Total params: {total_params:,}")
+
+# %% [code]
+start_time = time.time()
+encoded_docs = encode_corpus(best_models, my_model, corpus_loader, device)
+best_score = test(best_models, my_model, test_loader, device, eval_fn, encoded_docs, bm25_scores)
+
+encoded_docs = encode_corpus(last_models, my_model, corpus_loader, device)
+last_score = test(last_models, my_model, test_loader, device, eval_fn, encoded_docs, bm25_scores)
+
+result_test = {"Best model": best_score, "Last model": last_score}
+
+with open(f"{checkpoints_dir}/results/{state_dict_save_name}_test.json", "w", encoding="utf-8") as f:
+    json.dump(result_test, f, ensure_ascii=False, indent=2)
+    
+print('Test:', time.time() - start_time, 's --> Done!')
+
+# %% [code]
+def dict_to_df(data):
+    """
+    data format:
+    {
+        model_name: {
+            weight: {
+                topk: {
+                    metric: value
+                }
+            }
+        }
+    }
+    """
+
+    row_tuples = []
+    row_values = []
+
+    # ===== lấy model đầu tiên =====
+    first_model = next(iter(data.values()))
+
+    # ===== weight keys =====
+    weight_keys = list(first_model.keys())
+
+    # ===== topk keys =====
+    first_weight = next(iter(first_model.values()))
+    topk_keys = list(first_weight.keys())
+
+    # ===== metric keys =====
+    first_topk = next(iter(first_weight.values()))
+    metrics = list(first_topk.keys())
+
+    for weight in weight_keys:
+
+        for topk in topk_keys:
+
+            # ===== multi index row =====
+            row_tuples.append((weight, topk))
+
+            row = {}
+
+            for model_name, model_data in data.items():
+
+                for metric in metrics:
+
+                    row[(model_name, metric)] = (
+                        model_data[weight][topk][metric]
+                    )
+
+            row_values.append(row)
+
+    # ===== dataframe =====
+    df = pd.DataFrame(row_values)
+
+    # ===== multi columns =====
+    df.columns = pd.MultiIndex.from_tuples(
+        df.columns,
+        names=["model", "metric"]
+    )
+
+    # ===== multi index =====
+    df.index = pd.MultiIndex.from_tuples(
+        row_tuples,
+        names=["weight", "topk"]
+    )
+
+    return df
+
+result_test_df = dict_to_df(result_test)
+result_test_df.to_excel(f"{checkpoints_dir}/results/{state_dict_save_name}_test_df.xlsx")
+result_test_df
+
+# %% [code]
+key = ("Best model", "recall")
+
+result_test_df_best = (
+    result_test_df
+    .sort_values(by=key, ascending=False)
+    .groupby(level="weight")
+    .head(1)
+)
+result_test_df_best.to_excel(f"{checkpoints_dir}/results/{state_dict_save_name}_test_df_best.xlsx")
+result_test_df_best
+
+# %% [code]
+def get_avg_best_score(logs):
+    return float(np.mean([list(log.values())[-1]['best_score'] for log in logs]))
+    
+def get_avg_log(logs, epochs):
+    avg_log = {}
+
+    for epoch in range(1, epochs + 1):
+        val_score = 0.0
+        train_loss = 0.0
+        n_eval = 0
+
+        for idx in range(len(logs)):
+            log = logs[idx].get(epoch, logs[idx].get(str(epoch)))
+            if log is None:
+                continue
+
+            val_score += log.get('val_score', 0.0)
+            train_loss += log.get('train_loss', 0.0)
+            n_eval += 1
+
+        if n_eval == 0:
+            continue
+
+        avg_log[epoch] = {
+            'train_loss': train_loss / n_eval,
+            'val_score': val_score / n_eval if val_score != 0 else float('inf')
+        }
+
+    return avg_log
+
+def parse_label_key(label: str):
+    try:
+        first = float(label.split('_', 1)[0]) # số đầu: trước dấu _
+        last = float(re.findall(r'_(\d+(?:\.\d+)?)$', label)[0])
+        return first, last
+    except:
+        return (0, 0)
+
+def plot_training_logs(logs_dict, save_path=None, figsize=(24, 10)):
+    fig, axes = plt.subplots(1, 2, figsize=figsize)
+
+    # ===== Plot Train Loss =====
+    for name, log in logs_dict.items():
+        epochs = sorted(log.keys())
+        train_loss = [log[e]['train_loss'] for e in epochs]
+        axes[0].plot(epochs, train_loss, label=name)
+
+    axes[0].set_xlabel('Epoch')
+    axes[0].set_ylabel('Train Loss')
+    axes[0].set_title('Training Loss')
+    axes[0].grid(True)
+
+    # ===== Plot Validation Score =====
+    for name, log in logs_dict.items():
+        epochs = sorted(log.keys())
+        val_score = [log[e]['val_score'] for e in epochs]
+        axes[1].plot(epochs, val_score, label=name)
+
+    axes[1].set_xlabel('Epoch')
+    axes[1].set_ylabel('Validation Score')
+    axes[1].set_title('Validation Score')
+    axes[1].grid(True)
+
+    # ===== Shared Legend =====
+    handles, labels = axes[0].get_legend_handles_labels()
+    pairs = list(zip(handles, labels))
+    pairs_sorted = sorted(
+        pairs,
+        key=lambda x: parse_label_key(x[1])
+    )
+    handles_sorted, labels_sorted = zip(*pairs_sorted)
+    
+    axes[0].legend(
+        handles_sorted,
+        labels_sorted,
+        loc='center left',
+        bbox_to_anchor=(1.01, 0.5),
+        borderaxespad=0.
+    )
+
+    plt.tight_layout(rect=[0, 0, 1, 1])
+
+    if save_path is not None:
+        os.makedirs(os.path.dirname(save_path), exist_ok=True) if os.path.dirname(save_path) else None
+        plt.savefig(save_path, dpi=300, bbox_inches='tight')
+
+    plt.show()
+
+# %% [code]
+if not test_only:
+    snapshot_download(repo_id=repo_name, local_dir="", repo_type="model", allow_patterns=["**/*lr*.json"], ignore_patterns=[])
+    get_ipython().system('rm -rf .cache .gitattributes')
+
+# %% [code]
+if not test_only:
+    experiments = {}
+    for experiment in os.listdir(pretrained_dir):
+        experiment_logs = []
+        try:
+            for seed in SEEDS:
+                for fold_idx in range(nfolds):
+                    with open(f"{pretrained_dir}/{experiment}/logs/{experiment}_s{seed}_f{fold_idx}_logging.json", "r", encoding="utf-8") as f:
+                        experiment_log = json.load(f)
+                    experiment_logs.append(experiment_log)
+        except:
+            pass
+        experiments[experiment] = get_avg_log(experiment_logs, 1000)
+    experiments[state_dict_save_name] = get_avg_log(training_logs, 1000)
+
+# %% [code]
+if not test_only:
+    score = get_avg_best_score(training_logs)
+    state_dict_save_name, score
+
+# %% [code]
+if not test_only:
+    plot_training_logs(experiments, save_path=f'{checkpoints_dir}/logs/{state_dict_save_name}_log_plot.jpg', figsize=(18, 7.5))
+

11_12_13_lr_17/lasts/11_12_13_lr_17_s26092004_f0_last_ema.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec484758ccbb98bd4185af20d8cb19b080df3a837c407de38d03569b7a4f3407
+size 544007234

11_12_13_lr_17/lasts/11_12_13_lr_17_s26092004_f1_last_ema.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:885527f9be01fc8322a85502efea2864c60e5648a07ee381e8eeb38b6f8a1b5f
+size 544007234

11_12_13_lr_17/lasts/11_12_13_lr_17_s26092004_f2_last_ema.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:938789aef501b8e7db3c04215bab9a242204c68521fce2bee52140bc81b64370
+size 544007234

11_12_13_lr_17/lasts/11_12_13_lr_17_s26092004_f3_last_ema.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4903aec45da6ae86ccc278dcd5fb530e0f0db36a1cdfa48c09197171a5dec09f
+size 544007234

11_12_13_lr_17/lasts/11_12_13_lr_17_s26092004_f4_last_ema.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f65a71018b9e955d6613cd375bc1bd8fa27ad40af7dfef94fad1a5b1633db8a1
+size 544007234

11_12_13_lr_17/logs/11_12_13_lr_17_s26092004_f0_logging.json

@@ -0,0 +1 @@
+{"1": {"lr": [2e-05, 0.0005], "train_loss": 3.849350929260254, "total": 3.849350945207985, "contrastive_loss": 3.7087908588524248, "triplet_loss": 0.27320234113712377}, "2": {"lr": [1.988303923565381e-05, 0.0004969282409784868], "train_loss": 3.6270928382873535, "total": 3.6270927505748327, "contrastive_loss": 3.4903425644073995, "triplet_loss": 0.266304347826087}, "3": {"lr": [1.9535036904803962e-05, 0.0004877886008156408], "train_loss": 3.4080309867858887, "total": 3.408031042602947, "contrastive_loss": 3.2548078964386495, "triplet_loss": 0.30288461538461536}, "4": {"lr": [1.8964561979789496e-05, 0.00047280612778499774], "train_loss": 3.135387897491455, "total": 3.135388020288984, "contrastive_loss": 2.9909837397444607, "triplet_loss": 0.2842809364548495}, "5": {"lr": [1.8185661446562005e-05, 0.00045234974009654937], "train_loss": 3.102684497833252, "total": 3.1026846078725963, "contrastive_loss": 2.9599497102973453, "triplet_loss": 0.27968227424749165}, "6": {"lr": [1.7217514421272206e-05, 0.00042692314190604356], "train_loss": 2.8865349292755127, "total": 2.886535032138378, "contrastive_loss": 2.7526385967548075, "triplet_loss": 0.2637959866220736}, "7": {"lr": [1.60839598967785e-05, 0.00039715242044697206], "train_loss": 2.8706984519958496, "total": 2.870698552466555, "contrastive_loss": 2.7297577618755224, "triplet_loss": 0.27738294314381273}, "8": {"lr": [1.4812909747525698e-05, 0.00036377062968501693], "train_loss": 2.579120397567749, "total": 2.579120342548077, "contrastive_loss": 2.4513306888848243, "triplet_loss": 0.2548076923076923, "val_score": 0.8321726190476191, "best_score": 0.8321726190476191, "new_best_model": true, "recall": 0.8321726190476191, "mAP": 0.32399420233371123, "mRP": 0.3117083333333332}, "9": {"lr": [1.3435661446562005e-05, 0.0003275997400965494], "train_loss": 2.4573817253112793, "total": 2.4573817428537836, "contrastive_loss": 2.332941469938859, "triplet_loss": 0.24749163879598662, "val_score": 0.830029761904762, "best_score": 0.8321726190476191, "new_best_model": false, "recall": 0.830029761904762, "mAP": 0.3227199105568152, "mRP": 0.31034027777777773}, "10": {"lr": [1.1986127417882198e-05, 0.00028953039902753766], "train_loss": 2.2760982513427734, "total": 2.27609834702916, "contrastive_loss": 2.160639223845109, "triplet_loss": 0.23369565217391305, "val_score": 0.8321726190476191, "best_score": 0.8321726190476191, "new_best_model": true, "recall": 0.8321726190476191, "mAP": 0.3276874283509699, "mRP": 0.31656448412698407}, "11": {"lr": [1.0500000000000003e-05, 0.0002505], "train_loss": 2.2247889041900635, "total": 2.2247888061115177, "contrastive_loss": 2.11044505447847, "triplet_loss": 0.23056020066889632, "val_score": 0.8139880952380952, "best_score": 0.8321726190476191, "new_best_model": false, "recall": 0.8139880952380952, "mAP": 0.3163277409690729, "mRP": 0.30428521825396815}, "12": {"lr": [9.013872582117811e-06, 0.00021146960097246246], "train_loss": 2.0722544193267822, "total": 2.0722543977973453, "contrastive_loss": 1.9660775175062708, "triplet_loss": 0.2122700668896321, "val_score": 0.8139880952380952, "best_score": 0.8321726190476191, "new_best_model": false, "recall": 0.8139880952380952, "mAP": 0.31827680146132536, "mRP": 0.3060352182539681}, "13": {"lr": [7.564338553438001e-06, 0.00017340025990345064], "train_loss": 2.0603482723236084, "total": 2.0603482555784907, "contrastive_loss": 1.9539109041858278, "triplet_loss": 0.2121655518394649, "val_score": 0.7973214285714286, "best_score": 0.8321726190476191, "new_best_model": false, "recall": 0.7973214285714286, "mAP": 0.3080162509448222, "mRP": 0.297126984126984}, "14": {"lr": [6.1870902524743065e-06, 0.00013722937031498307], "train_loss": 1.9397445917129517, "total": 1.939744624007107, "contrastive_loss": 1.8402303740332357, "triplet_loss": 0.19742892976588627, "val_score": 0.7971130952380953, "best_score": 0.8321726190476191, "new_best_model": false, "recall": 0.7971130952380953, "mAP": 0.30994053721025444, "mRP": 0.297766369047619}, "15": {"lr": [4.916040103221507e-06, 0.00010384757955302797], "train_loss": 1.9679230451583862, "total": 1.9679230654917432, "contrastive_loss": 1.866120711616848, "triplet_loss": 0.20286371237458195, "val_score": 0.7804464285714287, "best_score": 0.8321726190476191, "new_best_model": false, "recall": 0.7804464285714287, "mAP": 0.2988514526643991, "mRP": 0.28777083333333353}, "16": {"lr": [3.7824855787278e-06, 7.40768580939564e-05], "train_loss": 1.8696411848068237, "total": 1.8696411541074414, "contrastive_loss": 1.773466690726902, "triplet_loss": 0.19199414715719063, "val_score": 0.777529761904762, "best_score": 0.8321726190476191, "new_best_model": false, "recall": 0.777529761904762, "mAP": 0.30044554811507945, "mRP": 0.2886006944444444}, "17": {"lr": [2.814338553438001e-06, 4.865025990345063e-05], "train_loss": 1.9341967105865479, "total": 1.934196752848035, "contrastive_loss": 1.8339760056307484, "triplet_loss": 0.19972826086956522, "val_score": 0.7604464285714286, "best_score": 0.8321726190476191, "new_best_model": false, "recall": 0.7604464285714286, "mAP": 0.2879113760550518, "mRP": 0.2753110119047619}, "18": {"lr": [2.0354380202105066e-06, 2.8193872215002235e-05], "train_loss": 1.8726832866668701, "total": 1.8726833177649456, "contrastive_loss": 1.775915445691367, "triplet_loss": 0.1939799331103679, "val_score": 0.7583630952380953, "best_score": 0.8321726190476191, "new_best_model": false, "recall": 0.7583630952380953, "mAP": 0.2899708530171327, "mRP": 0.2778735119047619}}

11_12_13_lr_17/logs/11_12_13_lr_17_s26092004_f1_logging.json

@@ -0,0 +1 @@
+{"1": {"lr": [2e-05, 0.0005], "train_loss": 3.860725164413452, "total": 3.860725122151965, "contrastive_loss": 3.7198208710022995, "triplet_loss": 0.2752926421404682}, "2": {"lr": [1.988303923565381e-05, 0.0004969282409784868], "train_loss": 3.6681532859802246, "total": 3.6681532843854514, "contrastive_loss": 3.530774374869356, "triplet_loss": 0.26672240802675584}, "3": {"lr": [1.9535036904803962e-05, 0.0004877886008156408], "train_loss": 3.4691953659057617, "total": 3.4691954137489547, "contrastive_loss": 3.3150232627639005, "triplet_loss": 0.3030936454849498}, "4": {"lr": [1.8964561979789496e-05, 0.00047280612778499774], "train_loss": 3.212824583053589, "total": 3.2128246907007734, "contrastive_loss": 3.067079269766409, "triplet_loss": 0.2861622073578595}, "5": {"lr": [1.8185661446562005e-05, 0.00045234974009654937], "train_loss": 3.2040324211120605, "total": 3.2040323621054556, "contrastive_loss": 3.056296664336852, "triplet_loss": 0.28971571906354515}, "6": {"lr": [1.7217514421272206e-05, 0.00042692314190604356], "train_loss": 2.9750967025756836, "total": 2.975096635595213, "contrastive_loss": 2.8368605801891724, "triplet_loss": 0.2709030100334448}, "7": {"lr": [1.60839598967785e-05, 0.00039715242044697206], "train_loss": 2.9426071643829346, "total": 2.9426071524221364, "contrastive_loss": 2.799535438767245, "triplet_loss": 0.28135451505016723}, "8": {"lr": [1.4812909747525698e-05, 0.00036377062968501693], "train_loss": 2.682934284210205, "total": 2.682934407007734, "contrastive_loss": 2.5520023613869145, "triplet_loss": 0.2594063545150502, "val_score": 0.8349198717948718, "best_score": 0.8349198717948718, "new_best_model": true, "recall": 0.8349198717948718, "mAP": 0.3240826978191562, "mRP": 0.31000934829059834}, "9": {"lr": [1.3435661446562005e-05, 0.0003275997400965494], "train_loss": 2.564674139022827, "total": 2.5646741988268187, "contrastive_loss": 2.436463014736622, "triplet_loss": 0.2543896321070234, "val_score": 0.8345032051282052, "best_score": 0.8349198717948718, "new_best_model": false, "recall": 0.8345032051282052, "mAP": 0.32246053654100537, "mRP": 0.3050430021367523}, "10": {"lr": [1.1986127417882198e-05, 0.00028953039902753766], "train_loss": 2.3795409202575684, "total": 2.3795408931464257, "contrastive_loss": 2.2601263244016514, "triplet_loss": 0.2408026755852843, "val_score": 0.8362820512820512, "best_score": 0.8362820512820512, "new_best_model": true, "recall": 0.8362820512820512, "mAP": 0.32689650827567507, "mRP": 0.30914369658119667}, "11": {"lr": [1.0500000000000003e-05, 0.0002505], "train_loss": 2.3244407176971436, "total": 2.324440844481605, "contrastive_loss": 2.205558445142663, "triplet_loss": 0.23933946488294314, "val_score": 0.8185737179487179, "best_score": 0.8362820512820512, "new_best_model": false, "recall": 0.8185737179487179, "mAP": 0.31795989498541577, "mRP": 0.30211591880341887}, "12": {"lr": [9.013872582117811e-06, 0.00021146960097246246], "train_loss": 2.158068895339966, "total": 2.158069010961016, "contrastive_loss": 2.0481330590901963, "triplet_loss": 0.22031772575250835, "val_score": 0.8200320512820513, "best_score": 0.8362820512820512, "new_best_model": false, "recall": 0.8200320512820513, "mAP": 0.32182467588353003, "mRP": 0.30531036324786326}, "13": {"lr": [7.564338553438001e-06, 0.00017340025990345064], "train_loss": 2.149510145187378, "total": 2.1495102081809154, "contrastive_loss": 2.0386587289663463, "triplet_loss": 0.2226170568561873, "val_score": 0.8012820512820513, "best_score": 0.8362820512820512, "new_best_model": false, "recall": 0.8012820512820513, "mAP": 0.3119528853573125, "mRP": 0.2956888354700855}, "14": {"lr": [6.1870902524743065e-06, 0.00013722937031498307], "train_loss": 2.021688938140869, "total": 2.0216889142192724, "contrastive_loss": 1.9180859211695234, "triplet_loss": 0.20558110367892976, "val_score": 0.7989903846153846, "best_score": 0.8362820512820512, "new_best_model": false, "recall": 0.7989903846153846, "mAP": 0.3150698477352125, "mRP": 0.29909161324786343}, "15": {"lr": [4.916040103221507e-06, 0.00010384757955302797], "train_loss": 2.0624663829803467, "total": 2.062466318392036, "contrastive_loss": 1.9556607147522993, "triplet_loss": 0.21362876254180602, "val_score": 0.7713942307692307, "best_score": 0.8362820512820512, "new_best_model": false, "recall": 0.7713942307692307, "mAP": 0.3039083502916837, "mRP": 0.2907532051282051}, "16": {"lr": [3.7824855787278e-06, 7.40768580939564e-05], "train_loss": 1.957850456237793, "total": 1.957850427531877, "contrastive_loss": 1.8569625803459449, "triplet_loss": 0.20035535117056857, "val_score": 0.774727564102564, "best_score": 0.8362820512820512, "new_best_model": false, "recall": 0.774727564102564, "mAP": 0.30521715040360875, "mRP": 0.2920552884615385}, "17": {"lr": [2.814338553438001e-06, 4.865025990345063e-05], "train_loss": 2.0349791049957275, "total": 2.0349790643290135, "contrastive_loss": 1.9296113591529056, "triplet_loss": 0.20944816053511706, "val_score": 0.7548237179487179, "best_score": 0.8362820512820512, "new_best_model": false, "recall": 0.7548237179487179, "mAP": 0.2929323419057794, "mRP": 0.2787393162393163}, "18": {"lr": [2.0354380202105066e-06, 2.8193872215002235e-05], "train_loss": 1.9656423330307007, "total": 1.9656423103051839, "contrastive_loss": 1.8640391882446696, "triplet_loss": 0.20171404682274247, "val_score": 0.7512820512820513, "best_score": 0.8362820512820512, "new_best_model": false, "recall": 0.7512820512820513, "mAP": 0.2927700959630647, "mRP": 0.2797809829059828}}

11_12_13_lr_17/logs/11_12_13_lr_17_s26092004_f2_logging.json

@@ -0,0 +1 @@
+{"1": {"lr": [2e-05, 0.0005], "train_loss": 3.8641021251678467, "total": 3.8641022647104935, "contrastive_loss": 3.723213935775502, "triplet_loss": 0.27487458193979936}, "2": {"lr": [1.988303923565381e-05, 0.0004969282409784868], "train_loss": 3.668407917022705, "total": 3.668408039820234, "contrastive_loss": 3.5310556673285953, "triplet_loss": 0.26734949832775917}, "3": {"lr": [1.9535036904803962e-05, 0.0004877886008156408], "train_loss": 3.4840831756591797, "total": 3.484083092730978, "contrastive_loss": 3.330072409333194, "triplet_loss": 0.3030936454849498}, "4": {"lr": [1.8964561979789496e-05, 0.00047280612778499774], "train_loss": 3.2300777435302734, "total": 3.23007783921666, "contrastive_loss": 3.084018056607964, "triplet_loss": 0.28825250836120403}, "5": {"lr": [1.8185661446562005e-05, 0.00045234974009654937], "train_loss": 3.186211347579956, "total": 3.1862113212662, "contrastive_loss": 3.0409927623327757, "triplet_loss": 0.2842809364548495}, "6": {"lr": [1.7217514421272206e-05, 0.00042692314190604356], "train_loss": 2.9956812858581543, "total": 2.99568120133518, "contrastive_loss": 2.8581351085650084, "triplet_loss": 0.2692307692307692}, "7": {"lr": [1.60839598967785e-05, 0.00039715242044697206], "train_loss": 2.9812674522399902, "total": 2.981267514436141, "contrastive_loss": 2.837190864078177, "triplet_loss": 0.282190635451505}, "8": {"lr": [1.4812909747525698e-05, 0.00036377062968501693], "train_loss": 2.7061684131622314, "total": 2.706168388443248, "contrastive_loss": 2.5743163245976173, "triplet_loss": 0.25961538461538464, "val_score": 0.8394871794871794, "best_score": 0.8394871794871794, "new_best_model": true, "recall": 0.8394871794871794, "mAP": 0.3159504038207164, "mRP": 0.3011132478632479}, "9": {"lr": [1.3435661446562005e-05, 0.0003275997400965494], "train_loss": 2.581252098083496, "total": 2.581252082135765, "contrastive_loss": 2.452596300820443, "triplet_loss": 0.25585284280936454, "val_score": 0.8374038461538461, "best_score": 0.8394871794871794, "new_best_model": false, "recall": 0.8374038461538461, "mAP": 0.31595295467456924, "mRP": 0.30317174145299153}, "10": {"lr": [1.1986127417882198e-05, 0.00028953039902753766], "train_loss": 2.3935935497283936, "total": 2.393593472381898, "contrastive_loss": 2.273653266421927, "triplet_loss": 0.24101170568561872, "val_score": 0.8419871794871795, "best_score": 0.8419871794871795, "new_best_model": true, "recall": 0.8419871794871795, "mAP": 0.3222575051299008, "mRP": 0.3078194444444446}, "11": {"lr": [1.0500000000000003e-05, 0.0002505], "train_loss": 2.3397023677825928, "total": 2.339702287246551, "contrastive_loss": 2.2210040140311453, "triplet_loss": 0.2387123745819398, "val_score": 0.8228205128205128, "best_score": 0.8419871794871795, "new_best_model": false, "recall": 0.8228205128205128, "mAP": 0.3118896490469409, "mRP": 0.29764022435897447}, "12": {"lr": [9.013872582117811e-06, 0.00021146960097246246], "train_loss": 2.1804616451263428, "total": 2.1804617687212584, "contrastive_loss": 2.069898930680393, "triplet_loss": 0.2217809364548495, "val_score": 0.8286538461538462, "best_score": 0.8419871794871795, "new_best_model": false, "recall": 0.8286538461538462, "mAP": 0.31701540507139453, "mRP": 0.302280982905983}, "13": {"lr": [7.564338553438001e-06, 0.00017340025990345064], "train_loss": 2.1661744117736816, "total": 2.1661744388848243, "contrastive_loss": 2.0551480194397995, "triplet_loss": 0.2226170568561873, "val_score": 0.8004326923076923, "best_score": 0.8419871794871795, "new_best_model": false, "recall": 0.8004326923076923, "mAP": 0.30549148966388545, "mRP": 0.2918141025641025}, "14": {"lr": [6.1870902524743065e-06, 0.00013722937031498307], "train_loss": 2.0381269454956055, "total": 2.038126967822429, "contrastive_loss": 1.9339911929739757, "triplet_loss": 0.20819397993311037, "val_score": 0.8030288461538462, "best_score": 0.8419871794871795, "new_best_model": false, "recall": 0.8030288461538462, "mAP": 0.31110761970475515, "mRP": 0.2976455662393164}, "15": {"lr": [4.916040103221507e-06, 0.00010384757955302797], "train_loss": 2.075420618057251, "total": 2.075420673076923, "contrastive_loss": 1.9684501316236414, "triplet_loss": 0.21331521739130435, "val_score": 0.7778365384615384, "best_score": 0.8419871794871795, "new_best_model": false, "recall": 0.7778365384615384, "mAP": 0.2960950131851172, "mRP": 0.28284722222222214}, "16": {"lr": [3.7824855787278e-06, 7.40768580939564e-05], "train_loss": 1.9671577215194702, "total": 1.9671577785326086, "contrastive_loss": 1.8663962884092808, "triplet_loss": 0.20004180602006688, "val_score": 0.7822115384615385, "best_score": 0.8419871794871795, "new_best_model": false, "recall": 0.7822115384615385, "mAP": 0.299882612921415, "mRP": 0.28617708333333347}, "17": {"lr": [2.814338553438001e-06, 4.865025990345063e-05], "train_loss": 2.042740821838379, "total": 2.0427407357206313, "contrastive_loss": 1.9371276651337792, "triplet_loss": 0.21206103678929766, "val_score": 0.7542948717948719, "best_score": 0.8419871794871795, "new_best_model": false, "recall": 0.7542948717948719, "mAP": 0.2851599067502712, "mRP": 0.2716145833333334}, "18": {"lr": [2.0354380202105066e-06, 2.8193872215002235e-05], "train_loss": 1.9720823764801025, "total": 1.972082437878867, "contrastive_loss": 1.8706252694528638, "triplet_loss": 0.2014005016722408, "val_score": 0.7561698717948718, "best_score": 0.8419871794871795, "new_best_model": false, "recall": 0.7561698717948718, "mAP": 0.28773541687864584, "mRP": 0.27583680555555545}}

11_12_13_lr_17/logs/11_12_13_lr_17_s26092004_f3_logging.json

@@ -0,0 +1 @@
+{"1": {"lr": [2e-05, 0.0005], "train_loss": 3.875497341156006, "total": 3.875497263012124, "contrastive_loss": 3.7347408026755855, "triplet_loss": 0.27508361204013376}, "2": {"lr": [1.988303923565381e-05, 0.0004969282409784868], "train_loss": 3.6717770099639893, "total": 3.6717770171404682, "contrastive_loss": 3.5345597793426005, "triplet_loss": 0.26755852842809363}, "3": {"lr": [1.9535036904803962e-05, 0.0004877886008156408], "train_loss": 3.4881162643432617, "total": 3.4881163121864547, "contrastive_loss": 3.3336840983617266, "triplet_loss": 0.3039297658862876}, "4": {"lr": [1.8964561979789496e-05, 0.00047280612778499774], "train_loss": 3.242488384246826, "total": 3.2424883890311453, "contrastive_loss": 3.0960501476275084, "triplet_loss": 0.28887959866220736}, "5": {"lr": [1.8185661446562005e-05, 0.00045234974009654937], "train_loss": 3.2089149951934814, "total": 3.208914970474498, "contrastive_loss": 3.0625361311794523, "triplet_loss": 0.2861622073578595}, "6": {"lr": [1.7217514421272206e-05, 0.00042692314190604356], "train_loss": 3.022059679031372, "total": 3.0220598201687916, "contrastive_loss": 2.8833042514762752, "triplet_loss": 0.2713210702341137}, "7": {"lr": [1.60839598967785e-05, 0.00039715242044697206], "train_loss": 3.005990505218506, "total": 3.005990427074624, "contrastive_loss": 2.86139997909699, "triplet_loss": 0.28448996655518394}, "8": {"lr": [1.4812909747525698e-05, 0.00036377062968501693], "train_loss": 2.735560655593872, "total": 2.7355605926003346, "contrastive_loss": 2.6027446223740593, "triplet_loss": 0.262123745819398, "val_score": 0.8403205128205128, "best_score": 0.8403205128205128, "new_best_model": true, "recall": 0.8403205128205128, "mAP": 0.3177980988629426, "mRP": 0.30497569444444445}, "9": {"lr": [1.3435661446562005e-05, 0.0003275997400965494], "train_loss": 2.6214733123779297, "total": 2.621473229449728, "contrastive_loss": 2.491391797528219, "triplet_loss": 0.25794314381270905, "val_score": 0.8381410256410255, "best_score": 0.8403205128205128, "new_best_model": false, "recall": 0.8381410256410255, "mAP": 0.31950706675654594, "mRP": 0.3073985042735044}, "10": {"lr": [1.1986127417882198e-05, 0.00028953039902753766], "train_loss": 2.434934139251709, "total": 2.434934277996969, "contrastive_loss": 2.313398992736204, "triplet_loss": 0.24519230769230768, "val_score": 0.8443910256410256, "best_score": 0.8443910256410256, "new_best_model": true, "recall": 0.8443910256410256, "mAP": 0.32661201509717136, "mRP": 0.31245005341880355}, "11": {"lr": [1.0500000000000003e-05, 0.0002505], "train_loss": 2.383178234100342, "total": 2.3831783026755855, "contrastive_loss": 2.2625046541858276, "triplet_loss": 0.2418478260869565, "val_score": 0.8224198717948719, "best_score": 0.8443910256410256, "new_best_model": false, "recall": 0.8224198717948719, "mAP": 0.3184463779083572, "mRP": 0.30494951923076935}, "12": {"lr": [9.013872582117811e-06, 0.00021146960097246246], "train_loss": 2.215616464614868, "total": 2.215616589804557, "contrastive_loss": 2.1034852094873537, "triplet_loss": 0.225752508361204, "val_score": 0.8240865384615385, "best_score": 0.8443910256410256, "new_best_model": false, "recall": 0.8240865384615385, "mAP": 0.32351596130528404, "mRP": 0.3094775641025641}, "13": {"lr": [7.564338553438001e-06, 0.00017340025990345064], "train_loss": 2.2090399265289307, "total": 2.2090398986204014, "contrastive_loss": 2.095944407791597, "triplet_loss": 0.22721571906354515, "val_score": 0.8050160256410256, "best_score": 0.8443910256410256, "new_best_model": false, "recall": 0.8050160256410256, "mAP": 0.31032289589268736, "mRP": 0.2953974358974359}, "14": {"lr": [6.1870902524743065e-06, 0.00013722937031498307], "train_loss": 2.071117877960205, "total": 2.0711177966267766, "contrastive_loss": 1.9657551947246028, "triplet_loss": 0.20997073578595318, "val_score": 0.8078365384615385, "best_score": 0.8443910256410256, "new_best_model": false, "recall": 0.8078365384615385, "mAP": 0.31303145021452317, "mRP": 0.2965435363247863}, "15": {"lr": [4.916040103221507e-06, 0.00010384757955302797], "train_loss": 2.105041742324829, "total": 2.1050417080372075, "contrastive_loss": 1.9968508717208402, "triplet_loss": 0.21676421404682275, "val_score": 0.7866826923076923, "best_score": 0.8443910256410256, "new_best_model": false, "recall": 0.7866826923076923, "mAP": 0.3014011297440984, "mRP": 0.28593936965811956}, "16": {"lr": [3.7824855787278e-06, 7.40768580939564e-05], "train_loss": 2.0002076625823975, "total": 2.0002076011836327, "contrastive_loss": 1.897946360916597, "triplet_loss": 0.20286371237458195, "val_score": 0.7877243589743589, "best_score": 0.8443910256410256, "new_best_model": false, "recall": 0.7877243589743589, "mAP": 0.30329969061609685, "mRP": 0.28867654914529906}, "17": {"lr": [2.814338553438001e-06, 4.865025990345063e-05], "train_loss": 2.0700764656066895, "total": 2.0700765246132944, "contrastive_loss": 1.9635234309678093, "triplet_loss": 0.21352424749163879, "val_score": 0.7631410256410256, "best_score": 0.8443910256410256, "new_best_model": false, "recall": 0.7631410256410256, "mAP": 0.28980782214506173, "mRP": 0.2753709935897435}, "18": {"lr": [2.0354380202105066e-06, 2.8193872215002235e-05], "train_loss": 1.9971362352371216, "total": 1.9971362467992266, "contrastive_loss": 1.8945594200721154, "triplet_loss": 0.20390886287625418, "val_score": 0.7589743589743589, "best_score": 0.8443910256410256, "new_best_model": false, "recall": 0.7589743589743589, "mAP": 0.29014007438356393, "mRP": 0.2742772435897434}}

11_12_13_lr_17/logs/11_12_13_lr_17_s26092004_f4_logging.json

@@ -0,0 +1 @@
+{"1": {"lr": [2e-05, 0.0005], "train_loss": 3.851969003677368, "total": 3.851969128867057, "contrastive_loss": 3.711077942098662, "triplet_loss": 0.27424749163879597}, "2": {"lr": [1.988303923565381e-05, 0.0004969282409784868], "train_loss": 3.640805721282959, "total": 3.640805860028219, "contrastive_loss": 3.503629448421823, "triplet_loss": 0.26714046822742477}, "3": {"lr": [1.9535036904803962e-05, 0.0004877886008156408], "train_loss": 3.4500555992126465, "total": 3.4500556869251673, "contrastive_loss": 3.2963773287259617, "triplet_loss": 0.3030936454849498}, "4": {"lr": [1.8964561979789496e-05, 0.00047280612778499774], "train_loss": 3.1876673698425293, "total": 3.1876673873850336, "contrastive_loss": 3.0420881290499584, "triplet_loss": 0.28532608695652173}, "5": {"lr": [1.8185661446562005e-05, 0.00045234974009654937], "train_loss": 3.18599796295166, "total": 3.1859980044157608, "contrastive_loss": 3.0406712560749374, "triplet_loss": 0.28365384615384615}, "6": {"lr": [1.7217514421272206e-05, 0.00042692314190604356], "train_loss": 2.9865071773529053, "total": 2.986507147849603, "contrastive_loss": 2.8489696285796406, "triplet_loss": 0.2692307692307692}, "7": {"lr": [1.60839598967785e-05, 0.00039715242044697206], "train_loss": 2.9650192260742188, "total": 2.9650193026233276, "contrastive_loss": 2.821347443953804, "triplet_loss": 0.28135451505016723}, "8": {"lr": [1.4812909747525698e-05, 0.00036377062968501693], "train_loss": 2.7059366703033447, "total": 2.705936699806647, "contrastive_loss": 2.5739331707906565, "triplet_loss": 0.26045150501672243, "val_score": 0.8276442307692308, "best_score": 0.8276442307692308, "new_best_model": true, "recall": 0.8276442307692308, "mAP": 0.31551935424721883, "mRP": 0.3047799145299148}, "9": {"lr": [1.3435661446562005e-05, 0.0003275997400965494], "train_loss": 2.595609188079834, "total": 2.5956092247596154, "contrastive_loss": 2.4664006568117682, "triplet_loss": 0.25668896321070234, "val_score": 0.8261858974358974, "best_score": 0.8276442307692308, "new_best_model": false, "recall": 0.8261858974358974, "mAP": 0.3137598860822817, "mRP": 0.3043637820512822}, "10": {"lr": [1.1986127417882198e-05, 0.00028953039902753766], "train_loss": 2.4035379886627197, "total": 2.4035379161005435, "contrastive_loss": 2.2831286172005645, "triplet_loss": 0.24289297658862877, "val_score": 0.8341025641025641, "best_score": 0.8341025641025641, "new_best_model": true, "recall": 0.8341025641025641, "mAP": 0.31814129485483644, "mRP": 0.30650267094017103}, "11": {"lr": [1.0500000000000003e-05, 0.0002505], "train_loss": 2.353267192840576, "total": 2.3532671976248953, "contrastive_loss": 2.233728925519962, "triplet_loss": 0.2418478260869565, "val_score": 0.8159775641025641, "best_score": 0.8341025641025641, "new_best_model": false, "recall": 0.8159775641025641, "mAP": 0.3084219487688237, "mRP": 0.29757905982905997}, "12": {"lr": [9.013872582117811e-06, 0.00021146960097246246], "train_loss": 2.196225881576538, "total": 2.196225781903219, "contrastive_loss": 2.0846955672554346, "triplet_loss": 0.2234531772575251, "val_score": 0.8196153846153846, "best_score": 0.8341025641025641, "new_best_model": false, "recall": 0.8196153846153846, "mAP": 0.31561128557013973, "mRP": 0.3008741987179487}, "13": {"lr": [7.564338553438001e-06, 0.00017340025990345064], "train_loss": 2.1827356815338135, "total": 2.1827355834552677, "contrastive_loss": 2.070810375404996, "triplet_loss": 0.22596153846153846, "val_score": 0.7988942307692307, "best_score": 0.8341025641025641, "new_best_model": false, "recall": 0.7988942307692307, "mAP": 0.3017346372184914, "mRP": 0.29122836538461544}, "14": {"lr": [6.1870902524743065e-06, 0.00013722937031498307], "train_loss": 2.04959774017334, "total": 2.0495976987092392, "contrastive_loss": 1.9451228623406145, "triplet_loss": 0.2091346153846154, "val_score": 0.8045192307692307, "best_score": 0.8341025641025641, "new_best_model": false, "recall": 0.8045192307692307, "mAP": 0.3062614875864873, "mRP": 0.29303044871794875}, "15": {"lr": [4.916040103221507e-06, 0.00010384757955302797], "train_loss": 2.0831048488616943, "total": 2.083104774704745, "contrastive_loss": 1.9759256114130435, "triplet_loss": 0.2148829431438127, "val_score": 0.7813942307692308, "best_score": 0.8341025641025641, "new_best_model": false, "recall": 0.7813942307692308, "mAP": 0.2919516331951227, "mRP": 0.2804193376068376}, "16": {"lr": [3.7824855787278e-06, 7.40768580939564e-05], "train_loss": 1.9813275337219238, "total": 1.9813275289376044, "contrastive_loss": 1.8798632159280937, "triplet_loss": 0.20192307692307693, "val_score": 0.7799358974358974, "best_score": 0.8341025641025641, "new_best_model": false, "recall": 0.7799358974358974, "mAP": 0.2952255528422196, "mRP": 0.28285336538461536}, "17": {"lr": [2.814338553438001e-06, 4.865025990345063e-05], "train_loss": 2.056546211242676, "total": 2.0565461123667435, "contrastive_loss": 1.950353309860995, "triplet_loss": 0.21195652173913043, "val_score": 0.7576602564102565, "best_score": 0.8341025641025641, "new_best_model": false, "recall": 0.7576602564102565, "mAP": 0.2823129175993759, "mRP": 0.27259561965811974}, "18": {"lr": [2.0354380202105066e-06, 2.8193872215002235e-05], "train_loss": 1.9893100261688232, "total": 1.9893100700250836, "contrastive_loss": 1.8868479648960075, "triplet_loss": 0.20464046822742474, "val_score": 0.7579807692307693, "best_score": 0.8341025641025641, "new_best_model": false, "recall": 0.7579807692307693, "mAP": 0.2839630115062406, "mRP": 0.2736477029914531}}

11_12_13_lr_17/r1s/11_12_13_lr_17_s26092004_f0_r1_vs0.83217_ema.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d08d0e413bbc0a5b6fc1b17c2eacdf18cb9888728858c713443f0b5c39dd3c39
+size 544008938

11_12_13_lr_17/r1s/11_12_13_lr_17_s26092004_f1_r1_vs0.83628_ema.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:669e5bc066647fcbe01cde1fc0f4e238eebfc7144825b5dd700edebfd078789e
+size 544008938

11_12_13_lr_17/r1s/11_12_13_lr_17_s26092004_f2_r1_vs0.84199_ema.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e7da7b0667f4cc5e86af4c9586f0c26ccba9136d5b04a5d3856b4ceccb046e8
+size 544008938

11_12_13_lr_17/r1s/11_12_13_lr_17_s26092004_f3_r1_vs0.84439_ema.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:197756e86c7d9b8ab126c812d689ebac7752b7478b7b46fa0c3e14a79890c359
+size 544008938

11_12_13_lr_17/r1s/11_12_13_lr_17_s26092004_f4_r1_vs0.83410_ema.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11e475b8653192cd7b806bf6330a83a3bc1dea7e1ccfaf9ff0dc7506467fadc8
+size 544008938

11_12_13_lr_17/results/11_12_13_lr_17_test.json

@@ -0,0 +1,176 @@
+{
+  "Best model": {
+    "0": {
+      "5": {
+        "recall": 0.7420933734939759,
+        "mAP": 0.3063965235943786,
+        "mRP": 0.31039784136546333
+      },
+      "10": {
+        "recall": 0.829066265060241,
+        "mAP": 0.32095892996589587,
+        "mRP": 0.31039784136546333
+      },
+      "15": {
+        "recall": 0.8708584337349398,
+        "mAP": 0.32718328062360497,
+        "mRP": 0.31039784136546333
+      }
+    },
+    "0.25": {
+      "5": {
+        "recall": 0.7948042168674698,
+        "mAP": 0.3253631191432417,
+        "mRP": 0.3285642570281146
+      },
+      "10": {
+        "recall": 0.8708584337349398,
+        "mAP": 0.341724170989515,
+        "mRP": 0.3285642570281146
+      },
+      "15": {
+        "recall": 0.9002259036144579,
+        "mAP": 0.34808647386819574,
+        "mRP": 0.3285642570281146
+      }
+    },
+    "0.5": {
+      "5": {
+        "recall": 0.7654367469879518,
+        "mAP": 0.30770310408300017,
+        "mRP": 0.312305471887552
+      },
+      "10": {
+        "recall": 0.8411144578313253,
+        "mAP": 0.3227786269084286,
+        "mRP": 0.312305471887552
+      },
+      "15": {
+        "recall": 0.8765060240963856,
+        "mAP": 0.3285299822089177,
+        "mRP": 0.312305471887552
+      }
+    },
+    "0.75": {
+      "5": {
+        "recall": 0.7473644578313253,
+        "mAP": 0.2939539407630535,
+        "mRP": 0.2997615461847405
+      },
+      "10": {
+        "recall": 0.827183734939759,
+        "mAP": 0.3087747418244413,
+        "mRP": 0.2997615461847405
+      },
+      "15": {
+        "recall": 0.8625753012048193,
+        "mAP": 0.31438866103775803,
+        "mRP": 0.2997615461847405
+      }
+    },
+    "1": {
+      "5": {
+        "recall": 0.7379518072289156,
+        "mAP": 0.28719157881526197,
+        "mRP": 0.2915097891566277
+      },
+      "10": {
+        "recall": 0.8181475903614458,
+        "mAP": 0.3014128488158991,
+        "mRP": 0.2915097891566277
+      },
+      "15": {
+        "recall": 0.853539156626506,
+        "mAP": 0.3070904997919313,
+        "mRP": 0.2915097891566277
+      }
+    }
+  },
+  "Last model": {
+    "0": {
+      "5": {
+        "recall": 0.704066265060241,
+        "mAP": 0.28999027359437884,
+        "mRP": 0.2899096385542182
+      },
+      "10": {
+        "recall": 0.7921686746987951,
+        "mAP": 0.30397970353636844,
+        "mRP": 0.2899096385542182
+      },
+      "15": {
+        "recall": 0.829066265060241,
+        "mAP": 0.3085009147339727,
+        "mRP": 0.2899096385542182
+      }
+    },
+    "0.25": {
+      "5": {
+        "recall": 0.7891566265060241,
+        "mAP": 0.32182270749665487,
+        "mRP": 0.32599774096385736
+      },
+      "10": {
+        "recall": 0.8610692771084337,
+        "mAP": 0.33729405967122533,
+        "mRP": 0.32599774096385736
+      },
+      "15": {
+        "recall": 0.8953313253012049,
+        "mAP": 0.34405000567886546,
+        "mRP": 0.32599774096385736
+      }
+    },
+    "0.5": {
+      "5": {
+        "recall": 0.7579066265060241,
+        "mAP": 0.30209515143909127,
+        "mRP": 0.3060805722891582
+      },
+      "10": {
+        "recall": 0.8324548192771084,
+        "mAP": 0.3166018986660941,
+        "mRP": 0.3060805722891582
+      },
+      "15": {
+        "recall": 0.8648343373493976,
+        "mAP": 0.32223763195054717,
+        "mRP": 0.3060805722891582
+      }
+    },
+    "0.75": {
+      "5": {
+        "recall": 0.7409638554216867,
+        "mAP": 0.2905525225903625,
+        "mRP": 0.2963541666666681
+      },
+      "10": {
+        "recall": 0.8207831325301205,
+        "mAP": 0.3051159424403973,
+        "mRP": 0.2963541666666681
+      },
+      "15": {
+        "recall": 0.8550451807228916,
+        "mAP": 0.3104866205407441,
+        "mRP": 0.2963541666666681
+      }
+    },
+    "1": {
+      "5": {
+        "recall": 0.7243975903614458,
+        "mAP": 0.28203177292503423,
+        "mRP": 0.2882906626506035
+      },
+      "10": {
+        "recall": 0.8132530120481928,
+        "mAP": 0.2967050219728128,
+        "mRP": 0.2882906626506035
+      },
+      "15": {
+        "recall": 0.8460090361445783,
+        "mAP": 0.3021089467169105,
+        "mRP": 0.2882906626506035
+      }
+    }
+  }
+}