Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,16 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+dataset/_train.xlsx filter=lfs diff=lfs merge=lfs -text
+dataset/test_.xlsx filter=lfs diff=lfs merge=lfs -text
+dataset/train.xlsx filter=lfs diff=lfs merge=lfs -text
+gradcam/KKLFKKILKYL-temp.png filter=lfs diff=lfs merge=lfs -text
+gradcam/KKLFKKiLKYL-diff.png filter=lfs diff=lfs merge=lfs -text
+gradcam/KKLFKKiLKYL-muta.png filter=lfs diff=lfs merge=lfs -text
+gradcam/KWKIKWPVKWFKML-temp.png filter=lfs diff=lfs merge=lfs -text
+gradcam/KWKIKWPVKWfKML-diff.png filter=lfs diff=lfs merge=lfs -text
+gradcam/KWKIKWPVKWfKML-muta.png filter=lfs diff=lfs merge=lfs -text
+vis/tsne_highlight.png filter=lfs diff=lfs merge=lfs -text
+vis/tsne_pointcloud.png filter=lfs diff=lfs merge=lfs -text
+vis/umap_before.png filter=lfs diff=lfs merge=lfs -text
+vis/umap_highlight.png filter=lfs diff=lfs merge=lfs -text

ImageMolEncoder.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85eebbe81192401d0b4337f89e0eea507092c396909ff83bd6b569fd89d49750
+size 44782591

README.md

@@ -0,0 +1,55 @@
+# AI-based D-amino acid substitution for optimizing antimicrobial peptides to treat multidrug-resistant bacterial infection
+This repository contains the code for the paper "AI-based D-amino acid substitution for optimizing antimicrobial peptides to treat multidrug-resistant bacterial infection"
+
+## Requirements
+```
+mamba_ssm==2.2.4
+numpy==1.26.3
+pandas==2.1.4
+rdkit==2024.3.5
+scikit_learn==1.4.1.post1
+scipy==1.13.0
+torch==2.2.0
+torchmetrics==1.3.1
+torchvision==0.17.0
+```
+You can install them with `pip install -r requirements.txt`
+
+Additionally, `mamba_ssm` is optional since it is not used for our final method.
+You can comment `mamba_ssm==2.2.4` in `requirements.txt` and `from mamba_ssm import Mamba` in `network.py` out if you don't want to install it and avoid use `--q-encoder mamba`.
+
+## Training
+There are two .py file for training: `main.py` and `main_simple.py`.
+
+`main.py`: Can train model with Classification and Regression tasks. Prefered with regression task.
+
+`main_simple.py`: Can ONLY train model with Classification task. Prefered with classification task. `simple` means a simple dataset that direct loads pre-processed data.
+
+example: 
+```
+python main-simple.py \
+    --q-encoder cnn \ # Encoder, can be cnn, lstm, gru, mamba, mha
+    --channels 16 \ # Encoder channels
+    --side-enc lstm \ # Side sequence Encoder, only lstm implemented, only use with cnn encoder
+    --fusion att \ # Fusion method, can be att, mlp or diff
+    --task cls \ # Task, can be cls or reg
+    --loss ce \ # Loss, can be ce or mse, some other losses can be found in code
+    --batch-size 32 \ # Batch size
+    --epochs 35 \ # Epochs
+    --gpu 0 \ # GPU index to use, -1 for cpu
+# ===CNN only options=== \
+    --pcs \ # Enable protease cleavage site dyeing for input pictures
+    --resize 768 \ # Resize input pictures, can be 1 or 2 numbers like 768 or 768 512
+# ===main_simple.py only options=== \
+    --llm-data # Use LLM augmented training data
+```
+Corresponding model weight checkpoints will be saved in the subdirectory of `run-cls` or `run-reg`, e.g. `/run-cls/cnn-att-16-lstm-pcs-simple-llm-768-oneway-ce-32-0.001-35/`
+
+For more arguments, please refer to the code of `main.py` or `main_simple.py`
+
+## Inference
+You can simple replace `main.py` with `infer.py` in your training command to do inference. Remember to add `--simple` if you used checkpoints trained from `main_simple.py`
+
+For case study scanning, please use `infer_case.py` with an additional argument `--case r2` or `--case YOUR_PEPTIDE_SEQUENCE`
+
+Inference results will be saved in the weights directory in `csv` format, e.g. `/run-cls/cnn-att-16-lstm-pcs-simple-llm-768-oneway-ce-32-0.001-35/preds_test.csv`

dataset.py

@@ -0,0 +1,865 @@
+import pandas as pd
+import numpy as np
+import itertools
+import torch
+from torch.utils.data import Dataset
+import re
+import json
+from typing import Literal
+import os
+# import io
+from rdkit import Chem
+from rdkit.Chem import AllChem
+from rdkit.Chem.Draw import rdMolDraw2D
+# from PIL import Image
+import torchvision.io as tvio
+# import torchvision.transforms as tvt
+import torchvision.transforms.v2.functional as tvtF
+
+# --- 辅助函数 ---
+
+# 定义20种常见氨基酸字母（按字母顺序）
+AMINO_ACIDS = ['A', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'K', 'L', 
+               'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'V', 'W', 'Y']
+AA_to_index = {aa: i for i, aa in enumerate(AMINO_ACIDS)}
+valid_aa = set(AMINO_ACIDS)
+
+def is_valid_sequence(seq):
+    """
+    判断序列是否只包含标准氨基酸字符（允许大写或小写，
+    对于小写表示 D 型氨基酸也视为合法）
+    """
+    for ch in seq:
+        if not ch.isalpha():
+            return False
+        if ch.upper() not in valid_aa:
+            return False
+    return True
+
+def parse_mic(mic_str):
+    """
+    解析 MIC 数据，支持以下几种格式：
+      1. 数字，例如 "5" -> 5.0 
+      2. ">{数字}" 或 "≥{数字}"（例如 ">4" 或 "≥ 4"）→ 数值乘以 1.5
+      3. 平均值±标准差，例如 "3.2 ± 0.4" → 取平均值 3.2
+      4. 范围形式，例如 "2.0 - 4.0" → (2.0 + 4.0)/2
+
+    注：符号与数字之间可能存在空格，大于等于符号为 "≥" 而非 ">="
+    """
+    if not isinstance(mic_str, str):
+        return float(mic_str)
+    
+    mic_str = mic_str.strip()
+    mic_str = re.sub(r'\s+', '', mic_str)
+    
+    # 匹配纯数字
+    if re.fullmatch(r'\d+(\.\d+)?', mic_str):
+        return float(mic_str)
+    
+    # 匹配 >{数字} 或 ≥{数字}
+    m = re.fullmatch(r'[>≥](\d+(\.\d+)?)', mic_str)
+    if m:
+        num = float(m.group(1))
+        return num * 1.5
+    
+    # 匹配 <{数字} 或 ≤{数字}
+    m = re.fullmatch(r'[<≤](\d+(\.\d+)?)', mic_str)
+    if m:
+        num = float(m.group(1))
+        return num * 0.75
+    
+    # 匹配 {数字}±{数字}
+    m = re.fullmatch(r'(\d+(\.\d+)?)[±](\d+(\.\d+)?)', mic_str)
+    if m:
+        return float(m.group(1))
+    
+    # 匹配 {数字}-{数字}
+    m = re.fullmatch(r'(\d+(\.\d+)?)-(\d+(\.\d+)?)', mic_str)
+    if m:
+        num1 = float(m.group(1))
+        num2 = float(m.group(3))
+        return (num1 + num2) / 2.0
+    
+    print(f"Warning: 无法解析 MIC 值 {mic_str}")
+    return np.nan
+
+def encode_sequence(seq, pad_length):
+    """
+    将多肽序列转换为固定大小 (pad_length, 21) 的张量：
+      - 每个残基对应一行；
+      - 第1列: 表示是否为 D 型氨基酸（若字符为小写，则置 1，否则为 0）；
+      - 后20列: 20种常见氨基酸的独热编码（先转为大写匹配）。
+    若序列长度小于 pad_length，则在末尾填充全 0 行。
+    """
+    n = len(seq)
+    arr = np.zeros((pad_length, 21), dtype=np.float32)
+    
+    # 对实际序列部分进行编码
+    for i, char in enumerate(seq):
+        if i >= pad_length:
+            break  # 超出部分不处理（数据集构造时已过滤掉长序列）
+        if char.islower():
+            d_indicator = 1.0
+            aa = char.upper()
+        else:
+            d_indicator = 0.0
+            aa = char
+        arr[i, 0] = d_indicator
+        if aa in AA_to_index:
+            idx = AA_to_index[aa]
+            arr[i, idx + 1] = 1.0
+        else:
+            print(f"Warning: 氨基酸 {aa} 不在标准列表中")
+    return torch.tensor(arr)
+
+def geometric_mean(values):
+    """
+    计算数值序列的几何平均值
+    """
+    log_vals = np.log(np.array(values))
+    return float(np.exp(log_vals.mean()))
+
+def process_label(ratio, task):
+    """
+    对比值 ratio 进行 log2 变换，并根据 task 参数返回最终标签：
+     - task="reg": 返回 log₂比值，并转换为 np.float32；
+     - task="cls": 根据 log₂比值进行分类：
+                  如果 x <= -0.5 返回 1，
+                  否则返回 0.
+    若 ratio 非正，返回 np.nan。
+    """
+    if ratio <= 0:
+        return np.nan
+    ratio_log = np.log2(ratio)
+    if task == "reg":
+        return np.float32(ratio_log)
+    elif task == "cls":
+        if ratio_log < 0.:
+            return 1
+        else:
+            return 0
+    else:
+        raise ValueError("未知的 task 类型，请使用 'reg' 或 'cls'")
+
+# --- 数据预处理与构建数据集 ---
+
+def load_data(xlsx_file, condition=None):
+    """
+    从 xlsx 文件中读取数据，将每个具体变种（同一原型-变种）对应的 MIC 值取几何平均，
+    并按照原型分组。对于原型和变种序列，若存在非标准氨基酸或非字母字符，则过滤掉该行数据。
+
+    返回：
+      groups: dict，其中 key 为原型序列，
+              value 为 dict，其 key 为变种序列（"SEQUENCE - D-type amino acid substitution"），
+              value 为该变种所有 MIC 值的几何平均
+    """
+    df = pd.read_excel(xlsx_file)
+    # df = df[df['TARGET ACTIVITY - ACTIVITY MEASURE VALUE'] != 'MBC']
+    
+    groups = {}
+    for _, row in df.iterrows():
+        orig = row["SEQUENCE - Original"]
+        variant = row["SEQUENCE - D-type amino acid substitution"]
+        mic_raw = row["TARGET ACTIVITY - CONCENTRATION"]
+        
+        # 过滤包含非标准氨基酸或非字母字符的序列（原型和变种均检查）
+        if not (isinstance(orig, str) and is_valid_sequence(orig)):
+            continue
+        if not (isinstance(variant, str) and is_valid_sequence(variant)):
+            continue
+        
+        mic_val = parse_mic(mic_raw)
+        
+        if orig not in groups:
+            groups[orig] = {}
+        if variant not in groups[orig]:
+            groups[orig][variant] = []
+        groups[orig][variant].append(mic_val)
+    
+    # 对每个变种计算几何平均（过滤掉 NaN 值）
+    groups_avg = {}
+    for orig, var_dict in groups.items():
+        groups_avg[orig] = {}
+        for variant, mic_list in var_dict.items():
+            mic_list = [x for x in mic_list if not np.isnan(x)]
+            if len(mic_list) == 0:
+                continue
+            groups_avg[orig][variant] = geometric_mean(mic_list)
+    return groups_avg
+
+
+def load_data_stability(xlsx_file, condition):
+    """
+    从 xlsx 文件中读取数据，将每个具体变种（同一原型-变种）对应的 MIC 值取几何平均，
+    并按照原型分组。对于原型和变种序列，若存在非标准氨基酸或非字母字符，则过滤掉该行数据。
+
+    返回：
+      groups: dict，其中 key 为原型序列，
+              value 为 dict，其 key 为变种序列（"SEQUENCE - D-type amino acid substitution"），
+              value 为该变种所有 MIC 值的几何平均
+    """
+    map_dict = {
+        '125fbs': '12.5% FBS',
+        '25fbs': '25% FBS',
+        'mhb': 'MHB',
+        'nacl': '150mM NaCl'
+    }
+    df = pd.read_excel(xlsx_file)
+    df = df[df['Condition'] == map_dict[condition]]
+    
+    groups = {}
+    for _, row in df.iterrows():
+        variant = row["SEQUENCE"]
+        orig = variant.upper()
+        mic_raw = row["Activity"]
+        
+        # 过滤包含非标准氨基酸或非字母字符的序列（原型和变种均检查）
+        if not (isinstance(orig, str) and is_valid_sequence(orig)):
+            continue
+        if not (isinstance(variant, str) and is_valid_sequence(variant)):
+            continue
+        
+        mic_val = parse_mic(mic_raw)
+        
+        if orig not in groups:
+            groups[orig] = {}
+        if variant not in groups[orig]:
+            groups[orig][variant] = []
+        groups[orig][variant].append(mic_val)
+    
+    # 对每个变种计算几何平均（过滤掉 NaN 值）
+    groups_avg = {}
+    for orig, var_dict in groups.items():
+        groups_avg[orig] = {}
+        for variant, mic_list in var_dict.items():
+            mic_list = [x for x in mic_list if not np.isnan(x)]
+            if len(mic_list) == 0:
+                continue
+            groups_avg[orig][variant] = geometric_mean(mic_list)
+    return groups_avg
+
+class PeptidePairDataset(Dataset):
+    def __init__(self, mode=Literal['train', 'test', '125fbs', 'nacl', '25fbs', 'mhb'], pad_length=30, task="cls", 
+                 include_reverse=False, include_self=False, one_way=False, gf=False) :
+        """
+        构建数据集：
+          - 从 xlsx 文件中读取数据，并按照原型分组，
+            同时过滤包含非标准氨基酸或非字母字符的行，以及变种序列长度超过 pad_length 的样本；
+          - 对于同一原型下不同变种构成配对；
+          - 参数 include_reverse: 是否启用正反组合（同时添加 (A, B) 和 (B, A)）；
+          - 参数 include_self: 是否启用自组合（添加 (A, A)，标签为 log₂(1)=0）；
+          - 参数 task: "reg" 表示回归任务（输出 32 位浮点数标签），"cls" 表示分类任务，
+                      将 log₂比值转为整数标签，规则为：
+                          log₂比值 ≤ -0.5 → 1，
+                          log₂比值 ≥ 0.5 → 2，
+                          -0.5 < log₂比值 < 0.5 → 0.
+                          
+        每个数据项返回：
+          - 变种多肽序列编码后的张量，形状为 (pad_length, 21)
+          - 另一个变种多肽序列编码后的张量，形状为 (pad_length, 21)
+          - 标签：根据 task 不同分别为 32 位浮点数或整数
+        """
+        if mode == "train":
+            loader = load_data
+            xlsx_file = os.path.join(os.path.dirname(__file__), 'dataset', 'train.xlsx')
+        elif mode in ["test", "r2_case", 'r2_case_', "125fbs", "nacl", "25fbs", "mhb"]:
+            one_way = True
+            if mode in ["test", "r2_case", 'r2_case_']:
+                loader = load_data
+                xlsx_file = os.path.join(os.path.dirname(__file__), 'dataset', f'{mode}.xlsx')
+            else:
+                loader = load_data_stability
+                xlsx_file = os.path.join(os.path.dirname(__file__), 'dataset', 'stability.xlsx')
+        else:
+            raise ValueError("未知的 mode，请使用 'train' 或 'test'")
+
+        self.data = []
+        self.pad_length = pad_length
+        self.task = task
+        groups_avg = loader(xlsx_file, mode)
+        if gf:
+            gf_dict = torch.load(os.path.join(os.path.dirname(__file__), 'dataset', 'protbert.pth'))
+        
+        # 针对每个原型，过滤掉长度超过 pad_length 的变种
+        for orig, variant_dict in groups_avg.items():
+            # a = len(self.data)
+            filtered_variants = {variant: mic for variant, mic in variant_dict.items() 
+                                 if len(variant) <= pad_length}
+            variants = list(filtered_variants.keys())
+            n_variants = len(variants)
+            if n_variants == 0:
+                continue
+            
+            if gf:
+                glob_feat = gf_dict[orig.upper()]
+
+            # 若启用自组合，则添加 (A, A) 样本，标签为 process_label(1, task) → log2(1)=0（再分类也为 0）
+            if include_self and (not one_way):
+                for variant in variants:
+                    encoded_seq = encode_sequence(variant, pad_length)
+                    label = process_label(1.0, task)  # log2(1)=0
+                    if gf:
+                        self.data.append(((encoded_seq, encoded_seq, glob_feat), label))
+                    else:
+                        self.data.append(((encoded_seq, encoded_seq), label))
+            
+            # 添加不同变种之间的样本
+            for i in [0] if one_way else range(n_variants):
+                for j in range(i + 1, n_variants):
+                    var1 = variants[i]
+                    var2 = variants[j]
+                    mic1 = filtered_variants[var1]
+                    mic2 = filtered_variants[var2]
+                    
+                    # 正向组合: (var1, var2) 标签为 log₂(mic2/mic1)
+                    ratio = mic2 / mic1 if mic1 != 0 else np.nan
+                    label = process_label(ratio, task)
+                    if np.isnan(label):
+                        continue
+                    encoded_var1 = encode_sequence(var1, pad_length)
+                    encoded_var2 = encode_sequence(var2, pad_length)
+                    if gf:
+                        self.data.append(((encoded_var1, encoded_var2, glob_feat), label))
+                    else:
+                        self.data.append(((encoded_var1, encoded_var2), label))
+                    
+                    # 若启用正反组合，则添加 (var2, var1)
+                    if include_reverse and (not one_way):
+                        rev_ratio = mic1 / mic2 if mic2 != 0 else np.nan
+                        rev_label = process_label(rev_ratio, task)
+                        if gf:
+                            self.data.append(((encoded_var2, encoded_var1, glob_feat), rev_label))
+                        else:
+                            self.data.append(((encoded_var2, encoded_var1), rev_label))
+            # b = len(self.data)
+            # print(f"{orig},{b - a}")
+    
+    def reg_sample_weight(self):
+        y = []
+        for _, label in self.data:
+            y.append(label)
+        y = np.array(y)
+        mu = np.mean(y)
+        sigma = np.std(y)
+        p = 1 / (sigma * np.sqrt(2 * np.pi)) * np.exp(-((y - mu) ** 2) / (2 * sigma ** 2))
+        
+        # 如果未提供 C，则使用 p 的中位数作为基准常数
+        C = np.median(p)
+        epsilon = 1e-6
+        
+        # 使用对数转化计算采样权重： p 值越低权重越高
+        weights = np.log(C / (p + epsilon))
+        
+        # 可选：对权重进行归一化处理，使得权重均值为1
+        weights_normalized = weights / np.mean(weights)
+        positive_weights = np.exp(weights_normalized)
+        
+        return torch.tensor(positive_weights, dtype=torch.float32)
+    
+    def __len__(self):
+        return len(self.data)
+    
+    def __getitem__(self, idx):
+        return self.data[idx]
+
+
+class PeptidePairPicDataset(Dataset):
+    def __init__(self, mode=Literal['train', 'test', '125fbs', 'nacl', '25fbs', 'mhb'], pad_length=30, task="reg", 
+                 include_reverse=False, include_self=False, one_way=False, gf=False,
+                 side_enc=None, pcs=False, resize=None) :
+        """
+        构建数据集：
+          - 从 xlsx 文件中读取数据，并按照原型分组，
+            同时过滤包含非标准氨基酸或非字母字符的行，以及变种序列长度超过 pad_length 的样本；
+          - 对于同一原型下不同变种构成配对；
+          - 参数 include_reverse: 是否启用正反组合（同时添加 (A, B) 和 (B, A)）；
+          - 参数 include_self: 是否启用自组合（添加 (A, A)，标签为 log₂(1)=0）；
+          - 参数 task: "reg" 表示回归任务（输出 32 位浮点数标签），"cls" 表示分类任务，
+                      将 log₂比值转为整数标签，规则为：
+                          log₂比值 ≤ -0.5 → 1，
+                          log₂比值 ≥ 0.5 → 2，
+                          -0.5 < log₂比值 < 0.5 → 0.
+                          
+        每个数据项返回：
+          - 变种多肽序列编码后的张量，形状为 (pad_length, 21)
+          - 另一个变种多肽序列编码后的张量，形状为 (pad_length, 21)
+          - 标签：根据 task 不同分别为 32 位浮点数或整数
+        """
+        if mode == "train":
+            loader = load_data
+            xlsx_file = os.path.join(os.path.dirname(__file__), 'dataset', 'train.xlsx')
+        elif mode in ["test", "r2_case", 'r2_case_', "125fbs", "nacl", "25fbs", "mhb"]:
+            one_way = True
+            if mode in ["test", "r2_case", 'r2_case_']:
+                loader = load_data
+                xlsx_file = os.path.join(os.path.dirname(__file__), 'dataset', f'{mode}.xlsx')
+            else:
+                loader = load_data_stability
+                xlsx_file = os.path.join(os.path.dirname(__file__), 'dataset', 'stability.xlsx')
+        else:
+            raise ValueError("未知的 mode，请使用 'train' 或 'test'")
+
+        self.data = []
+        self.pics = {}
+        self.pad_length = pad_length
+        self.task = task
+        self.gf = gf
+        self.side_enc = True if side_enc else False
+        self.pcs = pcs
+        self.resize = resize
+        groups_avg = loader(xlsx_file, mode)
+        if gf:
+            gf_dict = torch.load(os.path.join(os.path.dirname(__file__), 'dataset', 'protbert.pth'))
+        
+        # 针对每个原型，过滤掉长度超过 pad_length 的变种
+        for orig, variant_dict in groups_avg.items():
+            # a = len(self.data)
+            filtered_variants = {variant: mic for variant, mic in variant_dict.items() 
+                                 if len(variant) <= pad_length}
+            variants = list(filtered_variants.keys())
+            for variant in variants:
+                if self.pcs == 'mix' and variant == orig:
+                    self.pics[variant] = self.read_img(variant, False)
+                else:
+                    self.pics[variant] = self.read_img(variant, self.pcs)
+            n_variants = len(variants)
+            if n_variants == 0:
+                continue
+            
+            if gf:
+                glob_feat = gf_dict[orig.upper()]
+
+            # 若启用自组合，则添加 (A, A) 样本，标签为 process_label(1, task) → log2(1)=0（再分类也为 0）
+            if include_self and (not one_way):
+                for variant in variants:
+                    label = process_label(1.0, task)  # log2(1)=0
+                    if gf:
+                        self.data.append((variant, variant, glob_feat, label))
+                    else:
+                        self.data.append((variant, variant, label))
+            
+            # 添加不同变种之间的样本
+            for i in [0] if one_way else range(n_variants):
+                for j in range(i + 1, n_variants):
+                    var1 = variants[i]
+                    var2 = variants[j]
+                    mic1 = filtered_variants[var1]
+                    mic2 = filtered_variants[var2]
+                    
+                    # 正向组合: (var1, var2) 标签为 log₂(mic2/mic1)
+                    ratio = mic2 / mic1 if mic1 != 0 else np.nan
+                    label = process_label(ratio, task)
+                    if np.isnan(label):
+                        continue
+                    if gf:
+                        self.data.append((var1, var2, glob_feat, label))
+                    else:
+                        self.data.append((var1, var2, label))
+                    
+                    # 若启用正反组合，则添加 (var2, var1)
+                    if include_reverse and (not one_way):
+                        rev_ratio = mic1 / mic2 if mic2 != 0 else np.nan
+                        rev_label = process_label(rev_ratio, task)
+                        if gf:
+                            self.data.append((var2, var1, glob_feat, rev_label))
+                        else:
+                            self.data.append((var2, var1, rev_label))
+            # b = len(self.data)
+            # print(f"{orig},{b - a}")
+    
+    def reg_sample_weight(self):
+        y = []
+        for d in self.data:
+            label = d[-1]
+            y.append(label)
+        y = np.array(y)
+        mu = np.mean(y)
+        sigma = np.std(y)
+        p = 1 / (sigma * np.sqrt(2 * np.pi)) * np.exp(-((y - mu) ** 2) / (2 * sigma ** 2))
+        
+        # 如果未提供 C，则使用 p 的中位数作为基准常数
+        C = np.median(p)
+        epsilon = 1e-6
+        
+        # 使用对数转化计算���样权重： p 值越低权重越高
+        weights = np.log(C / (p + epsilon))
+        
+        # 可选：对权重进行归一化处理，使得权重均值为1
+        weights_normalized = weights / np.mean(weights)
+        positive_weights = np.exp(weights_normalized)
+        
+        return torch.tensor(positive_weights, dtype=torch.float32)
+    
+    def read_img(self, peptide, pcs):
+        image = draw_peptide(peptide, self.resize, pcs)
+        return image
+    
+    def __len__(self):
+        return len(self.data)
+    
+    def __getitem__(self, idx):
+        if self.gf:
+            seq1, seq2, glob_feat, label = self.data[idx]
+        else:
+            seq1, seq2, label = self.data[idx]
+        img1 = self.pics[seq1]
+        img2 = self.pics[seq2]
+
+        if self.side_enc:
+            img1 = (img1, encode_sequence(seq1, self.pad_length))
+            img2 = (img2, encode_sequence(seq2, self.pad_length))
+
+        if self.gf:
+            return (img1, img2, glob_feat), label
+        else:
+            return (img1, img2), label
+        
+
+class SimplePairClsDataset(Dataset):
+    def __init__(self, pad_length=30, llm=False, ftr2=False, gf=False,
+                 q_encoder=None, side_enc=None, pcs=False, resize=None):
+        if llm:
+            file_path = os.path.join(os.path.dirname(__file__), 'dataset', 'train_set_llm_aug.json')
+        elif ftr2:
+            file_path = os.path.join(os.path.dirname(__file__), 'dataset', 'finetune_for_r2_llm.json')
+        else:
+            file_path = os.path.join(os.path.dirname(__file__), 'dataset', 'train_set.json')
+        with open(file_path, 'r', encoding='utf-8') as f:
+            dataset = json.load(f)   
+
+        self.data = []
+        self.pics = {}
+        self.pad_length = pad_length
+        self.gf = gf
+        self.q_encoder = q_encoder
+        self.side_enc = True if side_enc else False
+        self.pcs = pcs
+        self.resize = resize
+        if gf:
+            self.gf_dict = torch.load(os.path.join(os.path.dirname(__file__), 'dataset', 'protbert.pth'))
+        
+        all_seqs = []
+        for orig, variants in dataset.items():
+            if len(orig) > pad_length:
+                continue
+            all_seqs.append(orig)
+            for label in ["1", "0"]:
+                for variant in variants[label]:
+                    self.data.append((orig, variant, int(label)))
+                    all_seqs.append(variant)
+        if q_encoder in ['cnn', 'rn18']:
+            for i in all_seqs:
+                if self.pcs == 'mix' and i.isupper():
+                    self.pics[i] = self.read_img(i, False)
+                else:
+                    self.pics[i] = self.read_img(i, self.pcs)
+    
+    def read_img(self, peptide, pcs):
+        image = draw_peptide(peptide, self.resize, pcs)
+        return image
+    
+    def __len__(self):
+        return len(self.data)
+    
+    def __getitem__(self, idx):
+        seq1, seq2, label = self.data[idx]
+        if self.q_encoder in ['cnn', 'rn18']:
+            img1 = self.pics[seq1]
+            img2 = self.pics[seq2]
+
+            if self.side_enc:
+                img1 = (img1, encode_sequence(seq1, self.pad_length))
+                img2 = (img2, encode_sequence(seq2, self.pad_length))
+        
+        else:
+            img1 = encode_sequence(seq1, self.pad_length)
+            img2 = encode_sequence(seq2, self.pad_length)
+
+        if self.gf:
+            return (img1, img2, self.gf_dict[seq1]), label
+        else:
+            return (img1, img2), label
+
+
+class PeptidePairCaseDataset(Dataset):
+    def __init__(self, case:str ='r2', pad_length=30, gf=False):
+
+        if case == 'r2':
+            self.template = 'KWKIKWPVKWFKML'
+        elif case == 'Indolicidin':
+            self.template = 'ILPWKWPWWPWRR'
+        elif case == 'Temporin-A':
+            self.template = 'FLPLIGRVLSGIL'
+        elif case == 'Melittin':
+            self.template = 'GIGAVLKVLTTGLPALISWIKRKRQQ'
+        elif case == 'Anoplin':
+            self.template = 'GLLKRIKTLL'
+        else:
+            self.template = case.upper().strip()
+        self.data = []
+        self.pad_length = pad_length
+        self.gf = gf
+
+        if gf:
+            self.glob_feat = torch.load(os.path.join(os.path.dirname(__file__), 'dataset', 'protbert.pth'))[self.template]
+        
+        pools = [(ch.upper(), ch.lower()) if ch != 'G' else (ch.upper(),) for ch in self.template]
+        # 笛卡尔积，即所有组合
+        self.variants = [''.join(chars) for chars in itertools.product(*pools)][1:]
+
+        self.template_seq = encode_sequence(self.template, self.pad_length)
+    
+    def __len__(self):
+        return len(self.variants)
+    
+    def __getitem__(self, idx):
+        variant  = self.variants[idx]
+        seq2, label = variant, variant
+        enc_seq1 = self.template_seq
+        enc_seq2 = encode_sequence(seq2, self.pad_length)
+
+        if self.gf:
+            return (enc_seq1, enc_seq2, self.glob_feat), label
+        else:
+            return (enc_seq1, enc_seq2), label
+
+
+
+class PeptidePairPicCaseDataset(Dataset):
+    def __init__(self, case:str ='r2', pad_length=30, side_enc=None, pcs=False, resize=None, gf=False):
+
+        if case == 'r2':
+            self.template = 'KWKIKWPVKWFKML'
+        elif case == 'Indolicidin':
+            self.template = 'ILPWKWPWWPWRR'
+        elif case == 'Temporin-A':
+            self.template = 'FLPLIGRVLSGIL'
+        elif case == 'Melittin':
+            self.template = 'GIGAVLKVLTTGLPALISWIKRKRQQ'
+        elif case == 'Anoplin':
+            self.template = 'GLLKRIKTLL'
+        else:
+            self.template = case.upper().strip()
+        self.data = []
+        self.pad_length = pad_length
+        self.side_enc = True if side_enc else False
+        self.pcs = pcs
+        self.resize = resize
+        self.gf = gf
+
+        if gf:
+            self.glob_feat = torch.load(os.path.join(os.path.dirname(__file__), 'dataset', 'protbert.pth'))[self.template]
+        
+        pools = [(ch.upper(), ch.lower()) if ch != 'G' else (ch.upper(),) for ch in self.template]
+        # 笛卡尔积，即所有组合
+        self.variants = [''.join(chars) for chars in itertools.product(*pools)][1:]
+
+        self.template_pic = self.read_img(self.template)
+        if self.side_enc:
+            self.template_seq = encode_sequence(self.template, self.pad_length)
+    
+    def read_img(self, peptide):
+        image = draw_peptide(peptide, self.resize, self.pcs)
+        return image
+    
+    def __len__(self):
+        return len(self.variants)
+    
+    def __getitem__(self, idx):
+        variant  = self.variants[idx]
+        seq2, label = variant, variant
+        img1 = self.template_pic
+        img2 = self.read_img(variant)
+
+        if self.side_enc:
+            img1 = (img1, self.template_seq)
+            img2 = (img2, encode_sequence(seq2, self.pad_length))
+
+        if self.gf:
+            return (img1, img2, self.glob_feat), label
+        else:
+            return (img1, img2), label
+
+
+aa_side = {
+    "A": "C", "R": "CCCNC(N)=N", "N": "CC(=O)N", "D": "CC(=O)O", "C": "CS",
+    "E": "CCC(=O)O", "Q": "CCC(=O)N", "G": "", "H": "Cc1cnc[nH]1", "I": "C(C)CC",
+    "L": "CC(C)C", "K": "CCCCN", "M": "CCSC", "F": "Cc1ccccc1", "P": "C1CCN1",
+    "S": "CO", "T": "C(C)O", "W": "Cc1c[nH]c2ccccc12", "Y": "Cc1ccc(O)cc1", "V": "C(C)C"
+}
+
+aa_tpl = {}
+for aa, R in aa_side.items():
+    for stereo, chir in (("L", "@"), ("D", "@@")):
+        if aa == "G":  # Gly 没手性
+            backbone = "N[C:{idx}]C"          # N-CA(带编号)-C
+        else:
+            backbone = f"N[C{chir}H:{'{idx}'}]({R})C"  # N-[C@H:idx](R)-C
+        aa_tpl[f"{aa}_{stereo}"]       = backbone + "(=O)"     # 中间残基
+        aa_tpl[f"{aa}_{stereo}_term"]  = backbone + "(=O)O"    # C 端
+
+def build_peptide_smiles(seq: str) -> str:
+    """
+    给定单字母序列，返回 backbone 带 [atom_map] 的 SMILES。
+    大写 = L 型， 小写 = D 型。编号 = 残基序号（1,2,3...） -> α-碳。
+    """
+    if not seq:
+        return ""
+
+    out = []
+    n = len(seq)
+    for i, aa in enumerate(seq, start=1):
+        key = f"{aa.upper()}_{'L' if aa.isupper() else 'D'}"
+        if i == n:
+            key += "_term"
+        out.append(aa_tpl[key].format(idx=i))
+    return "".join(out)
+
+protease_patterns = {
+    'trypsin':       re.compile(r'(?<=[KR])(?!P)'),
+    'chymotrypsin':  re.compile(r'(?<=[FYWL])(?!P)'),
+    'elastase':      re.compile(r'(?<=[AVSGT])(?!P)'),
+    'enterokinase':  re.compile(r'D{4}K(?=[^P])'),
+    'caspase':       re.compile(r'(?<=D)(?=[GSA])'),
+}
+
+def draw_peptide(sequence, size=[768], pcs=False):
+    """
+    根据输入序列生成多肽结构图，并基于常见蛋白酶识别模式高亮酶切位点肽键（红色）。
+    支持的酶及其正则模式（P1--P1'）：
+      • trypsin:       (?<=[KR])(?!P)
+      • chymotrypsin:  (?<=[FYWL])(?!P)
+      • elastase:      (?<=[AVSGT])(?!P)
+      • enterokinase:  D{4}K(?=[^P])
+      • caspase:       (?<=D)(?=[GSA])
+    """
+
+    # # 1. 生成带 atom map 的 SMILES（现在序号标注在α-碳上）
+    smiles = build_peptide_smiles(sequence)
+    mol = Chem.MolFromSmiles(smiles)
+    # if mol is None:
+    #     raise ValueError("SMILES 解析失败，请检查输入序列和侧链字典。")
+    AllChem.Compute2DCoords(mol)
+
+    highlight_bonds = []
+    bond_colors = {}
+    
+    # ----------------------------------------------------
+    # 2. 先标 D 型残基：高亮与α-碳相连的键为蓝色
+    d_positions = {i for i, aa in enumerate(sequence, start=1) if aa.islower()}
+
+    for atom in mol.GetAtoms():
+        if atom.GetAtomMapNum() in d_positions:
+            # 这个atom就是α-碳，高亮与它相连的所有键
+            for b in atom.GetBonds():
+                idx = b.GetIdx()
+                if idx not in highlight_bonds:
+                    highlight_bonds.append(idx)
+                bond_colors[idx] = (0.0, 0.0, 1.0)
+
+    # ----------------------------------------------------
+    # 3. 再标酶切键：红色（覆盖之前的蓝色）
+    if pcs:
+        cleavage_sites = set()
+        for pat in protease_patterns.values():
+            for m in pat.finditer(sequence):
+                cut = m.end()  # 切在 cut 之后
+                if 1 <= cut < len(sequence):
+                    cleavage_sites.add(cut)
+
+        for pos in cleavage_sites:
+            # 先找 P1 残基的 α-C
+            ca = next((a for a in mol.GetAtoms()
+                       if a.GetAtomMapNum() == pos), None)
+            if ca is None:
+                continue
+
+            # 找同残基的羧基碳 (sp², 含 O 双键)
+            carbonyl_c = None
+            for nb in ca.GetNeighbors():
+                if nb.GetSymbol() != "C":
+                    continue
+                # 判断是否有 "=O"
+                if any(bond.GetBondType() == Chem.BondType.DOUBLE and
+                       o.GetSymbol() == "O"
+                       for bond in nb.GetBonds()
+                       for o in (bond.GetBeginAtom(), bond.GetEndAtom())):
+                    carbonyl_c = nb
+                    break
+            if carbonyl_c is None:
+                continue
+
+            # 羧基碳连到的 N 就是下一残基的氮
+            peptide_bond = None
+            for b in carbonyl_c.GetBonds():
+                o_atom = b.GetOtherAtom(carbonyl_c)
+                if o_atom.GetSymbol() == "N":
+                    peptide_bond = b
+                    break
+            if peptide_bond is None:
+                continue
+
+            bidx = peptide_bond.GetIdx()
+            if bidx not in highlight_bonds:
+                highlight_bonds.append(bidx)
+            bond_colors[bidx] = (1.0, 0.0, 0.0)  # 红
+    
+    # 4. 设置画布大小
+    if len(size) == 1:
+        w = h = size[0]
+    else:
+        w, h = size
+
+    # 5. MolDraw2DCairo 接收 highlightBondColors
+    drawer = rdMolDraw2D.MolDraw2DCairo(w, h)
+    # 你也可以通过 drawer.drawOptions() 调整一些样式：bond line width、atom font 等
+    drawer.DrawMolecule(
+        mol,
+        highlightAtoms=[],
+        highlightBonds=highlight_bonds,
+        highlightAtomColors={},
+        highlightBondColors=bond_colors
+    )
+    drawer.FinishDrawing()
+
+    # 6. 把输出的 PNG bytes 转成 Tensor
+    png_bytes = bytearray(drawer.GetDrawingText())
+    byte_tensor = torch.frombuffer(png_bytes, dtype=torch.uint8)
+    img = tvio.decode_png(byte_tensor, mode=tvio.ImageReadMode.RGB)       # [3, H, W], uint8
+    img = tvtF.to_dtype(img, torch.float32)
+    img = tvtF.normalize(img, [0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
+    return img
+
+if __name__ == '__main__':
+    # 假设 xlsx 文件路径为 "data.xlsx"
+    # 设置 pad_length 为 50，同时启用正反组合和自组合
+    pad_length = 30
+    dataset = PeptidePairDataset('r2_case', pad_length, "cls", include_reverse=False, include_self=False, one_way=True)
+    
+    # 打印第一个数据项
+    if len(dataset) > 0:
+        (encoded_seq1, encoded_seq2), ratio = dataset[0]
+        print("第一个样本：")
+        print("变种1的编码张量形状：", encoded_seq1.shape)
+        print("变种2的编码张量形状：", encoded_seq2.shape)
+        print("标签比值（变种2/变种1）：", ratio)
+        print(f"数据集大小：{len(dataset)}")
+        label_pos = 0
+        for (_, _), i in dataset:
+            label_pos += i
+        print(label_pos)
+
+    else:
+        print("未读入组合数据！")
+
+    # # 测试 PeptidesDataset
+    # pad_length = 30
+    # dataset = PeptidesDataset(xlsx_file="./dataset/train.xlsx", pad_length=pad_length)
+    # print(f"PeptidesDataset 样本总数: {len(dataset)}")
+    # if len(dataset) > 0:
+    #     encoded_seq, label = dataset[0]
+    #     print("第一个样本：")
+    #     print("多肽编码张量形状：", encoded_seq.shape)
+    #     print("标签浓度值（几何平均后）：", label)
+    # else:
+    #     print("未读取到有效数据！")

dataset/_train.xlsx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03bfa373ecd3e21fd68313c0917ba5201985b1453ea8eedcf2e3fe0da8b911eb
+size 150386

dataset/finetune_for_r2_llm copy.json

@@ -0,0 +1,215 @@
+{
+    "KWKIKWPVKWFKML": {
+        "1": [
+            "kwkikwpvkwfkml",
+            "Kwkikwpvkwfkml",
+            "kWkikwpvkwfkml",
+            "kwKikwpvkwfkml",
+            "kwkiKwpvkwfkml",
+            "kwkikWpvkwfkml",
+            "kwkikwPvkwfkml",
+            "kwkikwpVkwfkml",
+            "kwkikwpvKwfkml",
+            "kwkikwpvkWfkml",
+            "kwkikwpvkwFkml",
+            "kwkikwpvkwfKml",
+            "kwkikwpvkwfkMl",
+            "kwkikwpvkwfkmL",
+            "KWkikwpvkwfkml",
+            "KwkIkwpvkwfkml",
+            "KWKikwpvkwfkml",
+            "KWKiKwpvkwfkml",
+            "KWKikWpvkwfkml",
+            "KWKikwPvkwfkml",
+            "KWKikwpVkwfkml",
+            "KWKikwpvKwfkml",
+            "KWKikwpvkWfkml",
+            "KWKikwpvkwFkml",
+            "KWKikwpvkwfKml",
+            "KWKikwpvkwfkMl",
+            "KWKikwpvkwfkmL",
+            "kwKiKwpvkwFkml",
+            "kwKiKwpvkwfKml",
+            "kwKiKwpvkwfkMl",
+            "kwKiKwpvkwfkmL",
+            "kWkIkwpvKwFkml",
+            "kWkIkwpvKwfKml",
+            "kWkIkwpvKwfkMl",
+            "kWkIkwpvKwfkmL",
+            "kWKikWpvkwFkml",
+            "kWKikWpvkwfKml",
+            "kWKikWpvkwfkMl",
+            "kWKikWpvkwfkmL",
+            "kwKikwPvKwFkml",
+            "kwKikwPvKwfKml",
+            "kwKikwPvKwfkMl",
+            "kwKikwPvKwfkmL",
+            "KWKikwpVkwFkml",
+            "KWKikwpVkwfKml",
+            "KWKikwpVkwfkMl",
+            "KWKikwpVkwfkmL",
+            "KWkikwpvkWFkml",
+            "KWkikwpvkWfKml",
+            "KWkikwpvkWfkMl",
+            "KWkikwpvkWfkmL",
+            "kwkikwpvkwfKML",
+            "KWKIKWPVKWFKML",
+            "kwKiKWpVkwfkml",
+            "kWKiKWpVkwfkml",
+            "kwkIKWpVkwfkml",
+            "kWkIKWpVkwfkml",
+            "kwKiKWpVkwfKml",
+            "kWKiKWpVkwfKml",
+            "kwkIKWpVkwfKml",
+            "kWkIKWpVkwfKml",
+            "kwKiKWpVkwfkMl",
+            "kWKiKWpVkwfkMl",
+            "kwkIKWpVkwfkMl",
+            "kWkIKWpVkwfkMl",
+            "kwKiKWpVkwfkmL",
+            "kWKiKWpVkwfkmL",
+            "kwkIKWpVkwfkmL",
+            "kWkIKWpVkwfkmL",
+            "kwKiKWpVkwfKML",
+            "kWKiKWpVkwfKML",
+            "kwkIKWpVkwfKML",
+            "kWkIKWpVkwfKML",
+            "kWKIKWpVkwfkml",
+            "kWKIKWpVkwfKml",
+            "kWKIKWpVkwfkMl",
+            "kWKIKWpVkwfkmL",
+            "kWKIKWpVkwfKML",
+            "KWKiKWpVkwfkml",
+            "KWKiKWpVkwfKml",
+            "KWKiKWpVkwfkMl",
+            "KWKiKWpVkwfkmL",
+            "KWKiKWpVkwfKML",
+            "KWKIKWpVkwfkml",
+            "KWKIKWpVkwfKml",
+            "KWKIKWpVkwfkMl",
+            "KWKIKWpVkwfkmL",
+            "KWKIKWpVkwfKML",
+            "kwkikWPvkwFKML",
+            "kWkikWPvkwFKML",
+            "kwKikWPvkwFKML",
+            "kWKikWPvkwFKML",
+            "kwkikWPvkwfKML",
+            "kWkikWPvkwfKML",
+            "kwKikWPvkwfKML",
+            "kWKikWPvkwfKML",
+            "kwkikWPvkwfkML",
+            "kWkikWPvkwfkML",
+            "kwKikWPvkwfkML",
+            "kWKikWPvkwfkML",
+            "kwkikWPvkwfkmL"
+        ],
+        "0": [
+            "KWKIKWPVKWFKML",
+            "kWKIKWPVKWFKML",
+            "KwKIKWPVKWFKML",
+            "KWkIKWPVKWFKML",
+            "KWKIkwPVKWFKML",
+            "KWKIKWpVKWFKML",
+            "KWKIKWPvKWFKML",
+            "KWKIKWPVKWFkML",
+            "KWKIKWPVKWfKML",
+            "KWKIKWPVKWFkMl",
+            "KWKIKWPVKWFkmL",
+            "KWKIKWPVKWFKMl",
+            "KWKIKWPVKWFKmL",
+            "KWKIKWPVKWFKmL",
+            "kWKIKWPVKWFKMl",
+            "KWkIKWPVKWFKMl",
+            "KWKIkwPVKWFKMl",
+            "KWKIKWpVKWFKMl",
+            "KWKIKWPvKWFKMl",
+            "KWKIKWPVKWFkMl",
+            "KWKIKWPVKWfKMl",
+            "KWKIKWPVKWFkMl",
+            "KWKIKWPVKWFkmL",
+            "KWKIKWPVKWFKmL",
+            "kWKIKWPVKWFKmL",
+            "KWkIKWPVKWFKmL",
+            "KWKIkwPVKWFKmL",
+            "KWKIKWpVKWFKmL",
+            "KWKIKWPvKWFKmL",
+            "KWKIKWPVKWFkmL",
+            "KWKIKWPVKWfKML",
+            "kWKIKWPVKWfKML",
+            "KWkIKWPVKWfKML",
+            "KWKIkwPVKWfKML",
+            "KWKIKWpVKWfKML",
+            "KWKIKWPvKWfKML",
+            "KWKIKWPVKWFkML",
+            "KWKIKWPVKWfkML",
+            "KWKIKWPVKWfkmL",
+            "kWKIKWPVKWfkmL",
+            "KWkIKWPVKWfkmL",
+            "KWKIkwPVKWfkmL",
+            "KWKIKWpVKWfkmL",
+            "KWKIKWPvKWfkmL",
+            "KWKIKWPVKWFkMl",
+            "KWKIKWPVKWFkmL",
+            "KWKIKWPVKWfKml",
+            "KWKIKWPVKWFkml",
+            "KWKIKWPVKWFkMl",
+            "KWKIKWPVKWFkmL",
+            "KWKIKWPVKWFKml",
+            "KWKIKWPVKWFkmL",
+            "kWKIKWPVKWFkmL",
+            "KWkIKWPVKWFkmL",
+            "KWKIkwPVKWFkmL",
+            "KWKIKWpVKWFkmL",
+            "KWKIKWPvKWFkmL",
+            "KWKIKWPVKWFkml",
+            "KWKIKWPVKWfkml",
+            "KWKIKWPVKWfkmL",
+            "kWKIKWPVKWfkml",
+            "KWkIKWPVKWfkml",
+            "KWKIkwPVKWfkml",
+            "KWKIKWpVKWfkml",
+            "KWKIKWPvKWfkml",
+            "KWKIKWPVKWFkMl",
+            "KWKIKWPVKWFkml",
+            "KWKIKWPVKWFkmL",
+            "KWKIKWPVKWfKml",
+            "KWKIKWPVKWFkml",
+            "KWKIKWPVKWFkmL",
+            "KWKIKWPVKWFkmL",
+            "KWKIKWPVKWFKml",
+            "KWKIKWPVKWFkmL",
+            "KWKIKWPVKWFkml",
+            "KWKIKWPVKWFkmL",
+            "KWKIKWPVKWfkmL",
+            "kWKIKWPVKWFKML",
+            "KWkIKWPVKWFKML",
+            "KWKIkwPVKWFKML",
+            "KWKIKWpVKWFKML",
+            "KWKIKWPvKWFKML",
+            "KWKIKWPVKWFKmL",
+            "KWKIKWPVKWFKMl",
+            "KWKIKWPVKWFkML",
+            "KWKIKWPVKWfKML",
+            "KWKIKWPVKWFkmL",
+            "KWKIKWPVKWFkML",
+            "KWKIKWPvKWFKmL",
+            "kWKIKWPVKWFkmL",
+            "KWkIKWPVKWFkML",
+            "KWKIKWPVKWfkMl",
+            "KWkIKWPVKWFkmL",
+            "KWKIKWPVKWFkml",
+            "KWKIKWPVKWfkml",
+            "KWKIkwPVKWfkMl",
+            "KWKIKWPVKWFkmL",
+            "KWKIKWPVKWfkmL",
+            "KWKIKWPVKWFKml",
+            "KWKIKWPVKWFkmL",
+            "KWKIKWPVKWFkml",
+            "KWKIKWPVKWFkmL",
+            "KWKIKWPVKWfkml",
+            "KWKIKWPVKWfkmL",
+            "KWKIKWPVKWFkmL",
+            "KWKIKWPVKWFkml"
+        ]
+    }
+}

dataset/finetune_for_r2_llm.json

@@ -0,0 +1,197 @@
+{
+    "KWKIKWPVKWFKML": {
+        "1": [
+            "kwKIKWPVKWFKML",
+            "kWkIKWPVKWFKML",
+            "kWKIkWPVKWFKML",
+            "kWKIKwPVKWFKML",
+            "kWKIKWPvKWFKML",
+            "kWKIKWPVkWFKML",
+            "kWKIKWPVKwFKML",
+            "kWKIKWPVKWfKML",
+            "kWKIKWPVKWFKmL",
+            "KwkIKWPVKWFKML",
+            "KwKIkWPVKWFKML",
+            "KwKIKwPVKWFKML",
+            "KwKIKWPvKWFKML",
+            "KwKIKWPVkWFKML",
+            "KwKIKWPVKwFKML",
+            "KwKIKWPVKWfKML",
+            "KwKIKWPVKWFKmL",
+            "KWkIkWPVKWFKML",
+            "KWkIKwPVKWFKML",
+            "KWkIKWPvKWFKML",
+            "KWkIKWPVkWFKML",
+            "KWkIKWPVKwFKML",
+            "KWkIKWPVKWfKML",
+            "KWkIKWPVKWFKmL",
+            "KWKIkwPVKWFKML",
+            "KWKIkWPvKWFKML",
+            "KWKIkWPVkWFKML",
+            "KWKIkWPVKwFKML",
+            "KWKIkWPVKWfKML",
+            "KWKIkWPVKWFKmL",
+            "KWKIKwPvKWFKML",
+            "KWKIKwPVkWFKML",
+            "KWKIKwPVKwFKML",
+            "KWKIKwPVKWfKML",
+            "KWKIKwPVKWFKmL",
+            "KWKIKWPvkWFKML",
+            "KWKIKWPvKwFKML",
+            "KWKIKWPvKWFKmL",
+            "KWKIKWPVkwFKML",
+            "KWKIKWPVkWFKmL",
+            "KWKIKWPVKwfKML",
+            "KWKIKWPVKwFKmL",
+            "KWKIKWPVKWfKmL",
+            "kwkIKWPVKWFKML",
+            "kwKIkWPVKWFKML",
+            "kwKIKwPVKWFKML",
+            "kwKIKWPvKWFKML",
+            "kwKIKWPVkWFKML",
+            "kwKIKWPVKwFKML",
+            "kwKIKWPVKWfKML",
+            "kwKIKWPVKWFKmL",
+            "kWkIkWPVKWFKML",
+            "kWkIKwPVKWFKML",
+            "kWkIKWPvKWFKML",
+            "kWkIKWPVkWFKML",
+            "kWkIKWPVKwFKML",
+            "kWkIKWPVKWfKML",
+            "kWkIKWPVKWFKmL",
+            "kWKIkwPVKWFKML",
+            "kWKIkWPvKWFKML",
+            "kWKIkWPVkWFKML",
+            "kWKIkWPVKwFKML",
+            "kWKIkWPVKWfKML",
+            "kWKIkWPVKWFKmL",
+            "kWKIKwPvKWFKML",
+            "kWKIKwPVkWFKML",
+            "kWKIKwPVKwFKML",
+            "kWKIKwPVKWfKML",
+            "kWKIKwPVKWFKmL",
+            "kWKIKWPvkWFKML",
+            "kWKIKWPvKwFKML",
+            "kWKIKWPvKWFKmL",
+            "kWKIKWPVkwFKML",
+            "kWKIKWPVkWFKmL",
+            "kWKIKWPVKwfKML",
+            "kWKIKWPVKwFKmL",
+            "kWKIKWPVKWfKmL",
+            "kwkIKwPVKWFKML",
+            "kwkIKWPvKWFKML",
+            "kwkIKWPVkWFKML",
+            "kwkIKWPVKwFKML",
+            "kwkIKWPVKWfKML",
+            "kwkIKWPVKWFKmL",
+            "KwKIkwPVKWFKML",
+            "KwKIKwPvKWFKML",
+            "KwKIKwPVkWFKML",
+            "KwKIKwPVKwFKML",
+            "KwKIKwPVKWfKML",
+            "KwKIKwPVKWFKmL",
+            "KwKIKWPvkWFKML",
+            "KwKIKWPvKwFKML",
+            "KwKIKWPvKWFKmL",
+            "KwKIKWPVkwFKML",
+            "KwKIKWPVkWFKmL",
+            "KwKIKWPVKwfKML",
+            "KwKIKWPVKwFKmL",
+            "KwKIKWPVKWfKmL"
+        ],
+        "0": [
+            "KWKiKWPVKWfKML",
+            "KWKiKWPVKWFKmL",
+            "KWKiKWPVkWFKML",
+            "KWKiKWPVKwFKML",
+            "KWKiKWPvKWFKML",
+            "KWKiKwPVKWFKML",
+            "kWKiKWPVKWFKML",
+            "kWKiKWPvKWFKML",
+            "kWKiKWPVkWFKML",
+            "kWKiKWPVKwFKML",
+            "KWKIKWpVKWfKML",
+            "KWKIKWpVKWFKmL",
+            "KWKIKWpVKwFKML",
+            "KWKIKWpVkWFKML",
+            "KWKIKWpVkwFKML",
+            "KWKIKWpVKWFkML",
+            "KWKIKWpVKWFkMl",
+            "kWKIKWpVKWFKML",
+            "kWKIKWpVKWfKML",
+            "kWKIKWpVkWFKML",
+            "kWKIKWpVKwFKML",
+            "kWKIKWpVKWFKmL",
+            "KWKIKWPVKWFkML",
+            "KWKIKWPVKWFkML",
+            "KWKIKWPVKWfkML",
+            "KWKIKWPVKWfkMl",
+            "KWKIKWPVKWfKMl",
+            "KWKIKWPVKWFkMl",
+            "KWKIKWPVKwFkML",
+            "KWKIKWPVKwFkMl",
+            "KWKIKWPVkwFkML",
+            "KWKIKWPVkwFkMl",
+            "KWKIKWpVkwFkML",
+            "kWKIKWpVKWFkML",
+            "kWKIKWpVKWFkMl",
+            "kWKIKWpVkwFkML",
+            "KWKiKWPVKWFkML",
+            "KWKiKWPVKWFkMl",
+            "KWKiKWPVkwFkML",
+            "KWKiKWPVkwFkMl",
+            "KWKIKWpVKWfkMl",
+            "KWKIKWpVKWfkML",
+            "KWKIkWPvKWfkML",
+            "kWKIKWPvKWfkML",
+            "KWKIKwPvKWfKML",
+            "KWKIKWPvkWfKML",
+            "KWKIKWPvKwfKML",
+            "KWKIKWPvKWfkML",
+            "KWKIKWPvKWfkMl",
+            "KWKIKWPvkWFkMl",
+            "KWKIKWPvKWFkMl",
+            "KWKIKWpvKWFkML",
+            "KWKIKWpvKWFkMl",
+            "KWKiKWPvKWFkMl",
+            "kWKIKWPVKWFkMl",
+            "KwKiKWPVKWFkML",
+            "KwKiKWPVKWFkMl",
+            "KwKiKWPVKWfkML",
+            "KwKiKWPVKWfkMl",
+            "KWkIkWpVKWFkML",
+            "KWkIkWpVKWFkMl",
+            "KWkIkWpVKWfkML",
+            "KWkIkWpVKWfkMl",
+            "KWKiKWpVKWFKML",
+            "KWKiKWpVKWfKML",
+            "KWKiKWpVKwFKML",
+            "KWKiKWpVKWFkML",
+            "KWKiKWpVKWFKmL",
+            "KWKIKWpVKWFKMl",
+            "kWKIKWpVKWFKMl",
+            "KwKIKWpVKWFKMl",
+            "KWkIKWpVKWFKMl",
+            "KWKIkWpVKWFKMl",
+            "KWKIKwPvKWFKMl",
+            "KWKIKWpVkWFKMl",
+            "KWKIKWpVKwFKMl",
+            "kWKIKWpVKwFKMl",
+            "kWKIKWpVKWfkML",
+            "kWKIKWpVKWfkMl",
+            "KwKiKWPVkwfkML",
+            "KwKiKWPVkwfkMl",
+            "KWKIKWPvkwfKML",
+            "KWKIKWPVkwfkML",
+            "kWKIKWPVkwfkML",
+            "KWKiKWPVkwfkML",
+            "KWKiKWPVkwfkMl",
+            "KWKIKWpVkwfkML",
+            "KWKIKWpVkwfkMl",
+            "kWKIKWpVkwfkML",
+            "kWKIKWpVkwfkMl",
+            "KWKIKWPVkWFkML",
+            "KWKIKWPVkWFkMl"
+        ]
+    }
+}

dataset/test_.xlsx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb17d223ff62391058b5c257977abf929a9cbb6c8cf29c7d7f15aeb6a585b7b9
+size 101949

dataset/train.xlsx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2f19bd66bb781214298e586c07500e911fa8d50666a4918d91b656e397632a9
+size 228312

dataset/train_set.json

@@ -0,0 +1,1736 @@
+{
+    "GIMSSLMKKLAAHIAK": {
+        "1": [
+            "GIMSSLMkKLAAHIAK",
+            "GIMSSLMKkLAAHIAK",
+            "GIMSSLMKKLAAHIAk",
+            "GIMSSLMkkLAAHIAK",
+            "GIMSSLMkKLAAHIAk",
+            "GIMSSLMKkLAAHIAk",
+            "GIMSSLMkkLAAHIAk"
+        ],
+        "0": []
+    },
+    "ILGTILGLLKSL": {
+        "1": [],
+        "0": [
+            "ILGTILGLLkSL",
+            "ilgtilgllksl"
+        ]
+    },
+    "KRLFKKLLKYLRKF": {
+        "1": [
+            "KRLFkkLLKYLRkF",
+            "krLFkkLLKYLRkF",
+            "krLFkkLLkYLrkF"
+        ],
+        "0": [
+            "KRLFKKLLKYLRkF"
+        ]
+    },
+    "ILGTILGLLKGL": {
+        "1": [
+            "ilgtilgllkgl"
+        ],
+        "0": [
+            "ILGTILGLLkGL"
+        ]
+    },
+    "IDWKKLLDAAKQIL": {
+        "1": [
+            "idwkklldaakqil"
+        ],
+        "0": [
+            "IDWkkLLDAAkQIL"
+        ]
+    },
+    "VWRRWRRFWRR": {
+        "1": [],
+        "0": [
+            "vwrrwrrfwrr",
+            "VWrrWrrFWrr"
+        ]
+    },
+    "FLKLLKKLL": {
+        "1": [
+            "fLKLLKKLL",
+            "FlKLLKKLL",
+            "FLkLLKKLL",
+            "flkllkkll"
+        ],
+        "0": [
+            "FLKlLKKLL",
+            "FLKLlKKLL",
+            "FLKLLkKLL",
+            "FLKLLKkLL",
+            "FLKLLKKlL",
+            "FLKLLKKLl"
+        ]
+    },
+    "KKVVFWVKFK": {
+        "1": [
+            "KKVVFWVKFk"
+        ],
+        "0": [
+            "KKVVFWVKfK",
+            "KKVVFWVkFK",
+            "KKVVFWvKFK",
+            "KKVVFwVKFK",
+            "KKVVfWVKFK",
+            "KKVvFWVKFK",
+            "KKvVFWVKFK",
+            "KkVVFWVKFK",
+            "kKVVFWVKFK"
+        ]
+    },
+    "KRIVKLILKWLR": {
+        "1": [
+            "KRIVkLILKWLR",
+            "KRIVKlILKWLR"
+        ],
+        "0": []
+    },
+    "KKVVFKVKFKK": {
+        "1": [
+            "kKVVFKVKFKk"
+        ],
+        "0": [
+            "kkVVFKVKFKK",
+            "KKVVFKVKFkk",
+            "kkVVFKVKFkk",
+            "KKVVFkVKFKK",
+            "kkVVFkVKFkk",
+            "kkvvfkvkfkk"
+        ]
+    },
+    "KWKSFLKTFKSAKKTVLHTALKAISS": {
+        "1": [
+            "KWKSFLKTFKSAkKTVLHTALKAISS"
+        ],
+        "0": [
+            "KWKSFLKTFKSAKkTVLHTALKAISS",
+            "KWKSFLKTFKsAKkTVLHTALKAISS",
+            "KWKSFLKTFKSAKktVLHTALKAISS",
+            "KWKSFLKTFKsAKktVLHTALKAISS",
+            "KWKSFLKTFKSaKKTVLHTALKAISS",
+            "KWKSFLKTfKSaKKTVLHTALKAISS",
+            "KWKSFLKTFKSaKKTvLHTALKAISS",
+            "KWKSFLKTfKSaKKTvLHTALKAISS",
+            "kwksflktfksakktvlhtalkaiss"
+        ]
+    },
+    "FLPLIIGALSSLLPKIF": {
+        "1": [],
+        "0": [
+            "FLPLIIGALSSLLPKiF",
+            "FLPLiiGALSSLLPKiF"
+        ]
+    },
+    "KLKKLLKKWLKLLKKLLK": {
+        "1": [
+            "KLKKLlKKWLKlLKKLLk",
+            "KLKKlLKKWlKLLKkLLK",
+            "KLKkLLKkWLKlLKKlLK",
+            "KLkKLlKKwLKlLKkLLk",
+            "KlKkLlKkWlKlLkKlLk",
+            "KLKKLLKKWlkllkkllk"
+        ],
+        "0": []
+    },
+    "KKAAAAAAAAAAAAWAAAAAAKKKK": {
+        "1": [
+            "kkAAAAAAAAAAAAWAAAAAAKKKK",
+            "KKAAAAAAAAAAAAwaAAAAAKKKK",
+            "KKAAAAAAAAAAAAWAaaAAAKKKK",
+            "KKAAAAAAAAAAAAWAAAaaAKKKK"
+        ],
+        "0": [
+            "KKaaAAAAAAAAAAWAAAAAAKKKK",
+            "KKAAaaAAAAAAAAWAAAAAAKKKK",
+            "KKAAAAaaAAAAAAWAAAAAAKKKK",
+            "KKAAAAAAaaAAAAWAAAAAAKKKK",
+            "KKAAAAAAAAaaAAWAAAAAAKKKK",
+            "KKAAAAAAAAAAaaWAAAAAAKKKK",
+            "KKAAAAAAAAAAAAWAAAAAakKKK",
+            "KKAAAAAAAAAAAAWAAAAAAKkkK"
+        ]
+    },
+    "FVPWFSKFLGRIL": {
+        "1": [],
+        "0": [
+            "FVPWFSkFLGRIL",
+            "FVPWFSKfLGRIL",
+            "FVPWFSKFlGRIL",
+            "FVPWFSKFLGrIL",
+            "FVPWFSKFLGRiL",
+            "FVPWFSKFLGRIl"
+        ]
+    },
+    "IRIKIRIK": {
+        "1": [
+            "irikirik",
+            "IRIkIrIK"
+        ],
+        "0": [
+            "IrIkIrIk"
+        ]
+    },
+    "IIRKIIRK": {
+        "1": [
+            "iirkiirk",
+            "IirKIirK"
+        ],
+        "0": []
+    },
+    "KKLFKKILKYL": {
+        "1": [
+            "KKLfKKILKYL",
+            "KKLFKKILkYL",
+            "KKLFKKIlKYL",
+            "KKLFkKILKYL",
+            "KKlFKKILKYL",
+            "KkLFKKILKYL",
+            "KKLFKkILKYL",
+            "kKLFKKILKYL",
+            "KKLFKKIlkYL",
+            "KKlFKkILkYL",
+            "KKLFKKilkYL",
+            "kklfkkilkyl",
+            "kkLfKKILKYL",
+            "KKLFKKilkyl",
+            "KKLFkkilkyl",
+            "KKLfkkilkyl",
+            "KKlfkkilkyl",
+            "Kklfkkilkyl",
+            "kklfKKILKYL",
+            "kklfkKILKYL",
+            "kklfkkILKYL",
+            "kklfkkiLKYL",
+            "kklfkkilKYL",
+            "kklfkkilkYL",
+            "kklfkkilkyL"
+        ],
+        "0": [
+            "KKLFKkilkyl",
+            "KKLFKKiLKYL",
+            "KKLFKKILKyL",
+            "KKLFKKILKYl",
+            "KKlFKKILkYL",
+            "KKLFKKIlkyl"
+        ]
+    },
+    "KFFKRLLKSVRRAVKKFRK": {
+        "1": [],
+        "0": [
+            "kFFkrLLkSVrrAVkkFrk",
+            "kffkrllksvrravkkfrk"
+        ]
+    },
+    "RWRWRWK": {
+        "1": [
+            "rWRWRWK",
+            "rWRWRwK",
+            "rWRWrWK",
+            "rWRwRWK",
+            "rWrWRWK",
+            "rwRWRWK",
+            "rWRWrwK",
+            "rWRwRwK",
+            "rWrWRwK",
+            "rwRWRwK",
+            "rWRwrWK",
+            "rWrWrWK",
+            "rwRWrWK",
+            "rWrwRWK",
+            "rwRwRWK",
+            "rwrWRWK",
+            "rWRwrwK",
+            "rWrWrwK",
+            "rwRWrwK",
+            "rWrwRwK",
+            "rwrWRwK",
+            "rWrwrWK",
+            "rwRwrWK",
+            "rwrWrWK",
+            "rwrwRWK",
+            "rWrwrwK",
+            "rwRwrwK",
+            "rwrWrwK",
+            "rwrwRwK",
+            "rwrwrWK",
+            "rwrwrwK"
+        ],
+        "0": [
+            "rwRwRwK"
+        ]
+    },
+    "KWKSFLKTFKSLKKTVLHTLLKAISS": {
+        "1": [
+            "KWKSFLkTFKSLKKTVLHTLLKAISS",
+            "KWKSFLKTFKSLKkTVLHTLLKAISS",
+            "KWKSFLKTFKSLKKTVLHTLLkAISS",
+            "KWKSFLkTFKSLKkTVLHTLLKAISS",
+            "KWKSFLKTFKSLKkTVLHTLLkAISS",
+            "KWKSFlKTFKSLKKTVLHTLLKAISS",
+            "KWKSFLKTFKSlKKTVLHTLLKAISS",
+            "KWKSFLKTFKSLKKTVLHTlLKAISS",
+            "KWKSFLKTFKSlKKTVLHTlLKAISS",
+            "KWKSFlKTFKSlKKTVLHTlLKAISS"
+        ],
+        "0": [
+            "KWKSFLkTFKSLKkTVLHTLLkAISS",
+            "KWKSFLkTFkSLKkTVLHTLLkAISS",
+            "KWkSFLkTFkSLKkTVLHTLLkAISS",
+            "kWkSFLkTFkSLKkTVLHTLLkAISS",
+            "KWKSFlKTFKSlKKTVLHTLLKAISS",
+            "KWKSFlKTFKSlKKTVlHTlLKAISS",
+            "KWKSFlKTFKSlKKTVlHTllKAISS"
+        ]
+    },
+    "GWLDVAKKIGKAAFNVAKNFL": {
+        "1": [],
+        "0": [
+            "GWLDvAKKIGKAAFNvAKNFL",
+            "GWLDVAKKIGKAAFNvAKNFL"
+        ]
+    },
+    "GFGMALKLLKKVL": {
+        "1": [
+            "GfGmalkllkkvl",
+            "GfGMALKLLKKVL"
+        ],
+        "0": [
+            "GFGMALKLLKKVl",
+            "GFGMALKLLKKvL",
+            "GFGMALKLLKkVL",
+            "GFGMALKLLkKVL",
+            "GFGMALKLlKKVL",
+            "GFGMALKlLKKVL",
+            "GFGMALkLLKKVL",
+            "GFGMAlKLLKKVL",
+            "GFGMaLKLLKKVL",
+            "GFGmALKLLKKVL"
+        ]
+    },
+    "RGLRRLGRKIAHGVKKYGPTVLRIIRIA": {
+        "1": [],
+        "0": [
+            "rglrrlgrkiahgvkkygptvlriiria",
+            "RGLRRLGRKIAHGVKKYGptvlriiria"
+        ]
+    },
+    "KVLGRLVKVLGRLV": {
+        "1": [
+            "kVLGRLVKVLGRLV"
+        ],
+        "0": [
+            "KVLGRLVkVLGRLV",
+            "kVLGRLVkVLGRLV"
+        ]
+    },
+    "RRLFRRILRWL": {
+        "1": [
+            "RRLfRRILRWL",
+            "RRLFrRILRWL",
+            "rrlfrrilrwl"
+        ],
+        "0": [
+            "rRLFRRILRWL",
+            "RrLFRRILRWL",
+            "RRlFRRILRWL",
+            "RRLFRrILRWL",
+            "RRLFRRiLRWL",
+            "RRLFRRIlRWL",
+            "RRLFRRILrWL",
+            "RRLFRRILRwL",
+            "RRLFRRILRWl"
+        ]
+    },
+    "KWKSFLKTFKSAVKTVLHTALKAISS": {
+        "1": [
+            "KWKSFLKTFKSAvKTVLHTALKAISS",
+            "KWKSFLKTFKsAVKTVLHTALKAISS"
+        ],
+        "0": [
+            "kwksflktfksavktvlhtalkaiss"
+        ]
+    },
+    "RRWVRRVRRVWRRVVRVVRRWVRR": {
+        "1": [],
+        "0": [
+            "RRWVRRvRRVWRRVvRvVRRWvRR",
+            "RRWVRRvRRvWRRVvRvvRRWvRR",
+            "RRWvRRvRRvWRRvvRvvRRWvRR"
+        ]
+    },
+    "TVGGLVKWILKTVKKFA": {
+        "1": [
+            "tvgglvkwilktvkkfa",
+            "TVGGLVKWILkTVKKFA"
+        ],
+        "0": [
+            "TVGGLVkWILkTVKkFA"
+        ]
+    },
+    "INLKALAALAKKIL": {
+        "1": [],
+        "0": [
+            "iNLKALAALAKKIL",
+            "InLKALAALAKKIL",
+            "inLKALAALAKKIL",
+            "inlkalaalakkil"
+        ]
+    },
+    "FLSLIPKAIKAVGVKAKKF": {
+        "1": [],
+        "0": [
+            "FLSLIPkAIkAVGVkAkkF",
+            "FLSLIPkAIKAVGVKAKKF"
+        ]
+    },
+    "KKLLKLLKLLL": {
+        "1": [
+            "kkllkllklll",
+            "KkLLKLLKLLL",
+            "KkLLkLLKLLL",
+            "KkLlKLLKLLL",
+            "kKLLKLLKLLl",
+            "kkLLKLLKLLl",
+            "KkllKLLKLLL",
+            "kkLLkLLKLLL",
+            "KkllKLlKLLL"
+        ],
+        "0": [
+            "kkLLKLLKLLL",
+            "KKLLKllKLLL",
+            "KKLLkllKLLL",
+            "KkllKlLKLLL",
+            "KKLLkllkLLL",
+            "KKllKllKLLL",
+            "KKlLkLlKlLL",
+            "KKLlkLLklLL",
+            "KklLKLLKllL",
+            "kkLLKLLKLll",
+            "kkLLkLLKLLl",
+            "KKllKLLklLL",
+            "KklLKlLKlLL",
+            "KKllKLlKLlL",
+            "KKLlkLLkLLl",
+            "KkllKllKLLL",
+            "KKllKllKlLL",
+            "kkLLkLLKLll",
+            "kkLLkLLkLLl",
+            "kKLLkllKLLl",
+            "KKLlkllkLLL",
+            "KkLlKlLkLlL",
+            "kKlLkLlKlLl"
+        ]
+    },
+    "KKVVFKVKFK": {
+        "1": [],
+        "0": [
+            "KKVVFKVKFk",
+            "kKVVFKVKFk",
+            "kkVVFKVKFk",
+            "KKVVfkvKFK",
+            "kKVVfkvKFk"
+        ]
+    },
+    "LKLLKKLLKKLLKLL": {
+        "1": [
+            "LKlLKkLlkKLLkLL"
+        ],
+        "0": []
+    },
+    "KLKLLKLLKLLKLLK": {
+        "1": [],
+        "0": [
+            "KLkLLkLlkLLKlLK"
+        ]
+    },
+    "KKKLLLLLLLLLKKK": {
+        "1": [
+            "KKkLLlLllLLLkKK"
+        ],
+        "0": []
+    },
+    "KKFKKTAKWLIKSAWLLLKSLALKMK": {
+        "1": [
+            "kkfkktakwliksawlllkslalkmk"
+        ],
+        "0": []
+    },
+    "WWWLRRRW": {
+        "1": [],
+        "0": [
+            "wwwlrrrw"
+        ]
+    },
+    "RRRWWWWV": {
+        "1": [],
+        "0": [
+            "rrrwwwwv"
+        ]
+    },
+    "KWFRVYRGIYRRR": {
+        "1": [],
+        "0": [
+            "kwfrvyrgiyrrr"
+        ]
+    },
+    "RRRYIGRYVRFWK": {
+        "1": [],
+        "0": [
+            "rrryigryvrfwk"
+        ]
+    },
+    "GKIIKLKASLKLL": {
+        "1": [
+            "gkiiklkaslkll"
+        ],
+        "0": []
+    },
+    "KLFKKLFKKLFK": {
+        "1": [],
+        "0": [
+            "kLFkkLFkkLFk"
+        ]
+    },
+    "GFFALIPKIISSPLFKTLLSAV": {
+        "1": [],
+        "0": [
+            "GFFALIpKIISSPLFKTllSAV"
+        ]
+    },
+    "KGFFALIPKIISSPLFKTLLSAV": {
+        "1": [],
+        "0": [
+            "KGFFALIpKIISSPLFKTllSAV"
+        ]
+    },
+    "RGLRRLGRKIAHGVKKYG": {
+        "1": [
+            "rglrrlgrkiahgvkkyg"
+        ],
+        "0": []
+    },
+    "FLGGLIKIVPAMICAVTKKC": {
+        "1": [
+            "flGGlikivpamicavtkkc"
+        ],
+        "0": []
+    },
+    "AKRLKKLAKKIWKWK": {
+        "1": [],
+        "0": [
+            "AkRLkkLAkkIWkWk"
+        ]
+    },
+    "VDKPPYLPRPRPIRRPGGR": {
+        "1": [
+            "VDkPPYLPrPrPIrrPGGr"
+        ],
+        "0": [
+            "VDKPPYLPrPRPIrRPGGR",
+            "VDKPPYLPrPRPIRrPGGR",
+            "VDKPPYLPrPRPIRRPGGr",
+            "VDKPPYLPRPrPIrRPGGR",
+            "VDKPPYLPRPrPIRrPGGR",
+            "VDKPPYLPRPrPIRRPGGr",
+            "VDKPPYLPRPRPIrRPGGr",
+            "VDKPPYLPRPRPIRrPGGr"
+        ]
+    },
+    "GIGAVLKVLTTGLPALISWIKRKRQQ": {
+        "1": [
+            "GIGAVlKVLTTGlPALISWiKRKRQQ",
+            "gigavlkvlttglpaliswikrkrqq"
+        ],
+        "0": [
+            "GIGAvLKvLTTGLPALiSWIkRKRQQ"
+        ]
+    },
+    "FWGALAKGALKLIPSLFSSFSKKD": {
+        "1": [
+            "fwGalakGalklipslfssfskkd"
+        ],
+        "0": []
+    },
+    "IRVKIRVKIRVK": {
+        "1": [
+            "irvkirvkirvk"
+        ],
+        "0": []
+    },
+    "LIKKALAALAKLNI": {
+        "1": [],
+        "0": [
+            "likkalaalaklni"
+        ]
+    },
+    "RSMRLSFRARGYGFR": {
+        "1": [
+            "rsmrlsfrarGyGfr"
+        ],
+        "0": []
+    },
+    "GLLKRIKTLL": {
+        "1": [],
+        "0": [
+            "GLLkRIkTLL",
+            "Gllkriktll"
+        ]
+    },
+    "KKLFKKILRYL": {
+        "1": [
+            "KKLfKKILRYL"
+        ],
+        "0": [
+            "KKLFKkilryl",
+            "kklfkkilryl"
+        ]
+    },
+    "FQWQRNMRKVR": {
+        "1": [
+            "fqwqrnmrkvr"
+        ],
+        "0": []
+    },
+    "KKKKKKAAFAAWAAFAA": {
+        "1": [],
+        "0": [
+            "kkkkkkaafaawaafaa"
+        ]
+    },
+    "RRWWRF": {
+        "1": [],
+        "0": [
+            "rrwwrf"
+        ]
+    },
+    "KWKSFLKTFKSALKTVLHTALKAISS": {
+        "1": [
+            "KWKSFLKTFKSAlKTVLHTALKAISS"
+        ],
+        "0": []
+    },
+    "KWKSFLKTFKSAAKTVLHTALKAISS": {
+        "1": [
+            "KWKSFLKTFKSAaKTVLHTALKAISS"
+        ],
+        "0": []
+    },
+    "KWKSFLKTFKSASKTVLHTALKAISS": {
+        "1": [],
+        "0": [
+            "KWKSFLKTFKSAsKTVLHTALKAISS"
+        ]
+    },
+    "KWKSFLKTFKLAVKTVLHTALKAISS": {
+        "1": [
+            "KWKSFLKTFKlAVKTVLHTALKAISS"
+        ],
+        "0": []
+    },
+    "KWKSFLKTFKVAVKTVLHTALKAISS": {
+        "1": [
+            "KWKSFLKTFKvAVKTVLHTALKAISS"
+        ],
+        "0": []
+    },
+    "KWKSFLKTFKAAVKTVLHTALKAISS": {
+        "1": [
+            "KWKSFLKTFKaAVKTVLHTALKAISS"
+        ],
+        "0": []
+    },
+    "KWKSFLKTFKKAVKTVLHTALKAISS": {
+        "1": [
+            "KWKSFLKTFKkAVKTVLHTALKAISS"
+        ],
+        "0": []
+    },
+    "GFKMALKLLKKVL": {
+        "1": [],
+        "0": [
+            "GFkMALKLLKKVL",
+            "GfkMALKLLKKVL"
+        ]
+    },
+    "AFGMALKLLKKVL": {
+        "1": [],
+        "0": [
+            "aFGMALKLLKKVL"
+        ]
+    },
+    "RRLLRLLRLLL": {
+        "1": [
+            "rrLLrLLrLLL"
+        ],
+        "0": []
+    },
+    "KKIIKIIKIII": {
+        "1": [
+            "kkIIkIIkIII"
+        ],
+        "0": []
+    },
+    "RRIIRIIRIII": {
+        "1": [
+            "rrIIrIIrIII"
+        ],
+        "0": []
+    },
+    "KRFKKFFKKVKKSVKKRLKKIFKKPMVIGVTIPF": {
+        "1": [],
+        "0": [
+            "krfkkffkkvkksvkkrlkkifkkpmviGvtipf"
+        ]
+    },
+    "KKRLKKIFKKPMVIGVTIPF": {
+        "1": [],
+        "0": [
+            "kkrlkkifkkpmviGvtipf"
+        ]
+    },
+    "RLFRRVKKVAGKIAKRIWK": {
+        "1": [],
+        "0": [
+            "rlfrrvkkvagkiakriwk"
+        ]
+    },
+    "FIRRIARLLRRIF": {
+        "1": [],
+        "0": [
+            "firriarllrrif"
+        ]
+    },
+    "GIGAVLKVLALISWIKRKR": {
+        "1": [],
+        "0": [
+            "GIGAvLKvLAlISWIkRKR"
+        ]
+    },
+    "WKKLKKLLKKLKKL": {
+        "1": [],
+        "0": [
+            "Wkklkkllkklkkl"
+        ]
+    },
+    "KFWSLLKKALRLWANVL": {
+        "1": [
+            "kFwSLLkKALRLwANVL"
+        ],
+        "0": []
+    },
+    "KFWKLLKKALRLWAKVL": {
+        "1": [
+            "kFwKLLkKALrLwAkVL"
+        ],
+        "0": [
+            "kFWKlLKkAlrLWAkVL"
+        ]
+    },
+    "WFKKLLKKALRLWKKVL": {
+        "1": [
+            "wFKKlLKkAlrLWKkVL"
+        ],
+        "0": []
+    },
+    "ILLKKLLKKI": {
+        "1": [
+            "illkkllkki"
+        ],
+        "0": []
+    },
+    "GRFKRFRKKFKKLFKKLS": {
+        "1": [
+            "GRfKRfRKKfKKLfKKLS"
+        ],
+        "0": [
+            "grfkrfrkkfkklfkkls"
+        ]
+    },
+    "RAGLQFPVGRVHRLLRK": {
+        "1": [
+            "raglqfpvgrvhrllrk"
+        ],
+        "0": []
+    },
+    "KLKLLLLLKLK": {
+        "1": [
+            "klklllllklk"
+        ],
+        "0": []
+    },
+    "KLKLLLKLK": {
+        "1": [
+            "klklllklk"
+        ],
+        "0": []
+    },
+    "FIKRIARLLRKIF": {
+        "1": [],
+        "0": [
+            "fikriarllrkif"
+        ]
+    },
+    "INLKAIAALAKKLL": {
+        "1": [],
+        "0": [
+            "inlkaiaalakkll"
+        ]
+    },
+    "FLPLIGRVLSGIL": {
+        "1": [],
+        "0": [
+            "flpligrvlsgil"
+        ]
+    },
+    "KLLKKAGKLLKKAGKLLKKAG": {
+        "1": [],
+        "0": [
+            "KlLkKaGkLlKkAGKlLkKaG"
+        ]
+    },
+    "LLAKKKGLLAKKKGLLAKKKG": {
+        "1": [
+            "LlAkKkGlLaKkKgLlAkKkG"
+        ],
+        "0": []
+    },
+    "RPFTRAQWFAIQHISPRTIAMRAINNYRWR": {
+        "1": [],
+        "0": [
+            "rpftraqwfaiqhisprtiamrainnyrwr"
+        ]
+    },
+    "RLWLAIWRR": {
+        "1": [
+            "rlwlaiwrr"
+        ],
+        "0": []
+    },
+    "KLWLAIWKK": {
+        "1": [
+            "klwlaiwkk"
+        ],
+        "0": []
+    },
+    "FLKLLKKLLFLKLLKKLL": {
+        "1": [
+            "fLKLLKKLLfLKLLKKLL"
+        ],
+        "0": []
+    },
+    "VDKPPYLPRPRPPRRIYNR": {
+        "1": [
+            "VDKPPYLPRPrpprriynr",
+            "VDKPPYLPRPRpPRRIYNR",
+            "VDKPPYLPRPRPPRrIYNr"
+        ],
+        "0": [
+            "VDKPPYLPRPRPPRriynr",
+            "VDKPPYLPRPRpprriynr",
+            "VDKPPYLPRPrPPRRIYNR",
+            "VDKPPYLPRpRPPRRIYNR",
+            "VDKPPYLPrPRPPRRIYNR",
+            "VDKPPYLpRPRPPRRIYNR",
+            "VDKPPYlPRPRPPRRIYNR",
+            "VDKPPyLPRPRPPRRIYNR",
+            "VDKPpYLPRPRPPRRIYNR",
+            "VDKppYLPRPRPPRRIYNR",
+            "VDKpPYLPRPRPPRRIYNR",
+            "vdkppylprprpprriynr",
+            "VDKPPYLPRPRPPRRIYNr",
+            "VDKPPYLPRPRPPRrIYNR"
+        ]
+    },
+    "VRLIVAVRIWRR": {
+        "1": [],
+        "0": [
+            "vrlivavriwrr"
+        ]
+    },
+    "VRLRWWRRRWRR": {
+        "1": [],
+        "0": [
+            "vrlrwwrrrwrr"
+        ]
+    },
+    "RRW": {
+        "1": [],
+        "0": [
+            "rRW",
+            "RrW",
+            "RRw",
+            "rrW",
+            "Rrw",
+            "rRw",
+            "rrw"
+        ]
+    },
+    "FLGTVLKVAAKVLPAALCQIFKKC": {
+        "1": [
+            "FlGTVlKVAAKVlPAAlCQIFKKC"
+        ],
+        "0": [
+            "FLGTVLkVAAkVLPAALCQIFkkC"
+        ]
+    },
+    "FLGTVLKVLAKVLPAALCQIFKKC": {
+        "1": [
+            "FlGTVlKVlAKVlPAAlCQIFKKC"
+        ],
+        "0": []
+    },
+    "FLGTVLRVAARVLPAALCQIFRRC": {
+        "1": [],
+        "0": [
+            "FLGTVLrVAArVLPAALCQIFrrC"
+        ]
+    },
+    "RWKIFKKIEKMGRNIRDGIVKAGPAIQVLGSAKAI": {
+        "1": [],
+        "0": [
+            "rwkifkkiekmgrnirdgivkagpaiqvlgsakai"
+        ]
+    },
+    "GPLGVRGKRLWDIVRRWVGWL": {
+        "1": [
+            "GPlGvRGKRLWDIVRRWVGWL"
+        ],
+        "0": []
+    },
+    "RIVQRIKKWLR": {
+        "1": [
+            "rivqrikkwlr"
+        ],
+        "0": []
+    },
+    "KRIWQRIK": {
+        "1": [
+            "kriwqrik"
+        ],
+        "0": []
+    },
+    "KRIWQRIKDF": {
+        "1": [
+            "kriwqrikdf"
+        ],
+        "0": []
+    },
+    "KYKKALKKLAKLL": {
+        "1": [
+            "kykkalkklakll"
+        ],
+        "0": []
+    },
+    "VQWRAIRVRVIR": {
+        "1": [
+            "vqwrairvrvir"
+        ],
+        "0": []
+    },
+    "GFAWNVCVYRNGVRVCHRRAN": {
+        "1": [],
+        "0": [
+            "GfawnvcvyrnGvrvchrran"
+        ]
+    },
+    "RKRWWRWWKWWKR": {
+        "1": [],
+        "0": [
+            "RKrWWrWwkWWkR"
+        ]
+    },
+    "WRWWKWW": {
+        "1": [],
+        "0": [
+            "WrWwkWW"
+        ]
+    },
+    "WWRWWKWW": {
+        "1": [],
+        "0": [
+            "WWrWwkWW"
+        ]
+    },
+    "RRGKKLLLLLKKKG": {
+        "1": [
+            "rrgkklllllkkkg"
+        ],
+        "0": []
+    },
+    "LLWIALRKK": {
+        "1": [
+            "llwialrkk"
+        ],
+        "0": []
+    },
+    "PRPRPRP": {
+        "1": [],
+        "0": [
+            "prprprp"
+        ]
+    },
+    "KWLKKWLKWLKK": {
+        "1": [],
+        "0": [
+            "kwLkkwLkwLkk"
+        ]
+    },
+    "ILRWPWWPWRRK": {
+        "1": [],
+        "0": [
+            "ilrwpwwpwrrk"
+        ]
+    },
+    "KRKIFLRTKILV": {
+        "1": [
+            "KrKiFlRtKiLv"
+        ],
+        "0": [
+            "kRkIfLrTkIlV"
+        ]
+    },
+    "VLIKTRLFIKRK": {
+        "1": [
+            "vLiKtRlFiKrK"
+        ],
+        "0": []
+    },
+    "KWKLFKKIEKVGQNIRDGIIKAGPAVAVVGQATQIAK": {
+        "1": [],
+        "0": [
+            "kwklfkkiekvgqnirdgiikagpavavvgqatqiak"
+        ]
+    },
+    "GIGKFLHSAKKFGKAFVGEIMNS": {
+        "1": [
+            "gigkflhsakkfgkafvgeimns"
+        ],
+        "0": []
+    },
+    "KWKLFKKIEKVGQGIGAVLKVLTTGL": {
+        "1": [],
+        "0": [
+            "kwklfkkiekvgqgigavlkvlttgl"
+        ]
+    },
+    "KWKLFKKIGIGAVLKVLTTGLPALIS": {
+        "1": [
+            "kwklfkkigigavlkvlttglpalis"
+        ],
+        "0": []
+    },
+    "KWKLFKKGIGAVLKV": {
+        "1": [
+            "kwklfkkgigavlkv"
+        ],
+        "0": []
+    },
+    "KWKLFKKIGAVLKVL": {
+        "1": [
+            "kwklfkkigavlkvl"
+        ],
+        "0": []
+    },
+    "KWKLFKKGAVLKVLT": {
+        "1": [
+            "kwklfkkgavlkvlt"
+        ],
+        "0": []
+    },
+    "KWKLFKKAVLKVLTT": {
+        "1": [
+            "kwklfkkavlkvltt"
+        ],
+        "0": []
+    },
+    "KWKLFKKVLKVLTTG": {
+        "1": [
+            "kwklfkkvlkvlttg"
+        ],
+        "0": []
+    },
+    "GSKKPVPIIYCNRRTGKCQRM": {
+        "1": [],
+        "0": [
+            "gskkpvpiiycnrrtgkcqrm"
+        ]
+    },
+    "RRWQWRMKK": {
+        "1": [
+            "rrwqwrmkk"
+        ],
+        "0": []
+    },
+    "FKCRRWQWRMKKLGA": {
+        "1": [
+            "fkcrrwqwrmkklga"
+        ],
+        "0": []
+    },
+    "PKLLKTFLSKWIG": {
+        "1": [],
+        "0": [
+            "pkllktflskwig",
+            "pkllktflskwiG"
+        ]
+    },
+    "KLPLIGRVLSGIL": {
+        "1": [
+            "klpligrvlsgil"
+        ],
+        "0": []
+    },
+    "KKHRKHRKHRKHGGSGGSKNLRRIIRKGIHIIKKYG": {
+        "1": [],
+        "0": [
+            "kkhrkhrkhrkhggsggsknlrriirkgihiikkyg"
+        ]
+    },
+    "FKRIVQRIKDFLRNLV": {
+        "1": [],
+        "0": [
+            "FKRiVQRiKDFlRNLV"
+        ]
+    },
+    "GWGSFFKKAAHVGKHVGKAALTHYL": {
+        "1": [],
+        "0": [
+            "gwgsffkkaahvgkhvgkaalthyl",
+            "GwGsffkkaahvGkhvGkaalthyl"
+        ]
+    },
+    "RRGWVLALVLRYGRR": {
+        "1": [],
+        "0": [
+            "RRGWVLALVlRYGRR"
+        ]
+    },
+    "RRGWVLALYLRYGRR": {
+        "1": [],
+        "0": [
+            "RRGWVLALYlRYGRR"
+        ]
+    },
+    "RRGWALRLVLAY": {
+        "1": [],
+        "0": [
+            "RRGWALRLVlAY"
+        ]
+    },
+    "KWKKLLKKPLLKKLLKKL": {
+        "1": [
+            "kwkkllkkpllkkllkkl"
+        ],
+        "0": []
+    },
+    "NKKAGLFVVQFPKKY": {
+        "1": [
+            "nkkaglfvvqfpkky"
+        ],
+        "0": []
+    },
+    "LVKKLLKLAMGFG": {
+        "1": [
+            "lvkkllklamgfg"
+        ],
+        "0": []
+    },
+    "WLRRIKAWLRRIKA": {
+        "1": [
+            "wlrrikawlrrika"
+        ],
+        "0": []
+    },
+    "RRGWARRLAFAFGRR": {
+        "1": [
+            "rrgwarrlafafgrr"
+        ],
+        "0": []
+    },
+    "GKKLLKKLKKLLKKG": {
+        "1": [],
+        "0": [
+            "GKKllKKlKKllKKG"
+        ]
+    },
+    "GLLSVLGSVAKHVLPHVVPVIAEHL": {
+        "1": [],
+        "0": [
+            "GllsvlGsvakhvlphvvpviaehl"
+        ]
+    },
+    "EFKRIVQRIKDFLRNLV": {
+        "1": [],
+        "0": [
+            "EfKRiVQRiKDfLRNLV"
+        ]
+    },
+    "GLFDVIKKVASVIGGL": {
+        "1": [
+            "GlfdvikkvasviGGl"
+        ],
+        "0": []
+    },
+    "GIGKFLKKAKKFGKAFVKILKK": {
+        "1": [
+            "GiGkflkkakkfGkafvkilkk"
+        ],
+        "0": []
+    },
+    "GFKKLLKGAAKALVKTVLF": {
+        "1": [],
+        "0": [
+            "GFKkLLKGAAKALVKTVLF"
+        ]
+    },
+    "GFKDLLKKAAKALVKTVLF": {
+        "1": [],
+        "0": [
+            "GFKDLLKkAAKALVKTVLF"
+        ]
+    },
+    "GFKDLLKGAKKALVKTVLF": {
+        "1": [],
+        "0": [
+            "GFKDLLKGAKkALVKTVLF"
+        ]
+    },
+    "GFKDLLKGAAKALKKTVLF": {
+        "1": [],
+        "0": [
+            "GFKDLLKGAAKALkKTVLF"
+        ]
+    },
+    "GFKDLLKGAAKALVKTVKF": {
+        "1": [],
+        "0": [
+            "GFKDLLKGAAKALVKTVkF"
+        ]
+    },
+    "KLWKKWKKWLK": {
+        "1": [],
+        "0": [
+            "klwkkwkkwlk"
+        ]
+    },
+    "RLWRRWRRWLR": {
+        "1": [
+            "rlwrrwrrwlr"
+        ],
+        "0": []
+    },
+    "GMWSKILGHLIR": {
+        "1": [
+            "GmwskilGhlir"
+        ],
+        "0": [
+            "GMWSKIlGHLIR",
+            "GMWSKiLGHLIR",
+            "GMWSkILGHLIR"
+        ]
+    },
+    "GKWMSLLKHILK": {
+        "1": [
+            "Gkwmsllkhilk"
+        ],
+        "0": [
+            "GKWMSLLKhILK",
+            "GKwMSLLKHILK"
+        ]
+    },
+    "GVCRCVCRRGVCRCVCRR": {
+        "1": [
+            "GvcrcvcrrGvcrcvcrr"
+        ],
+        "0": []
+    },
+    "RGGRLCYCRRRFCVCVGR": {
+        "1": [
+            "rGGrlcycrrrfcvcvGr"
+        ],
+        "0": []
+    },
+    "RRWCFRVCYRGFCYRKCR": {
+        "1": [],
+        "0": [
+            "rrwcfrvcyrGfcyrkcr"
+        ]
+    },
+    "GLFVGLAKVAAHVVPAIAEHF": {
+        "1": [],
+        "0": [
+            "GlfvGlakvaahvvpaiaehf"
+        ]
+    },
+    "ILGKLLKTAAGLLSNL": {
+        "1": [],
+        "0": [
+            "ILGKLLkTAAGLLSNL"
+        ]
+    },
+    "ILGKLLSTAAKLLSNL": {
+        "1": [],
+        "0": [
+            "ILGKLLSTAAkLLSNL"
+        ]
+    },
+    "ILGKLLKTAAKLLSNL": {
+        "1": [],
+        "0": [
+            "ILGKLLkTAAkLLSNL"
+        ]
+    },
+    "WLLKRWKKLL": {
+        "1": [
+            "wllkrwkkll"
+        ],
+        "0": []
+    },
+    "KLLKWWKKLL": {
+        "1": [
+            "kllkwwkkll"
+        ],
+        "0": []
+    },
+    "RRIRPRPPRLPRPRPRPLPYPRP": {
+        "1": [],
+        "0": [
+            "rrIRPRPPRLPRPRPRPLPYPRP"
+        ]
+    },
+    "KRWWKWWRR": {
+        "1": [
+            "krwwkwwrr"
+        ],
+        "0": []
+    },
+    "GIMSSLMKKLKKIIAK": {
+        "1": [
+            "Gimsslmkklkkiiak"
+        ],
+        "0": []
+    },
+    "GILSSLLKKLKKIIAK": {
+        "1": [
+            "Gilssllkklkkiiak"
+        ],
+        "0": []
+    },
+    "GILSSLWKKLKKIIAK": {
+        "1": [],
+        "0": [
+            "Gilsslwkklkkiiak"
+        ]
+    },
+    "FFFLSRIF": {
+        "1": [],
+        "0": [
+            "ffflsrif"
+        ]
+    },
+    "FIRSLFFF": {
+        "1": [
+            "firslfff"
+        ],
+        "0": []
+    },
+    "IKIPSFFRNILKKVGKEAVSLIAGALKQS": {
+        "1": [],
+        "0": [
+            "IKIPSFFrNILKKVGKEAVSLIAGALKQS"
+        ]
+    },
+    "WWWLRKIW": {
+        "1": [
+            "wwwlrkiw"
+        ],
+        "0": []
+    },
+    "LLGMIPVAIKAISALSKL": {
+        "1": [
+            "LlGMIPVAIKAISALSKL"
+        ],
+        "0": []
+    },
+    "RLLRKFFRKLKKSV": {
+        "1": [],
+        "0": [
+            "rllrkffrklkksv"
+        ]
+    },
+    "GGLRSLGRKILRAWKKYGPIIVPIIRIG": {
+        "1": [
+            "GGlrslGrkilrawkkyGpiivpiiriG"
+        ],
+        "0": []
+    },
+    "WKIVFWWRR": {
+        "1": [],
+        "0": [
+            "wkivfwwrr"
+        ]
+    },
+    "RRWRIVVIRVRR": {
+        "1": [
+            "rrwrivvirvrr"
+        ],
+        "0": []
+    },
+    "GFGSLLGKALRLGANVL": {
+        "1": [
+            "GfGsllGkalrlGanvl"
+        ],
+        "0": []
+    },
+    "GFGSLLGKALRLWKKVL": {
+        "1": [],
+        "0": [
+            "GFGSLLGKALRLwKkVL",
+            "GFGSLLGKAlrLwKkVL"
+        ]
+    },
+    "GKWKKILGKLIR": {
+        "1": [],
+        "0": [
+            "GkwkkilGklir"
+        ]
+    },
+    "KKWRKWLKWLAKK": {
+        "1": [],
+        "0": [
+            "kkwrkwlkwlakk"
+        ]
+    },
+    "KWRRWIRWL": {
+        "1": [],
+        "0": [
+            "kwrrwirwl"
+        ]
+    },
+    "RRWVRRVRRWVRRVVRVVRRWVRR": {
+        "1": [
+            "RRWvRRvRRWvRRvvRvvRRWvRR"
+        ],
+        "0": []
+    },
+    "VFRLKKWIQKVI": {
+        "1": [
+            "vfrlkkwiqkvi"
+        ],
+        "0": []
+    },
+    "IVKQIWKKLRFV": {
+        "1": [
+            "ivkqiwkklrfv"
+        ],
+        "0": []
+    },
+    "LPLIAGLWGKIW": {
+        "1": [],
+        "0": [
+            "LPLIAGLwGKIw"
+        ]
+    },
+    "FVQWFSKFLGRIL": {
+        "1": [],
+        "0": [
+            "fqvqwfskflgril"
+        ]
+    },
+    "FVPWFSKFLPRIL": {
+        "1": [],
+        "0": [
+            "FVPWFSKFLpRIL"
+        ]
+    },
+    "FFHHIFRAIVHVAKTIHRLVTG": {
+        "1": [
+            "FFHHIFRaIVHVaKTIHRLVTG"
+        ],
+        "0": []
+    },
+    "HFLKTLVNLAKKIL": {
+        "1": [],
+        "0": [
+            "HFLkTLVNLAKKIL"
+        ]
+    },
+    "HFLGKLVNLAKKIL": {
+        "1": [],
+        "0": [
+            "HFLGkLVNLAKKIL"
+        ]
+    },
+    "HFLGTLKNLAKKIL": {
+        "1": [],
+        "0": [
+            "HFLGTLkNLAKKIL"
+        ]
+    },
+    "HFLGTLVKLAKKIL": {
+        "1": [],
+        "0": [
+            "HFLGTLVkLAKKIL"
+        ]
+    },
+    "HFLGTLVNLAKKIL": {
+        "1": [],
+        "0": [
+            "HFLGTLVNLAkKIL",
+            "HFLGTLVNLAKkIL"
+        ]
+    },
+    "ACPIFTKIQGTYRGRAKCR": {
+        "1": [],
+        "0": [
+            "ACPiFTKiQGTYrGrAKCR"
+        ]
+    },
+    "KLALKLALKALKAAKLA": {
+        "1": [
+            "KLalKLALKALKAAKLA"
+        ],
+        "0": [
+            "klALKLALKALKAAKLA",
+            "KLaLKLALKALKAAKLA",
+            "KLALklALKALKAAKlA",
+            "KLALklALKALKAAKLA",
+            "KLALKLalKALKAALKLA",
+            "KLALKLALkaLKAALKLA",
+            "KLALKLALKAlkAALKLA",
+            "KLALKLALKALKaaLKLA",
+            "KLALKLALKALKAAlkLA",
+            "KLALKLALKALKAALKla"
+        ]
+    },
+    "KWKLFKKIPKFLHLAKKF": {
+        "1": [],
+        "0": [
+            "KWKLFKKIpKFLHLAKKF"
+        ]
+    },
+    "FFGSVLKLIPKIL": {
+        "1": [],
+        "0": [
+            "ffGsvlklipkil"
+        ]
+    },
+    "IKLSPKTKDNLKKVLKGAIKGAIAVAKMV": {
+        "1": [
+            "IKLSPkTKDNLKKVLKGAIKGAIAVAKMV"
+        ],
+        "0": []
+    },
+    "IKLSPETKKNLKKVLKGAIKGAIAVAKMV": {
+        "1": [
+            "IKLSPETKkNLKKVLKGAIKGAIAVAKMV"
+        ],
+        "0": []
+    },
+    "IKLSPKTKKNLKKVLKGAIKGAIAVAKMV": {
+        "1": [
+            "IKLSPkTKkNLKKVLKGAIKGAIAVAKMV"
+        ],
+        "0": []
+    },
+    "GLKKIFKAGLGSLVKGIAAHVAS": {
+        "1": [],
+        "0": [
+            "GLKkIFKAGLGSLVKGIAAHVAS"
+        ]
+    },
+    "GLKKIFKKGLGSLVKGIAAHVAS": {
+        "1": [
+            "GLKkIFKKGLGSLVKGIAAHVAS"
+        ],
+        "0": []
+    },
+    "GLKKIFKAGLGSLKKGIAAHVAS": {
+        "1": [],
+        "0": [
+            "GLKkIFKAGLGSLKKGIAAHVAS"
+        ]
+    },
+    "GLKKIFKAGLGSLVKGIKAHVAS": {
+        "1": [],
+        "0": [
+            "GLKkIFKAGLGSLVKGIKAHVAS"
+        ]
+    },
+    "ILGKLLSTAAGLLSKL": {
+        "1": [
+            "ILGKLLSTAAGLLSkL"
+        ],
+        "0": []
+    },
+    "ILGKLLSTAAKLLSKL": {
+        "1": [],
+        "0": [
+            "ILGKLLSTAAkLLSKL"
+        ]
+    },
+    "GFKRIVQRIKDFLRNLV": {
+        "1": [],
+        "0": [
+            "GFKRiVQRiKDFlRNLV"
+        ]
+    },
+    "GLKALKKVFKGIHKAIKLINNHVQ": {
+        "1": [],
+        "0": [
+            "GLkALKKVFkGIHkAIKLINNHVQ"
+        ]
+    },
+    "KFFKKLKNSVKKRAKKFFKKPRVIGVSIPF": {
+        "1": [],
+        "0": [
+            "kffkklknsvkkrakkffkkprvigvsipf"
+        ]
+    },
+    "KFFKKLKKAVKKGFKKFAKV": {
+        "1": [],
+        "0": [
+            "kffkklkkavkkGfkkfakv"
+        ]
+    },
+    "WGIRRILKYGKRS": {
+        "1": [
+            "wglrrllkygkrs"
+        ],
+        "0": []
+    },
+    "IKKILSKIKKLL": {
+        "1": [],
+        "0": [
+            "IKKILSkIKKLL"
+        ]
+    },
+    "IKKIVSKIKKVLK": {
+        "1": [],
+        "0": [
+            "IkKIVSKIKKVLK"
+        ]
+    },
+    "KGKPRPYPPRPPPHPRPIRV": {
+        "1": [],
+        "0": [
+            "kgkprpypprppphprpirv"
+        ]
+    },
+    "GKWMKLLKKILK": {
+        "1": [],
+        "0": [
+            "Gkwmkllkkilk"
+        ]
+    },
+    "GKWVKLLKKILK": {
+        "1": [],
+        "0": [
+            "Gkwvkllkkilk"
+        ]
+    },
+    "KWMKLLKKILK": {
+        "1": [],
+        "0": [
+            "kwmkllkkilk"
+        ]
+    },
+    "LRRLLlRWLRRLLRR": {
+        "1": [],
+        "0": [
+            "LRRllRWlRRLLRR"
+        ]
+    },
+    "ILKKIWKPIKKLF": {
+        "1": [],
+        "0": [
+            "ILKKIWKpIKKLF"
+        ]
+    },
+    "RWLKLPGRWLKL": {
+        "1": [],
+        "0": [
+            "RWLKLpGRWLKL"
+        ]
+    },
+    "RWFKFPGRWFKF": {
+        "1": [],
+        "0": [
+            "RWFKFpGRWFKF"
+        ]
+    },
+    "RWLRLPGRWLRL": {
+        "1": [],
+        "0": [
+            "RWLRLpGRWLRL"
+        ]
+    },
+    "RWFRFPGRWFRF": {
+        "1": [],
+        "0": [
+            "RWFRFpGRWFRF"
+        ]
+    },
+    "RWLHLPGRWLHL": {
+        "1": [],
+        "0": [
+            "RWLHLpGRWLHL"
+        ]
+    },
+    "RWFHFPGRWFHF": {
+        "1": [],
+        "0": [
+            "RWFHFpGRWFHF"
+        ]
+    },
+    "GIFSKLAPKKIKNLLISGLKG": {
+        "1": [],
+        "0": [
+            "GIFSKLApKKIKNLLISGLKG"
+        ]
+    },
+    "WGRRGWGPGRRYVRW": {
+        "1": [
+            "WGRRGWGpGRRYVRW"
+        ],
+        "0": []
+    },
+    "KKYRYHLKPF": {
+        "1": [
+            "kkyryhlkpf"
+        ],
+        "0": []
+    },
+    "RFLRRIFFFF": {
+        "1": [],
+        "0": [
+            "rflrriffff"
+        ]
+    },
+    "FFFFLRRIF": {
+        "1": [
+            "FFFFLrrIF"
+        ],
+        "0": [
+            "FFFFLrRIF",
+            "FFFFLRrIF"
+        ]
+    },
+    "WLLWIALRKKR": {
+        "1": [
+            "wllwialrkkr"
+        ],
+        "0": []
+    },
+    "WLVWIWRRR": {
+        "1": [
+            "wlvwiwrrr"
+        ],
+        "0": []
+    }
+}

dataset/train_set_llm_aug.json

@@ -0,0 +1,2719 @@
+{
+    "GIMSSLMKKLAAHIAK": {
+        "1": [
+            "GIMSSLMkKLAAHIAK",
+            "GIMSSLMKkLAAHIAK",
+            "GIMSSLMKKLAAHIAk",
+            "GIMSSLMkkLAAHIAK",
+            "GIMSSLMkKLAAHIAk",
+            "GIMSSLMKkLAAHIAk",
+            "GIMSSLMkkLAAHIAk",
+            "gIMSSLMkKLAAHIAK",
+            "GiMSSLMKkLAAHIAK",
+            "GIMsSLMKKlAAHIAK",
+            "GIMSSlmKkLAAHIAK",
+            "GIMSsLMkKLAAHIaK"
+        ],
+        "0": [
+            "gIMSSLMKKLAAHIAK",
+            "GImSSLMKKLAAHIAK",
+            "GIMsSLMKKLAAHIAK",
+            "GIMSSlMKKLAAHIAK",
+            "GIMSSLMkklAAHIAK"
+        ]
+    },
+    "ILGTILGLLKSL": {
+        "1": [
+            "iLGTILGLLKSL",
+            "ILgTILGLLKSL",
+            "ILGtILGLLKSL",
+            "ILGTILGLLKsL",
+            "ILGTILGLLKSl"
+        ],
+        "0": [
+            "ILGTILGLLkSL",
+            "ilgtilgllksl",
+            "ILGTILGLLksL",
+            "ILGTILGLLkSl",
+            "ILGTIlGLLkSL",
+            "ILGTiLGLLkSL"
+        ]
+    },
+    "KRLFKKLLKYLRKF": {
+        "1": [
+            "KRLFkkLLKYLRkF",
+            "krLFkkLLKYLRkF",
+            "krLFkkLLkYLrkF",
+            "KRlFkkLLKYLRkF",
+            "KRLfkkLLKYLRkF",
+            "KRLFkkllKYLRkF",
+            "KRLFkkLLkYLrkF",
+            "KRLFkkLLKyLRkF"
+        ],
+        "0": [
+            "KRLFKKLLKYLRkF",
+            "KRLFkKLLKYLRkF",
+            "KRLFKkLLKYLRkF",
+            "KRlFKKLLKYLRkF",
+            "kRLFkKLLKYLRKF",
+            "KRLFKKLLkYLRKF"
+        ]
+    },
+    "ILGTILGLLKGL": {
+        "1": [
+            "ilgtilgllkgl",
+            "IlGtiLgllkgl",
+            "ILgTilgllkgl",
+            "ilgtiLgllkgL",
+            "ilGTilgllkgl",
+            "ILgtilGllkgl"
+        ],
+        "0": [
+            "ILGTILGLLkGL",
+            "ILGTiLgLLKGL",
+            "ilgTILGLLKGL",
+            "ILgtiLGLLKGL",
+            "ILGTILGlLKGL",
+            "ILGTILgLLKGl"
+        ]
+    },
+    "IDWKKLLDAAKQIL": {
+        "1": [
+            "idwkklldaakqil",
+            "IDwkkllDaakQil",
+            "IDwkkllDAaKQIl",
+            "idwKKlldaaKqil",
+            "iDwkklLDAakqil",
+            "IDwkkLldaakqIl"
+        ],
+        "0": [
+            "IDWkkLLDAAkQIL",
+            "iDWKKLLDAAKQIL",
+            "IdWKKLLDAAKQIL",
+            "IDWKKLLdAAKQIL",
+            "IDWKKLLDAaKQIL",
+            "IDWKKLLDAAkQIL"
+        ]
+    },
+    "VWRRWRRFWRR": {
+        "1": [
+            "vWRRWRRFWRR",
+            "VwRRWRRFWRR",
+            "VWRRwRRFWRR",
+            "VWRRWRRfWRR",
+            "VWRRWRRFwRR"
+        ],
+        "0": [
+            "vwrrwrrfwrr",
+            "VWrrWrrFWrr",
+            "VWrrWrrFWRR",
+            "VWRRWrrFWrr",
+            "VWrrWRRFWrr",
+            "VwrrWrrFWrr",
+            "VWrrwrrFWrr"
+        ]
+    },
+    "FLKLLKKLL": {
+        "1": [
+            "fLKLLKKLL",
+            "FlKLLKKLL",
+            "FLkLLKKLL",
+            "flkllkkll",
+            "flKLLKKLL",
+            "fLkLLKKLL",
+            "FlkLLKKLL",
+            "flkLLKKLL",
+            "flkLLKKll"
+        ],
+        "0": [
+            "FLKlLKKLL",
+            "FLKLlKKLL",
+            "FLKLLkKLL",
+            "FLKLLKkLL",
+            "FLKLLKKlL",
+            "FLKLLKKLl",
+            "FLKllKKLL",
+            "FLKlLkKLL",
+            "FLKLlkKLL",
+            "FLKllkKLL",
+            "FLKllKKll"
+        ]
+    },
+    "KKVVFWVKFK": {
+        "1": [
+            "KKVVFWVKFk",
+            "KKVVFWVKfk",
+            "KKVVFWVkFk",
+            "KKVVFWvKFk",
+            "KKVVFwVKFk",
+            "KKVVfWVKFk"
+        ],
+        "0": [
+            "KKVVFWVKfK",
+            "KKVVFWVkFK",
+            "KKVVFWvKFK",
+            "KKVVFwVKFK",
+            "KKVVfWVKFK",
+            "KKVvFWVKFK",
+            "KKvVFWVKFK",
+            "KkVVFWVKFK",
+            "kKVVFWVKFK",
+            "kkVVFWVKFK",
+            "KKvvFWVKFK",
+            "KKVVfWVKfK",
+            "KKVVFwvKFK",
+            "KKVVFWVkfK"
+        ]
+    },
+    "KRIVKLILKWLR": {
+        "1": [
+            "KRIVkLILKWLR",
+            "KRIVKlILKWLR",
+            "KRIVklILKWLR",
+            "KRIVkLIlKWLR",
+            "KRIVkLILkWLR",
+            "KRIVKlilKWLR",
+            "KRIVkLiLkWLR"
+        ],
+        "0": [
+            "kRIVKLILKWLR",
+            "KRivKLILKWLR",
+            "KrIVKLILKWLR",
+            "KRIVkliLKWLR",
+            "KRIVKLiLKWLR"
+        ]
+    },
+    "KKVVFKVKFKK": {
+        "1": [
+            "kKVVFKVKFKk",
+            "kKVVFKVKFKK",
+            "KKVVFKVKFKk",
+            "KkVVFKVKFKK",
+            "KKVVFKVKFkK",
+            "kKVVFKVKFkK"
+        ],
+        "0": [
+            "kkVVFKVKFKK",
+            "KKVVFKVKFkk",
+            "kkVVFKVKFkk",
+            "KKVVFkVKFKK",
+            "kkVVFkVKFkk",
+            "kkvvfkvkfkk",
+            "KKvVFKVKFKK",
+            "KKVVfKVKFKK",
+            "KkVvFKVKFKK",
+            "KKVvFKVkFKK"
+        ]
+    },
+    "KWKSFLKTFKSAKKTVLHTALKAISS": {
+        "1": [
+            "kWKSFLKTFKSAKKTVLHTALKAISS",
+            "KwKSFLKTFKSAKKTVLHTALKAISS",
+            "KWkSFLKTFKSAKKTVLHTALKAISS",
+            "KWKsFLKTFKSAKKTVLHTALKAISS",
+            "KWKSFLkTFKSAKKTVLHTALKAISS"
+        ],
+        "0": [
+            "KWKSFLKTFKSAKkTVLHTALKAISS",
+            "KWKSFLKTFKsAKkTVLHTALKAISS",
+            "KWKSFLKTFKSAKktVLHTALKAISS",
+            "KWKSFLKTFKsAKktVLHTALKAISS",
+            "KWKSFLKTFKSaKKTVLHTALKAISS",
+            "KWKSFLKTfKSaKKTVLHTALKAISS",
+            "KWKSFLKTFKSaKKTvLHTALKAISS",
+            "KWKSFLKTfKSaKKTvLHTALKAISS",
+            "KWKSFLKTFKSAkKTVLHTALKAISS",
+            "kwksflktfksakktvlhtalkaiss",
+            "KWKSFLKTfKSAKKTVLHTALKAISS",
+            "KWKSFLKTFkSAKKTVLHTALKAISS",
+            "KWKSFLKTFKSAKKtVLHTALKAISS",
+            "KWKSFLKTFKSAKKTvLHTALKAISS",
+            "KWKSFLKTfkSAKKTVLHTALKAISS"
+        ]
+    },
+    "FLPLIIGALSSLLPKIF": {
+        "1": [
+            "fLPLIIGALSSLLPKIF",
+            "FLPLIIGALsSLLPKIF",
+            "FLPLIIGALSSLLPkIF",
+            "FLPLiIGALSSLLPKIF",
+            "FLPLIIgaLSSLLPKIF"
+        ],
+        "0": [
+            "FLPLIIGALSSLLPKiF",
+            "FLPLiiGALSSLLPKiF",
+            "FLPLIIGALSSLLPkiF",
+            "FLPLiIGALSSLLPKiF",
+            "FLPLIIGaLSSLLPKiF",
+            "FLPLIIGALSSllPKiF",
+            "FLPLiiGALSSLLPkiF"
+        ]
+    },
+    "KLKKLLKKWLKLLKKLLK": {
+        "1": [
+            "KLKKLlKKWLKlLKKLLk",
+            "KLKKlLKKWlKLLKkLLK",
+            "KLKkLLKkWLKlLKKlLK",
+            "KLkKLlKKwLKlLKkLLk",
+            "KlKkLlKkWlKlLkKlLk",
+            "KLKKLLKKWlkllkkllk",
+            "klKKLLKKWLKLLKKLLK",
+            "KLKKLLkkWLKLLKKLLK",
+            "kLKKLLKKWLKLLKKLLk",
+            "KLKKLLKKWLkLLKKLLK"
+        ],
+        "0": [
+            "KlkKLLKKWLKLLKKLLK",
+            "KLKklLKKWLKLLKKLLK",
+            "KLKKLLKKWLKLLKKlLK",
+            "klkKLLKKWLKLLKKLLK",
+            "KLKKLLKKWLKLLKKLlk"
+        ]
+    },
+    "KKAAAAAAAAAAAAWAAAAAAKKKK": {
+        "1": [
+            "kkAAAAAAAAAAAAWAAAAAAKKKK",
+            "KKAAAAAAAAAAAAwaAAAAAKKKK",
+            "KKAAAAAAAAAAAAWAaaAAAKKKK",
+            "KKAAAAAAAAAAAAWAAAaaAKKKK",
+            "KKAAAAAAAAAAAAWAAAAaaKKKK",
+            "kKAAAAAAAAAAAAwAAAAAAKKKK",
+            "KKAAAAAAAAAAAAwAAAaAAKKKK"
+        ],
+        "0": [
+            "KKaaAAAAAAAAAAWAAAAAAKKKK",
+            "KKAAaaAAAAAAAAWAAAAAAKKKK",
+            "KKAAAAaaAAAAAAWAAAAAAKKKK",
+            "KKAAAAAAaaAAAAWAAAAAAKKKK",
+            "KKAAAAAAAAaaAAWAAAAAAKKKK",
+            "KKAAAAAAAAAAaaWAAAAAAKKKK",
+            "KKAAAAAAAAAAAAWAAAAAakKKK",
+            "KKAAAAAAAAAAAAWAAAAAAKkkK",
+            "KKAAAAAAAaaAAAWAAAAAAKKKK",
+            "KKAAAAAAAAAaaAWAAAAAAKKKK"
+        ]
+    },
+    "FVPWFSKFLGRIL": {
+        "1": [
+            "fVPWFSKFLGRIL",
+            "FvPWFSKFLGRIL",
+            "FVpWFSKFLGRIL",
+            "FVPwFSKFLGRIL",
+            "FVPWfSKFLGRIL"
+        ],
+        "0": [
+            "FVPWFSkFLGRIL",
+            "FVPWFSKfLGRIL",
+            "FVPWFSKFlGRIL",
+            "FVPWFSKFLGrIL",
+            "FVPWFSKFLGRiL",
+            "FVPWFSKFLGRIl",
+            "FVPWFSKflGRIL",
+            "FVPWFSKfLGrIL",
+            "FVPWFSKFlGrIL",
+            "FVPWFSKFLgriL"
+        ]
+    },
+    "IRIKIRIK": {
+        "1": [
+            "irikirik",
+            "IRIkIrIK",
+            "irikIRIK",
+            "IRIKirik",
+            "IRikirIK",
+            "IRIkiriK",
+            "irIKIRik"
+        ],
+        "0": [
+            "IrIkIrIk",
+            "iRiKiRiK",
+            "iRIkIriK",
+            "IRiKirIk",
+            "iRIKIRIK",
+            "IRIKIRIk"
+        ]
+    },
+    "IIRKIIRK": {
+        "1": [
+            "iirkiirk",
+            "IirKIirK",
+            "IIRKiirk",
+            "IiRkIiRk",
+            "iIrKiIrK",
+            "Iirkiirk",
+            "iirkiirK"
+        ],
+        "0": [
+            "iIRKIIRK",
+            "IIRkIIRK",
+            "IirKIIRK",
+            "IIRKiiRK",
+            "iiRKiirk"
+        ]
+    },
+    "KKLFKKILKYL": {
+        "1": [
+            "KKLfKKILKYL",
+            "KKLFKKILkYL",
+            "KKLFKKIlKYL",
+            "KKLFkKILKYL",
+            "KKlFKKILKYL",
+            "KkLFKKILKYL",
+            "KKLFKkILKYL",
+            "kKLFKKILKYL",
+            "KKLFKKIlkYL",
+            "KKlFKkILkYL",
+            "KKLFKKilkYL",
+            "kklfkkilkyl",
+            "kkLfKKILKYL",
+            "KKLFKKilkyl",
+            "KKLFkkilkyl",
+            "KKLfkkilkyl",
+            "KKlfkkilkyl",
+            "Kklfkkilkyl",
+            "kklfKKILKYL",
+            "kklfkKILKYL",
+            "kklfkkILKYL",
+            "kklfkkiLKYL",
+            "kklfkkilKYL",
+            "kklfkkilkYL",
+            "kklfkkilkyL",
+            "KkLFkKILKYL",
+            "kKLFKKILkYL",
+            "KKLFKKIlKyL",
+            "KKLfKkILKYL",
+            "KKlFKKiLKYL"
+        ],
+        "0": [
+            "KKLFKkilkyl",
+            "KKLFKKiLKYL",
+            "KKLFKKILKyL",
+            "KKLFKKILKYl",
+            "KKlFKKILkYL",
+            "KKLFKKIlkyl",
+            "KKLFKKILkyL",
+            "kKLFKKILKYl",
+            "KkLFKKILKyL",
+            "KKLfKKILkyl",
+            "KKLFKKiLkyL"
+        ]
+    },
+    "KFFKRLLKSVRRAVKKFRK": {
+        "1": [
+            "KffkRLLKSVRRAVKKFRK",
+            "kFfkrLLkSVrrAVKKfrK",
+            "KFfKRLlKSvrRAVKkFRK",
+            "kFFKrLLkSVRravKkFrK",
+            "KfFKRlLKSVRRAVKKfRK"
+        ],
+        "0": [
+            "kFFkrLLkSVrrAVkkFrk",
+            "kffkrllksvrravkkfrk",
+            "kffkrLLksvRRaVKKfrk",
+            "KFFKrlLkSvrrAVKKfRk",
+            "kffKrlLkSVrravkKFRk",
+            "KfFkRllKSVrrAvkKfrK",
+            "kFFKrlLksvrraVkkfRk"
+        ]
+    },
+    "KWKSFLKTFKSLKKTVLHTLLKAISS": {
+        "1": [
+            "KWKSFLkTFKSLKKTVLHTLLKAISS",
+            "KWKSFLKTFKSLKkTVLHTLLKAISS",
+            "KWKSFLKTFKSLKKTVLHTLLkAISS",
+            "KWKSFLkTFKSLKkTVLHTLLKAISS",
+            "KWKSFLKTFKSLKkTVLHTLLkAISS",
+            "KWKSFlKTFKSLKKTVLHTLLKAISS",
+            "KWKSFLKTFKSlKKTVLHTLLKAISS",
+            "KWKSFLKTFKSLKKTVLHTlLKAISS",
+            "KWKSFLKTFKSlKKTVLHTlLKAISS",
+            "KWKSFlkTFKSLKKTVLHTLLKAISS",
+            "KWKSFLkTFKSLKKTVLHTLLkAISS",
+            "KWKSFLKTFKSlKKTVLHTLLkAISS",
+            "KWKSFLKTFKSLkKTVLHTlLKAISS",
+            "KWKSFLKTFKSLKKTVLHTlLkAISS"
+        ],
+        "0": [
+            "KWKSFLkTFKSLKkTVLHTLLkAISS",
+            "KWKSFLkTFkSLKkTVLHTLLkAISS",
+            "KWkSFLkTFkSLKkTVLHTLLkAISS",
+            "kWkSFLkTFkSLKkTVLHTLLkAISS",
+            "KWKSFlKTFKSlKKTVLHTLLKAISS",
+            "KWKSFlKTFKSlKKTVLHTlLKAISS",
+            "KWKSFlKTFKSlKKTVlHTlLKAISS",
+            "KWKSFlKTFKSlKKTVlHTllKAISS",
+            "KWKSFlkTFKSlKKTVLHTLLKAISS",
+            "KWKSFlKTFkSLKKTVLHTLLKAISS",
+            "KWKSFLKTFkSlkKTVLHTLLKAISS",
+            "KWKSFLKTFKsLkKTVLHTLLKAISS",
+            "kwkSFLKTFKSLKKTVLHTLLKAISS"
+        ]
+    },
+    "GWLDVAKKIGKAAFNVAKNFL": {
+        "1": [
+            "gWLDVAKKIGKAAFNVAKNFL",
+            "GwLDVAKKIGKAAFNVAKNFL",
+            "GWlDVAKKIGKAAFNVAKNFL",
+            "GWLdVAKKIGKAAFNVAKNFL",
+            "GWLDvAKKIGKAAFNVAKNFL"
+        ],
+        "0": [
+            "GWLDvAKKIGKAAFNvAKNFL",
+            "GWLDVAKKIGKAAFNvAKNFL",
+            "gWLDVAKKIGKAAFNvAKNFL",
+            "GwLDVAKKIGKAAFNvAKNFL",
+            "GWlDVAKKIGKAAFNvAKNFL",
+            "GWLdVAKKIGKAAFNvAKNFL",
+            "GWLDVaKKIGKAAFNvAKNFL"
+        ]
+    },
+    "GFGMALKLLKKVL": {
+        "1": [
+            "GfGmalkllkkvl",
+            "GfGMALKLLKKVL",
+            "gfGMALKLLKKVL",
+            "GfgMALKLLKKVL",
+            "GfGmALKLLKKVL",
+            "GfGMalkllkkvl",
+            "GfGMALKLLKKvl"
+        ],
+        "0": [
+            "GFGMALKLLKKVl",
+            "GFGMALKLLKKvL",
+            "GFGMALKLLKkVL",
+            "GFGMALKLLkKVL",
+            "GFGMALKLlKKVL",
+            "GFGMALKlLKKVL",
+            "GFGMALkLLKKVL",
+            "GFGMAlKLLKKVL",
+            "GFGMaLKLLKKVL",
+            "GFGmALKLLKKVL",
+            "gFGMALKLLKKVL",
+            "GFgMALKLLKKVL",
+            "gFgMALKLLKKVL",
+            "GFGmaLKLLKKVL",
+            "gFgmalkllkkvl"
+        ]
+    },
+    "RGLRRLGRKIAHGVKKYGPTVLRIIRIA": {
+        "1": [
+            "rGLRRLGRKIAHGVKKYGPTVLRIIRIA",
+            "RGLRRLGRKiAHGVKKYGPTVLRIIRIA",
+            "RGLRRLGRKIAHGVkkYGPTVLRIIRIA",
+            "RGLRRLGRKIAHGVKKYGPtVLRIIRIa",
+            "RGLRrLGRKiahGVKKYGPTVLRIIRIA"
+        ],
+        "0": [
+            "rglrrlgrkiahgvkkygptvlriiria",
+            "RGLRRLGRKIAHGVKKYGptvlriiria",
+            "RGLRRLGRKIAHgvkkygptvlriiria",
+            "RGLRRLGRKIAHGVKKYgpTVLRIIRIA",
+            "rglrrlGRKIAHGVKKYGPTVLRIIRIA",
+            "RGLRRlgrkiahgvkkYGptvlriiria"
+        ]
+    },
+    "KVLGRLVKVLGRLV": {
+        "1": [
+            "kVLGRLVKVLGRLV",
+            "kvLGRLVKVLGRLV",
+            "kVlGRLVKVLGRLV",
+            "kVLgRLVKVLGRLV",
+            "kVLGrLVKVLGRLV",
+            "kVLGRlVKVLGRLV"
+        ],
+        "0": [
+            "KVLGRLVkVLGRLV",
+            "kVLGRLVkVLGRLV",
+            "KvLGRLVkVLGRLV",
+            "KVlGRLVkVLGRLV",
+            "KVLgRLVkVLGRLV",
+            "KVLGrLVkVLGRLV",
+            "KVLGRlVkVLGRLV"
+        ]
+    },
+    "RRLFRRILRWL": {
+        "1": [
+            "RRLfRRILRWL",
+            "RRLFrRILRWL",
+            "rrlfrrilrwl",
+            "RRLfrRILRWL",
+            "RRLfRRILRwL",
+            "RRLFrRILrWL",
+            "rrlfrRILRWL",
+            "RRLfrRILRwL"
+        ],
+        "0": [
+            "rRLFRRILRWL",
+            "RrLFRRILRWL",
+            "RRlFRRILRWL",
+            "RRLFRrILRWL",
+            "RRLFRRiLRWL",
+            "RRLFRRIlRWL",
+            "RRLFRRILrWL",
+            "RRLFRRILRwL",
+            "RRLFRRILRWl",
+            "rRLFRrILRWL",
+            "RrLFRRiLRWL",
+            "RRlFRRIlRWL",
+            "RRLFRrILrWL",
+            "RRLFRRILRwl"
+        ]
+    },
+    "KWKSFLKTFKSAVKTVLHTALKAISS": {
+        "1": [
+            "KWKSFLKTFKSAvKTVLHTALKAISS",
+            "KWKSFLKTFKsAVKTVLHTALKAISS",
+            "KWKSFLkTfKSAVKTVLHTALKAISS",
+            "kWKSFLKTFKSAvkTVLHTALKAISS",
+            "KWKSFLKTFKSAVKTVLhTaLKAISS",
+            "KwKSfLKTFKsavKTVLHTALKAISS",
+            "KWKsFLKtFKSAVKtVLHTALKAISS"
+        ],
+        "0": [
+            "kwksflktfksavktvlhtalkaiss",
+            "KWKSFLKTFKSAvKTVLhtaLKAISS",
+            "kwksfLKTFKSAVKTVLHTALKAISS",
+            "KWKSFlktfkSAVKTVLHTALKAISS",
+            "kWKsfLKTFKSAVKTVLHTalkaiss",
+            "KwKsFLKTFksAVKtVLHTaLKAISs"
+        ]
+    },
+    "RRWVRRVRRVWRRVVRVVRRWVRR": {
+        "1": [
+            "rRWVRRVRRVWRRVVRVVRRWVrR",
+            "RRwVRRVRRVwRRVVRVVRRWVRR",
+            "RRWVRrVRRVWRRVVrVVRRWVrR",
+            "RRWVrRVRRVWRRVVRVVRRwVRR",
+            "rRWVRRVRRVWrrVVRVVRRWVRr"
+        ],
+        "0": [
+            "RRWVRRvRRVWRRVvRvVRRWvRR",
+            "RRWVRRvRRvWRRVvRvvRRWvRR",
+            "RRWvRRvRRvWRRvvRvvRRWvRR",
+            "RRWvRRVRRVWRRVvRVvRRWvRR",
+            "RRWVRRVRRvWRRvVRvvRRWvRR"
+        ]
+    },
+    "TVGGLVKWILKTVKKFA": {
+        "1": [
+            "tvgglvkwilktvkkfa",
+            "TVGGLVKWILkTVKKFA",
+            "tVGGLVKWILKTVKKFA",
+            "TVgGLVKWILKTVKKFA",
+            "TVGGlVKWILKTVKKFA",
+            "TVGGLvKWILKTVKKFA",
+            "TVGGLVKWILKTVkKFA"
+        ],
+        "0": [
+            "TVGGLVkWILkTVKkFA",
+            "TVGGLVkWILkTVKKFA",
+            "TVGGLVkWILKTVKkFA",
+            "TVGGLVKWILkTVKkFA",
+            "TVGGLVkWILkTVKkfA",
+            "tVGGLVkWILkTVKkFA"
+        ]
+    },
+    "INLKALAALAKKIL": {
+        "1": [
+            "INLKAlAALAKKIL",
+            "INLKALaALAKKIL",
+            "INLKALAALaKKIL",
+            "INLKaLAALAKKIL",
+            "INLkALaALAKKIL"
+        ],
+        "0": [
+            "iNLKALAALAKKIL",
+            "InLKALAALAKKIL",
+            "inLKALAALAKKIL",
+            "inlkalaalakkil",
+            "iNlKALAALAKKIL",
+            "iNLKAaAALAKKIL",
+            "iNlkALAALAKKIL",
+            "InLkALAALAKKIL",
+            "inlKALAALAKKIL"
+        ]
+    },
+    "FLSLIPKAIKAVGVKAKKF": {
+        "1": [
+            "FlSLIPKAIKAVGVKAKKF",
+            "FLsLIPKAIKAVGVKAKKF",
+            "FLSLiPKAIKAVGVKAKKF",
+            "FLSLIPkAkKAVGVKAKKF"
+        ],
+        "0": [
+            "FLSLIPkAIkAVGVkAkkF",
+            "FLSLIPkAIKAVGVKAKKF",
+            "FLSLIPkAIkAVGVKAkKF",
+            "FLSLIPKaiKAVGVKAKKF",
+            "FLSLIPKAIkAVgVKAKKF",
+            "FLSLIPkAIKAVGvkAKKF",
+            "fLSLIPKAIKAVGVKAKkF"
+        ]
+    },
+    "KKLLKLLKLLL": {
+        "1": [
+            "kkllkllklll",
+            "KkLLKLLKLLL",
+            "KkLLkLLKLLL",
+            "KkLlKLLKLLL",
+            "kKLLKLLKLLl",
+            "kkLLKLLKLLl",
+            "KkllKLLKLLL",
+            "kkLLkLLKLLL",
+            "KkllKLlKLLL",
+            "KkLLKLLkLLL",
+            "kklLKLLKLLL",
+            "KkLLkLLKLLl",
+            "kkLLKLlKLLL",
+            "kKlLKLLKLLL"
+        ],
+        "0": [
+            "kkLLKLLKLLL",
+            "KKLLKllKLLL",
+            "KKLLkllKLLL",
+            "KkllKlLKLLL",
+            "KKLLkllkLLL",
+            "KKllKllKLLL",
+            "KKlLkLlKlLL",
+            "KKLlkLLklLL",
+            "KklLKLLKllL",
+            "kkLLKLLKLll",
+            "kkLLkLLKLLl",
+            "KKllKLLklLL",
+            "KklLKlLKlLL",
+            "KKllKLlKLlL",
+            "KKLlkLLkLLl",
+            "KkllKllKLLL",
+            "KKllKllKlLL",
+            "kkLLkLLKLll",
+            "kkLLkLLkLLl",
+            "kKLLkllKLLl",
+            "KKLlkllkLLL",
+            "KkLlKlLkLlL",
+            "kKlLkLlKlLl",
+            "kKLLKLLKLLL",
+            "KKLLKLLKLLl",
+            "KKllKLLKLLL",
+            "KKLLkLLKLLL",
+            "KKLLKlLKLLL"
+        ]
+    },
+    "KKVVFKVKFK": {
+        "1": [
+            "KKVVfKvKFK",
+            "KKVvFKVKFK",
+            "kKVVFkVKFK",
+            "KKVvFkVkFK",
+            "kKvvFKVKFK"
+        ],
+        "0": [
+            "KKVVFKVKFk",
+            "kKVVFKVKFk",
+            "kkVVFKVKFk",
+            "KKVVfkvKFK",
+            "kKVVfkvKFK",
+            "kkVVFkVkFK",
+            "KKVVFKVKfK",
+            "kKVVfkKFKk",
+            "KKVvFkKfKf",
+            "kKVvFkvKfK"
+        ]
+    },
+    "LKLLKKLLKKLLKLL": {
+        "1": [
+            "LKlLKkLlkKLLkLL",
+            "lkLlKKlLKkLLKLL",
+            "LkLLkKLlKKlLKLl",
+            "LKLlKkLlKkLlKlL",
+            "lKLLkKLLkKLLkLL"
+        ],
+        "0": [
+            "KKkLLlLllLLLkKK",
+            "LKllkkllKKLLKLL",
+            "LkLlKkLlKkLLKLL",
+            "lKLLkKLLkklLKLL",
+            "lklLKKLLKKLlkll",
+            "LKLlkkllkKLLKLL"
+        ]
+    },
+    "KLKLLKLLKLLKLLK": {
+        "1": [
+            "kLKLLKLLKLLKLLK",
+            "KlKLLKLLKLLKLLK",
+            "KLKLLkLLKLLKLLK",
+            "KLKLLKLLkLLKLLK",
+            "KLKLLKLLKLLkLLK"
+        ],
+        "0": [
+            "KLkLLkLlkLLKlLK",
+            "KLKLLKLLKLLKLLk",
+            "kLkLlKLKlLKLKLL",
+            "KkLLkLLlkLkLLKk",
+            "KlkLkLKLLkLlKLk",
+            "klKLklKLLklKLLk"
+        ]
+    },
+    "KKKLLLLLLLLLKKK": {
+        "1": [
+            "KKKLLLLlllllKKK",
+            "kkkLLLlllLLLkkk"
+        ],
+        "0": [
+            "KKKLLlLLlLLlKKK",
+            "kkklllLLLLLLkkk"
+        ]
+    },
+    "KKFKKTAKWLIKSAWLLLKSLALKMK": {
+        "1": [
+            "kkfkktakwliksawlllkslalkmk",
+            "KKFkktaKwliksawlllkslalkmk",
+            "kkfKKtaKwliksawlllkslalkmk",
+            "kkfkkTAKwliksawlllkslalkmk",
+            "kkfkktaKWliksawlllkslalkmk",
+            "kkfkktaKwLIksawlllkslalkmk"
+        ],
+        "0": [
+            "KKFKKTAkwliksawlllkslalkmk",
+            "KKFKKTAKwliksawlllkslalkmk",
+            "KKFKKTAKwlIksawlllkslalkmk",
+            "KKFKKTAKwliKSAWLLLKSLALKMK",
+            "KKFKKTAKwliksawlllKSLALKMK"
+        ]
+    },
+    "WWWLRRRW": {
+        "1": [
+            "wWWLRRRW",
+            "WwWLRRRW",
+            "WWwLRRRW",
+            "WWWlRRRW",
+            "WWWLrRRW"
+        ],
+        "0": [
+            "wwwlrrrw",
+            "Wwwlrrrw",
+            "wWwlrrrw",
+            "wwWlrrrw",
+            "wwwLrrrw",
+            "wwwlRrrw"
+        ]
+    },
+    "RRRWWWWV": {
+        "1": [
+            "rRRWWWWV",
+            "RRRWWWWv",
+            "RRRwWWWV",
+            "RrrWWWWV",
+            "RRRwwWWv"
+        ],
+        "0": [
+            "rrrwwwwv",
+            "rrRWWWWv",
+            "RRRwwwwV",
+            "rrrWWWwv",
+            "RrwWWWWv",
+            "rrRwWWwv"
+        ]
+    },
+    "KWFRVYRGIYRRR": {
+        "1": [
+            "KwFRVYRGIYRRR",
+            "KWfRVYRGIYRRR",
+            "KWFRvYRGIYRRR",
+            "KWFRVyrGIYRRR",
+            "KWFrVYRGiYRRR"
+        ],
+        "0": [
+            "kwfrvyrgiyrrr",
+            "kwfrvyrgiyrrR",
+            "kWfrvyrgiyrrr",
+            "kwfRvyrgiyrrr",
+            "kwfrvyRgiyrrr",
+            "kwfrvyrgIyRrr"
+        ]
+    },
+    "RRRYIGRYVRFWK": {
+        "1": [
+            "RRRYIGRYvRFWK",
+            "rRRYIGRYVRFWK",
+            "RRRyIGRYVRFWK",
+            "RRRYIGRyVRFWK",
+            "RRRYIGRYVrFWK"
+        ],
+        "0": [
+            "rrryigryvrfwk",
+            "rrryigrYVRFWK",
+            "RRRYIgryVRFWK",
+            "rrryiGRYvRFWK",
+            "rRRyIGRYVrfwK",
+            "RRRYIGRyVrFwK"
+        ]
+    },
+    "GKIIKLKASLKLL": {
+        "1": [
+            "gkiiklkaslkll",
+            "GkIIKLKASLKLL",
+            "GKiIKLKASLKLL",
+            "GKIiKLKASLKLL",
+            "GKIIkLKASLKLL",
+            "GKIIKlKASLKLL"
+        ],
+        "0": [
+            "GKIIKLkASLKLL",
+            "GKIIKLKaSLKLL",
+            "GKIIKLKAsLKLL",
+            "GKIIKLKASlKLL",
+            "GKIIKLKASLkLL"
+        ]
+    },
+    "KLFKKLFKKLFK": {
+        "1": [
+            "KlFkKlFkKlFk",
+            "KlfKKlFKKlFk",
+            "kLfKkLfKkLfK",
+            "KLFkKLFkKLFk",
+            "kLFKkLFKkLFK"
+        ],
+        "0": [
+            "kLFkkLFkkLFk",
+            "klfkkfkkfkkk",
+            "KlfklfklfklK",
+            "klkklkkklkkk"
+        ]
+    },
+    "GFFALIPKIISSPLFKTLLSAV": {
+        "1": [
+            "gFFALIPKIISSPLFKTLLSAV",
+            "GFFALIPkiISSPLFKTLLSAV",
+            "GFFALIPKIIsSPLFKTLLSAV",
+            "GFFALiPKIISSPLFKTLLSAV",
+            "GFFALIPKIISSPLFKtLLSAV"
+        ],
+        "0": [
+            "GFFALIpKIISSPLFKTllSAV",
+            "GFFALIPKIISSPLFKTLLsaV",
+            "GFFALIPKIISSPlFKTLLSAV",
+            "GfFALIPKIISSPLfKTLLSAV",
+            "GFFALIPKIISSPLFKTlLSAV",
+            "GFFALIPkIISSPLFKTLLSAV"
+        ]
+    },
+    "KGFFALIPKIISSPLFKTLLSAV": {
+        "1": [
+            "kGFFALIPKIISSPLFKTLLSAV",
+            "KGfFALIPKIISSPLFKTLLSAV",
+            "KGFFALIPkIISSPLFKTLLSAV",
+            "KGFFALIPKIISsPLFKTLLSAV",
+            "KGFFALIPKIISSPLfKTLLSAV"
+        ],
+        "0": [
+            "KGFFALIpKIISSPLFKTllSAV",
+            "KGFFALIPKIISSPLFKTllSAv",
+            "KGFFALIPKIISSPLFKTlLSAV",
+            "KGFFALIpKIISSPLFKTlLSAV",
+            "KGFFALIpKIISSPLFktLLSAV",
+            "KGfFALIPKIISsPLFKTllSAV"
+        ]
+    },
+    "RGLRRLGRKIAHGVKKYG": {
+        "1": [
+            "rglrrlgrkiahgvkkyg",
+            "rGLRRLGRKIAHGVKKYG",
+            "RgLRRLGRKIAHGVKKYG",
+            "RGlRRLGRKIAHGVKKYG",
+            "RGLrRLGRKIAHGVKKYG",
+            "RGLRrLGRKIAHGVKKYG"
+        ],
+        "0": [
+            "RGLRRlGRKIAHGVKKYG",
+            "RGLRRLgRKIAHGVKKYG",
+            "RGLRRLGrKIAHGVKKYG",
+            "RGLRRLGRkIAHGVKKYG",
+            "RGLRRLGRKiAHGVKKYG"
+        ]
+    },
+    "FLGGLIKIVPAMICAVTKKC": {
+        "1": [
+            "flGGlikivpamicavtkkc",
+            "flGGLikivpamicavtkkc",
+            "flGGliKivpamicavtkkc",
+            "flGGlikivpamicavtKkc",
+            "flGGlikivpamicavtkkC"
+        ],
+        "0": [
+            "FLGGlikivpamicavtkkc",
+            "flgglikivpamicavtkkc",
+            "flGgLIKivpamicavtkkc",
+            "fLGGlikivpamicavtkkc",
+            "FlGGLikivpamicavtkkc"
+        ]
+    },
+    "AKRLKKLAKKIWKWK": {
+        "1": [
+            "aKRLKKLAKKIWKWK",
+            "AKRlKKLAKKIWKWK",
+            "AKRLKKLAKkIWKWK",
+            "AKRLKklAKKIWKWK",
+            "aKRLKKlAKKIWKkK"
+        ],
+        "0": [
+            "AkRLkkLAkkIWkWk",
+            "AKrLkkkAKkIWkWk",
+            "AkRLKkLAKKkwKWK",
+            "akRLkKLAkkIWKWK",
+            "AkRLkklAKKIWKWk",
+            "akrLkKkAKkIWKWk"
+        ]
+    },
+    "VDKPPYLPRPRPIRRPGGR": {
+        "1": [
+            "VDkPPYLPRPRPIRRPGGR",
+            "VDKpPYLPRPRPIRRPGGR",
+            "VDKPPyLPRPRPIRRPGGR",
+            "VDKPPYLPRPRPIRRPGgR",
+            "VDKPPYLPRPRPIRRPgGR"
+        ],
+        "0": [
+            "VDKPPYLPrPRPIrRPGGR",
+            "VDKPPYLPrPRPIRrPGGR",
+            "VDKPPYLPrPRPIRRPGGr",
+            "VDKPPYLPRPrPIrRPGGR",
+            "VDKPPYLPRPrPIRrPGGR",
+            "VDKPPYLPRPrPIRRPGGr",
+            "VDKPPYLPRPRPIrRPGGr",
+            "VDKPPYLPRPRPIRrPGGr",
+            "VDkPPYLPrPrPIrrPGGr",
+            "VDKPPYLPRPRPIrrPGGR",
+            "VDKPPYLPRPRPIrrPGGr",
+            "VDKPPYLPrPrPIRrPGGR",
+            "VDKPPYLPRPrPIrrPGGR"
+        ]
+    },
+    "GIGAVLKVLTTGLPALISWIKRKRQQ": {
+        "1": [
+            "GIGAVlKVLTTGlPALISWiKRKRQQ",
+            "gigavlkvlttglpaliswikrkrqq",
+            "GIGAvLKVLTTgLPALISwIKRKRQQ",
+            "GIGAVLKVlTTGLPALISWIKRkRQQ",
+            "GIGAVLKvLTTGLPAlISWiKRKRQQ",
+            "gIGAVLkVLTTGLPALiSWIKRKRQQ",
+            "GIGaVlKVLTTGlPALISWikRKRQQ"
+        ],
+        "0": [
+            "GIGAVLKVLTTgLPALIsWIKRKRQQ",
+            "GIGAVLKvLTTGLpALISWIKRKRqQ",
+            "gIgAVLKVLTTGLPALISWiKRKRQQ",
+            "GIGAVLKVLTtGLPALISWIKrKRQQ",
+            "GIGAVlKVLTtGLPALiSWIKRKRQq"
+        ]
+    },
+    "FWGALAKGALKLIPSLFSSFSKKD": {
+        "1": [
+            "fwGalakGalklipslfssfskkd",
+            "fwgalakgalklipslfssfskkd",
+            "FwGalakGalklIPSLFSSFSKKD",
+            "FWgALaKGALKliPSlFssfskkd",
+            "fwGAlakgalKLIPsLfSSFSKkD",
+            "FwgaLAkgaLKlipsLfssfSKKd"
+        ],
+        "0": [
+            "FwgalakGAlklIpslFsSFSkKd",
+            "FWGALaKGalkLIPsLFSSfSkkD",
+            "fWGalAKgaLklIpSLfssFSKKd",
+            "FWgALAkgaLkliPSLFSsfSkkD",
+            "FWgaLAKgaLKLIpslFSSfskkd"
+        ]
+    },
+    "IRVKIRVKIRVK": {
+        "1": [
+            "irvkirvkirvk",
+            "irvkirvKirvk",
+            "iRvKiRvKiRvK",
+            "IRvkIRvkIRvk",
+            "IRVkiRVkiRVk",
+            "irvkiRvKirVK"
+        ],
+        "0": [
+            "IrVkIrVkIrVk",
+            "irVKIRVKIRVK",
+            "iRvkIrVKIRVk",
+            "IRvKIrVKiRvK",
+            "iRvKirvKIRVk"
+        ]
+    },
+    "LIKKALAALAKLNI": {
+        "1": [
+            "lIKKALAALAKLNI",
+            "LIkKALAALAKLNI",
+            "LIKkALAALAKLNI",
+            "LIKKAlAALAKLNI",
+            "LIKKALAAlAKLNI"
+        ],
+        "0": [
+            "likkalaalaklni",
+            "likkaLAALAKLNI",
+            "LIKKalAALAKLNI",
+            "LIKKALaaLAKLNI",
+            "LIKKALAALaKLNI",
+            "lIKKALAALAKLNi"
+        ]
+    },
+    "RSMRLSFRARGYGFR": {
+        "1": [
+            "rsmrlsfrarGyGfr",
+            "rsmrlSfRARGyGfR",
+            "RSmRLSFRARGyGfr",
+            "RSMRLsfRaRGygFR",
+            "RSmRLsfRARgyGFR",
+            "RSmrLSFRaRGYGfR"
+        ],
+        "0": [
+            "RSMRLSFRaRgYGFR",
+            "rsmRLSFRARGygFr",
+            "RSmrlSFRARgYgFr",
+            "RSMRLsFrarGyGFr",
+            "rsmRLSFrARGygfR"
+        ]
+    },
+    "GLLKRIKTLL": {
+        "1": [
+            "Gllkriktll",
+            "gllkriktll",
+            "gLlkriktll",
+            "gllkriKtll",
+            "gllkrikTlL",
+            "gllkRikTll"
+        ],
+        "0": [
+            "GLLkRIkTLL",
+            "GLlKRIKTLL",
+            "GLLKRIkTLl",
+            "GLLKrIKTLL",
+            "GllkriktLl",
+            "GLLKriKTLL"
+        ]
+    },
+    "KKLFKKILRYL": {
+        "1": [
+            "KKLfKKILRYL",
+            "KKLFKKiLRYL",
+            "KKLFKKILRyL",
+            "KKlFKKILRYL",
+            "KKLFKKIlRYL",
+            "KKLFKKILRYl"
+        ],
+        "0": [
+            "KKLFKkilryl",
+            "kklfkkilryl",
+            "kkLFKKILRYL",
+            "KKLFKKILryL",
+            "KKlfkkiLRYL",
+            "kKlFKKILRYL",
+            "kklFkkilryl"
+        ]
+    },
+    "FQWQRNMRKVR": {
+        "1": [
+            "fqwqrnmrkvr",
+            "Fqwqrnmrkvr",
+            "fQwqrnmrkvr",
+            "fqWqrnmrkvr",
+            "fqwQrnmrkvr",
+            "fqwqRnmrkvr"
+        ],
+        "0": [
+            "fQWQRNMRKVR",
+            "FqWQRNMRKVR",
+            "FQwQRNMRKVR",
+            "FQWqRNMRKVR",
+            "FQWQrNMRKVR"
+        ]
+    },
+    "KKKKKKAAFAAWAAFAA": {
+        "1": [
+            "kkkkkkAAFAAWAAFAA",
+            "KKKKKKaafaaWaafaa",
+            "KkKkKkAaFaAwAaFaA",
+            "KKKKKKAAFAAwaafaa",
+            "KKKKKKAAfaawaafAA"
+        ],
+        "0": [
+            "kkkkkkaafaawaafaa",
+            "KKKKKKAAFAAwAAFAA",
+            "kkKKKKAAFAAWAAFAA",
+            "KKKKKKAAFAAWAAFaa",
+            "KKKKKKaaFaaWaaFaa",
+            "KKkkkkAAFAAWAAFAA"
+        ]
+    },
+    "RRWWRF": {
+        "1": [
+            "rRWWRF",
+            "RrWWRF",
+            "RRWwRF",
+            "RRWWrF",
+            "RRWWRf",
+            "rrWWRF",
+            "rRWwRF",
+            "rRWWrF",
+            "rRWWRf",
+            "RrWwRF"
+        ],
+        "0": [
+            "rrwwrf",
+            "RRwWRF",
+            "rRwWRF",
+            "RrwWRF",
+            "RRwwRF",
+            "RRwWrF",
+            "RRwWRf"
+        ]
+    },
+    "KWKSFLKTFKSALKTVLHTALKAISS": {
+        "1": [
+            "KWKSFLKTFKSAlKTVLHTALKAISS",
+            "KWKSFlKTFKSALKTVLHTALKAISS",
+            "KWKSFLKtFKSALKTVLHTALKAISS",
+            "KWKSFLKTFKSaLKTVLHTALKAISS",
+            "KWKSFLKTFKSALKTVlHTALKAISS",
+            "KWKSFLKTFKSALKTVLHtALKAISS"
+        ],
+        "0": [
+            "kWKSFLKTFKSALKTVLHTALKAISS",
+            "KwKSFLKTFKSALKTVLHTALKAISS",
+            "KWkSFLKTFKSALKTVLHTALKAISS",
+            "KWKSfLKTFKSALKTVLHTALKAISS"
+        ]
+    },
+    "KWKSFLKTFKSAAKTVLHTALKAISS": {
+        "1": [
+            "KWKSFLKTFKSAaKTVLHTALKAISS",
+            "KWKSFLKTFKsaAKTVLHTALKAISS",
+            "KWKSFLKTFKSAAkTVLHTALKAISS",
+            "KWKSFLKTFKSAAKTvLHTALKAISS",
+            "KWKSFLKTFKSAAKTVLHTaLKAISS",
+            "KWKSFLKTFKSAAKTVLHTALKaISS"
+        ],
+        "0": [
+            "kWKSFLKTFKSAAKTVLHTALKAISS",
+            "KwKSFLKTFKSAAKTVLHTALKAISS",
+            "KWKSfLKTFKSAAKTVLHTALKAISS",
+            "KWKSFLKTFKSAAKTVLHTALKAIsS",
+            "KWKSFLKTFKSAAKTVLHTALKAISs"
+        ]
+    },
+    "KWKSFLKTFKSASKTVLHTALKAISS": {
+        "1": [
+            "kWKSFLKTFKSASKTVLHTALKAISS",
+            "KwKSFLKTFKSASKTVLHTALKAISS",
+            "KWKsFLKTFKSASKTVLHTALKAISS",
+            "KWKSFlKTFKSASKTVLHTALKAISS",
+            "KWKSFLKtFKSASKTVLHTALKAISS"
+        ],
+        "0": [
+            "KWKSFLKTFKSAsKTVLHTALKAISS",
+            "kWKSFLKTFKSAsKTVLHTALKAISS",
+            "KWkSFLKTFKSAsKTVLHTALKAISS",
+            "KWKSfLKTFKSAsKTVLHTALKAISS",
+            "KWKSFLkTFKSAsKTVLHTALKAISS",
+            "KWKSFLKTfKSAsKTVLHTALKAISS"
+        ]
+    },
+    "KWKSFLKTFKLAVKTVLHTALKAISS": {
+        "1": [
+            "KWKSFLKTFKlAVKTVLHTALKAISS",
+            "KWKSFLKtFKLAVKTVLHTALKAISS",
+            "KWKSFLKTFKLAVKtvLHTALKAISS",
+            "KWKSFLKTFKLAvKTVLHTALKAISS",
+            "kWKSFLKTFKLAVKTVLHTALKAISS",
+            "KWKsFLKTFKLAVKTVLHTALKAISS"
+        ],
+        "0": [
+            "KWKSFLKTFkLAVKTVLHTALKAISS",
+            "KWKSFLKTFKLAVKTVLhTALKAISS",
+            "KWKSFlKTFKLAVKTVLHTALKAISS",
+            "KWKSFLKTFKLAVKTVLHTALkAISS",
+            "KWKSFLKTFKLAVKTVLHTAlKAISS"
+        ]
+    },
+    "KWKSFLKTFKVAVKTVLHTALKAISS": {
+        "1": [
+            "KWKSFLKTFKvAVKTVLHTALKAISS",
+            "kWKSFLKTFKVAVKTVLHTALKAISS",
+            "KWKSfLKTFKVAVKTVLHTALKAISS",
+            "KWKSFLKTFKVaVKTVLHTALKAISS",
+            "KWKSFLKTFKVAVKtVLHTALKAISS",
+            "KWKSFLKTFKVAVKTVLHtALKAISS"
+        ],
+        "0": [
+            "KWKSFlKTFKVAVKTVLHTALKAISS",
+            "KWKSFLkTfKVAVKTVLHTALKAISS",
+            "KWKSFLKTFKVavKTVLHTALKAISS",
+            "KWKSFLKTFKVAVKTVlHTALKAISS",
+            "KWKSFLKTFKVAVKTVLHTALKaiSS"
+        ]
+    },
+    "KWKSFLKTFKAAVKTVLHTALKAISS": {
+        "1": [
+            "KWKSFLKTFKaAVKTVLHTALKAISS",
+            "KWKSFLKtFKAAVKTVLHTALKAISS",
+            "KWKSFLKTFkAAVKTVLHTALKAISS",
+            "KWKSFLKTFKAvVKTVLHTALKAISS",
+            "KWKSFLKTFKAAVKtVLHTALKAISS",
+            "KWKSFLKTFKAAVKTvLHTALKAISS"
+        ],
+        "0": [
+            "kWKSFLKTFKAAVKTVLHTALKAISS",
+            "KWKsFLKTFKAAVKTVLHTALKAISS",
+            "KWKSFlKTFKAAVKTVLHTALKAISS",
+            "KWKSFLKTfKAAVKTVLHTALKAISS",
+            "KWKSFLKTFKAAVKTVLhTALKAISS"
+        ]
+    },
+    "KWKSFLKTFKKAVKTVLHTALKAISS": {
+        "1": [
+            "KWKSFLKTFKkAVKTVLHTALKAISS",
+            "kWKSFLKTFKKAVKTVLHTALKAISS",
+            "KWkSFLKTFKKAVKTVLHTALKAISS",
+            "KWKSFLkTFKKAVKTVLHTALKAISS",
+            "KWKSFLKTFKKAVkTVLHTALKAISS"
+        ],
+        "0": [
+            "KwKSFLKTFKKAVKTVLHTALKAISS",
+            "KWKsFLKTFKKAVKTVLHTALKAISS",
+            "KWKSfLKTFKKAVKTVLHTALKAISS",
+            "KWKSFlKTFKKAVKTVLHTALKAISS",
+            "KWKSFLKtFKKAVKTVLHTALKAISS"
+        ]
+    },
+    "GFKMALKLLKKVL": {
+        "1": [
+            "GFKMALKLLKKvl",
+            "GFKMALklLKKVL",
+            "GFKMALKLLKkVl",
+            "GFKMALKLLKkvl"
+        ],
+        "0": [
+            "GFkMALKLLKKVL",
+            "GfkMALKLLKKVL",
+            "GfKMALKLLKKVL",
+            "GFkMaLKLLKKVL",
+            "GFKMALKLLkKVL",
+            "GfkMaLKLLKKVL",
+            "GfKmaLKLLKKVL"
+        ]
+    },
+    "AFGMALKLLKKVL": {
+        "1": [
+            "AFGMALKLLKKvL",
+            "AFGMALKLLKKVl",
+            "AFGmALKLLKKVL",
+            "AFGMaLKLLKKVL",
+            "AFGMAlKLLKKVL"
+        ],
+        "0": [
+            "aFGMALKLLKKVL",
+            "aFGMALkLLKKVL",
+            "AFgMALKLLKKVL",
+            "aFGMALKLLKKvL",
+            "AFGMALKllKKVL",
+            "AFGMALKLLkkVL"
+        ]
+    },
+    "RRLLRLLRLLL": {
+        "1": [
+            "rrLLrLLrLLL",
+            "rrlLrLLrLLL",
+            "rrLlrLLrLLL",
+            "rRLLrlLrLLL",
+            "rrLLrLlrLLL",
+            "rRlLrLLrLLL"
+        ],
+        "0": [
+            "rRLLRLLRLLL",
+            "RrLLRLLRLLL",
+            "RRlLRLLRLLL",
+            "rRlLRLLRLLL",
+            "RRLLrLLRLLL"
+        ]
+    },
+    "KKIIKIIKIII": {
+        "1": [
+            "kkIIkIIkIII",
+            "kkiIkIIkIII",
+            "kkIikIIkIII",
+            "kkIIkiIkIII",
+            "kkIIkIikIII",
+            "kkIIkIIkiII"
+        ],
+        "0": [
+            "kKIIKIIKIII",
+            "KkIIKIIKIII",
+            "KKIIKIIkIII",
+            "KKiiKIIKIII",
+            "KKIIKIIKIIi"
+        ]
+    },
+    "RRIIRIIRIII": {
+        "1": [
+            "RRIIRIIRIII",
+            "RRIIRIIRIIi",
+            "RRIiRIIRIII",
+            "RRIIRiIRIII",
+            "RRIIRIiRIII"
+        ],
+        "0": [
+            "rrIIrIIrIII",
+            "rRIIRIIRIII",
+            "RrIIRIIRIII",
+            "RRIIrIIRIII",
+            "RRIIRIIrIII",
+            "rRIIrIIRIII"
+        ]
+    },
+    "ALWKKLLKK": {
+        "1": [
+            "AlWkkllkk",
+            "aLWkkllkk",
+            "AlWkkllkK",
+            "ALWkkllkk",
+            "alwkkllKK",
+            "alWkkllKk"
+        ],
+        "0": [
+            "ALwkkLLKK",
+            "aLwKkLLKK",
+            "ALWKKllKK",
+            "ALWKKLLkk",
+            "ALwKKLLKK"
+        ]
+    },
+    "KRFKKFFKKVKKSVKKRLKKIFKKPMVIGVTIPF": {
+        "1": [
+            "kRFKKFFKKVKKSVKKRLKKIFKKPMVIGVTIPF",
+            "KRFKKFFKKVKKSVKKRLKKIFKKPMVIGVTIpF",
+            "KRFKKFFKKVKKSVKKRlKKIFKKPMVIGVTIPF",
+            "KRFKKFFKKvKKSVKKRLKkIFKKPMVIGVTIPF",
+            "KRFKKFFKKVKKSVKKRLKKIFKKPMVIGvtIPF"
+        ],
+        "0": [
+            "krfkkffkkvkksvkkrlkkifkkpmviGvtipf",
+            "Krfkkffkkvkksvkkrlkkifkkpmvigvtipf",
+            "krfkkffkkvkksvkkrLkkifkkpmviGvtipf",
+            "krfkkffkkVkksvkkrlkkifkkpmvigvtipF",
+            "krfkkffkkvkksvKkrlkkifkkpmviGvtipf"
+        ]
+    },
+    "KKRLKKIFKKPMVIGVTIPF": {
+        "1": [
+            "kKRLKKIFKKPMVIGVTIPF",
+            "KKRLKKIFKKPMVIGVTIPf",
+            "kkRLKKIFKKPMVIGVTIPF",
+            "KKRLKKIFKKPMVIGVTIpf",
+            "kKRLKKIFKKPMVIGVTIPf"
+        ],
+        "0": [
+            "kkrlkkifkkpmviGvtipf",
+            "Kkrlkkifkkpmvigvtipf",
+            "kkrlKkifkkpmvigvtipf",
+            "kkrlkkifkKpmvigvtipf",
+            "kkrlkkifkkpmvIgvtipf",
+            "kkrlkkifkkpmvigVtipf"
+        ]
+    },
+    "RLFRRVKKVAGKIAKRIWK": {
+        "1": [
+            "rLFRRVKKVAGKIAKRIWK",
+            "RLfrRVKKVAGKIAKRIWK",
+            "RLFRRVKKVAGKiAKRIWK",
+            "RLFRRVKKVAGKIAKrIWK",
+            "RLFRRvkkVAGKIAKRIWK"
+        ],
+        "0": [
+            "rlfrrvkkvagkiakriwk",
+            "rlFrrVKKVAGKIAKRIWK",
+            "RLFrRVKKVAGKIAkRIWK",
+            "RLFRRvKKVagkIAKRIWK",
+            "RLFRRVKKVAgkiakriWK",
+            "RLFRRVKKvAGKIAKrIwK"
+        ]
+    },
+    "FIRRIARLLRRIF": {
+        "1": [
+            "fIRRIARLLRRIF",
+            "FiRRIARLLRRIF",
+            "FIrRIARLLRRIF",
+            "FIrrIARLLRRIF",
+            "FIRRiARLLRRIF"
+        ],
+        "0": [
+            "firriarllrrif",
+            "fiRRIARLLRRIF",
+            "firRIARLLRRIF",
+            "firrIARLLRRIF",
+            "firriARLLRRIF",
+            "firriaRLLRRIF"
+        ]
+    },
+    "GIGAVLKVLALISWIKRKR": {
+        "1": [
+            "gIGAVLKVLALISWIKRKR",
+            "GIGaVLKVLALISWIKRKR",
+            "GIGAVLkVLALISWIKRKR",
+            "GIGAVLKVLAlISWIKRKR",
+            "GIGAVLKVLALISWiKRKR"
+        ],
+        "0": [
+            "GIGAvLKvLAlISWIkRKR",
+            "GIGAvLKVLALISWIKRKR",
+            "GIGAVLKvLALISWIKRKR",
+            "GIGAVLKVLAlISWIKRKR"
+        ]
+    },
+    "FKCRRWQWRMKKLG": {
+        "1": [
+            "fkcrrwqwrmkklg",
+            "Fkcrrwqwrmkklg",
+            "fKcrrwqwrmkklg",
+            "fkCrrwqwrmkklg",
+            "fkcRrwqwrmkklg",
+            "fkcrRwqwrmkklg"
+        ],
+        "0": [
+            "fKCRRWQWRMKKLG",
+            "FkCRRWQWRMKKLG",
+            "FKcRRWQWRMKKLG",
+            "FKCrRWQWRMKKLG",
+            "FKCRrWQWRMKKLG"
+        ]
+    },
+    "WKKLKKLLKKLKKL": {
+        "1": [
+            "WKKlKKLLKKLKKL",
+            "WKKLKKlLKKLKKL",
+            "WKKLKKLlKKLKKL",
+            "WKKLKKLLKKlKKL",
+            "WKKLKKLLKKLKKl"
+        ],
+        "0": [
+            "Wkklkkllkklkkl",
+            "wKKLKKLLKKLKKL",
+            "wkKLKKLLKKLKKL",
+            "wkkLKKLLKKLKKL",
+            "wkklKKLLKKLKKL",
+            "wkklkKLLKKLKKL"
+        ]
+    },
+    "KFWSLLKKALRLWANVL": {
+        "1": [
+            "kFwSLLkKALRLwANVL",
+            "kFwSLLkKALRLwANvL",
+            "KFwSLLkKALRLwANVL",
+            "kFwSLLKKALRLwANVL",
+            "kFWSLLkKALRLwANVL",
+            "kFWsLLkKALRLwANVL"
+        ],
+        "0": [
+            "KFWSLLKKALRLWANVL",
+            "kFWSLLKKALRLWANVL",
+            "kfWSLLKKALRLWANVL",
+            "KFWSLLKKALRLWANvL",
+            "KFWSllKKALRLWANVL"
+        ]
+    },
+    "KFWKLLKKALRLWAKVL": {
+        "1": [
+            "kFwKLLkKALrLwAkVL",
+            "KfWkLlKkAlRlWAKVL",
+            "kFWKLLkKAlRLwAKvL",
+            "KfWKLLKkALrLWaKVl",
+            "KFwKLlKKaLRlWAkvL",
+            "kfWKLLkkALrLWAKvL"
+        ],
+        "0": [
+            "kFWKlLKkAlrLWAkVL",
+            "kFWKLlkkalRLWAKVL",
+            "KfwkllKKALRLWAKvL",
+            "KFwkllkKALRLWAKVl",
+            "kfwKLLKKALRLWAkvl",
+            "KFWKLLKKalrlwaKVL"
+        ]
+    },
+    "WFKKLLKKALRLWKKVL": {
+        "1": [
+            "wFKKlLKkAlrLWKkVL",
+            "wFKKlLKKAlrlWKkVL",
+            "wFKKlLKkAlRlWKkVL",
+            "wFKKlLkkAlrLWKkVL",
+            "wfKKlLKkAlrLWKkVL",
+            "wFKKlLKkALrLWkkVL"
+        ],
+        "0": [
+            "WFKKLlKKALRLWKKVL",
+            "WFKKLLKkaLRLWKKVL",
+            "WFkKLLKKALRLWKKVL",
+            "WFKKlLKKALRLWKKVL",
+            "WFKKLLKKALrlWKkVL"
+        ]
+    },
+    "ACPIFTKIQGTYRGRAKCR": {
+        "1": [
+            "aCPIFTKIQGTYRGRAKCR",
+            "AcPIFTKIQGTYRGRAKCR",
+            "ACpIFTKIQGTYRGRAKCR",
+            "ACPIfTKIQGTYRGRAKCR",
+            "ACPIFtKIQGTYRGRAKCR"
+        ],
+        "0": [
+            "ACPiFTKiQGTYrGrAKCR",
+            "ACPiFTKiQGTYrGrAKCr",
+            "aCPiFTKiQGTYrGrAKCR",
+            "AcPiFTKiQGTYrGrAKCR",
+            "ACPifTKiQGTYrGrAKCR",
+            "ACPiFTKiQGTYrGrAkCR"
+        ]
+    },
+    "ILLKKLLKKI": {
+        "1": [
+            "illkkllkki",
+            "Illkkllkki",
+            "iLlkkllkki",
+            "ilLkkllkki",
+            "illKkllkki",
+            "illkKllkki"
+        ],
+        "0": [
+            "iLLKKLLKKI",
+            "IlLKKLLKKI",
+            "ILlKKLLKKI",
+            "ILLkKLLKKI",
+            "ILLKkLLKKI"
+        ]
+    },
+    "GRFKRFRKKFKKLFKKLS": {
+        "1": [
+            "grfkrfrkkfkklfkkls",
+            "Grfkrfrkkfkklfkkls",
+            "gRfkrfrkkfkklfkkls",
+            "grfkrfrkkfkklfkklS",
+            "grfkrFrkkfKklfkkls"
+        ],
+        "0": [
+            "gRFKRFRKKFKKLFKKLS",
+            "GRFKRFRKKFKKLFKKLs",
+            "gRFKRFRKKFKKLFKKLs",
+            "grfKRFRKKFKKLFKKLS",
+            "grfkRFRKKFKKLFKKLS"
+        ]
+    },
+    "RAGLQFPVGRVHRLLRK": {
+        "1": [
+            "raglqfpvgrvhrllrk",
+            "Raglqfpvgrvhrllrk",
+            "rAglqfpvgrvhrllrk",
+            "rAgLqfpvgrvhrllrk",
+            "RaglqfpvgrVhrllrk",
+            "raglqfpvgrVhrllrk"
+        ],
+        "0": [
+            "rAGLQFPVGRVHRLLRK",
+            "RaglQFPVGRVHRLLRK",
+            "RAGLQfpVGRVHRLLRK",
+            "RAGLQFPvgRVHRLLRK",
+            "RAGLQFPVGRvhRLLRK"
+        ]
+    },
+    "KLKLLLLLKLK": {
+        "1": [
+            "klklllllklk",
+            "KLklllllklk",
+            "klKLllllklk",
+            "KLKlLllLklk",
+            "klkllLlklKk",
+            "klKlLlLLKlk"
+        ],
+        "0": [
+            "kLklLLlllkK",
+            "KLkLlllLkLk",
+            "KlklllllKLK",
+            "kLKLLlLKkLk",
+            "KLKLllLLkKk"
+        ]
+    },
+    "KLKLLLKLK": {
+        "1": [
+            "klklllklk",
+            "kLKLLLKLK",
+            "KLKllLKLK",
+            "kLkLlLkLk",
+            "kLKlllkLK"
+        ],
+        "0": [
+            "KLkLLLKLK",
+            "KlKlllklK",
+            "kLKLllKLk",
+            "KLKLLkLKk"
+        ]
+    },
+    "FIKRIARLLRKIF": {
+        "1": [
+            "fIKRIARLLRKIF",
+            "FIKRIArLLRKIF",
+            "FIKrIARLLRKIF",
+            "fIKRIARLLRKIf",
+            "FIkRIARLLrKIF"
+        ],
+        "0": [
+            "fikriarllrkif",
+            "Fikriarllrkif",
+            "fikriarllrkiF",
+            "fikriArllrkif",
+            "fIkrIarllrkif",
+            "fiKriarLlrkif"
+        ]
+    },
+    "INLKAIAALAKKLL": {
+        "1": [
+            "iNLKAIAALAKKLL",
+            "InLKAIAALAKKLL",
+            "INlKAIAALAKKLL",
+            "INLkAIAALAKKLL",
+            "INLKaIAALAKKLL"
+        ],
+        "0": [
+            "inlkaiaalakkll",
+            "Inlkaiaalakkll",
+            "iNlkaiaalakkll",
+            "inLkaiaalakkll",
+            "inlKaiaalakkll",
+            "inlkAiaalaakll"
+        ]
+    },
+    "FLPLIGRVLSGIL": {
+        "1": [
+            "fLPLIGRVLSGIL",
+            "FLPlIGRVLSGIL",
+            "FLPliGRVLSGIL",
+            "FLpLIGRVLSGIL",
+            "FlPLIGRvLSGIL"
+        ],
+        "0": [
+            "flpligrvlsgil",
+            "FLPLiGRVLSGIL",
+            "FLPLIGrVLSGIL",
+            "FLPLigrvLSGIL",
+            "FLPLIGRVLsGIL",
+            "flPLIGRvlsGIL"
+        ]
+    },
+    "KLLKKAGKLLKKAGKLLKKAG": {
+        "1": [
+            "KlLkKaGkLlKkAGkLlKkAG",
+            "kLlKkAGKlLkKaGkLlKkAG",
+            "KLLkkaGKLLkkaGKLLkkaG",
+            "KlkKKAGKlkKKAGKlkKKAG",
+            "KkLKKAGKkLKKAGKkLKKAG"
+        ],
+        "0": [
+            "KlLkKaGkLlKkAGKlLkKaG",
+            "KlLkKaGKLLKKAGkLlKkAG",
+            "KkLKKAGKlLKKAGkLlKkAG",
+            "kLlKkAGKlLKKAGKlLkKaG",
+            "KlLkKaGkLLKKAGkLlKkAG",
+            "KlLKKAGkLlKkAGKlLkKaG"
+        ]
+    },
+    "LLAKKKGLLAKKKGLLAKKKG": {
+        "1": [
+            "LlAkKkGlLaKkKgLlAkKkG",
+            "LlAkKkGlLaKkKgLlAkKKg",
+            "LlAkKkGlLaKkKgLlAKkkG",
+            "LlAkKkGlLaKkKgllAkKkG",
+            "lLAkKkGlLaKkKgLlAkKkG",
+            "LlAkKkGllaKkKgLlAkKkG"
+        ],
+        "0": [
+            "llakkkgllaKKKGLLAKKKG",
+            "LLAKKKGLLAKkkgllakkkg",
+            "LlAKkkgLLaKkKgLlAkKkg",
+            "llAkKKglLAkKKglLAkKkG"
+        ]
+    },
+    "RPFTRAQWFAIQHISPRTIAMRAINNYRWR": {
+        "1": [
+            "rPFTRAQWFAIQHISPRTIAMRAINNYRWR",
+            "RpFTRAQWFAIQHISPRTIAMRAINNYRWR",
+            "RPFTRaQWFAIQHISPRTIAMRAINNYRWR",
+            "RPFTRAQWFAiQHISPRTIAMRAINNYRWR",
+            "RPftRAQWFAIQHISPRTIAMRAINNYRWR"
+        ],
+        "0": [
+            "rpftraqwfaiqhisprtiamrainnyrwr",
+            "rpftraqwfaIQHISPRTIAMRAINNYRWR",
+            "RPFTRaqwfaiqhisprtiamrainNYRWR",
+            "rpftraqwfaIQHISPRTIAmrainnyrwr",
+            "rPfrAqWfAiQhIsPrTiAmRainNynRwR",
+            "RpFtRaQwFaIqHispRtIaMRAINNYRWR"
+        ]
+    },
+    "RLWLAIWRR": {
+        "1": [
+            "rlwlaiwrr",
+            "rLwlaiwrr",
+            "rlwLaiwrr",
+            "rlwlAiwrr",
+            "rlwlaIwrr",
+            "rlwlaiwRr"
+        ],
+        "0": [
+            "rLWLAIWRR",
+            "RlWLAIWRR",
+            "RLwLAIWRR",
+            "RLWlAIWRR",
+            "RLWLaIWRR"
+        ]
+    },
+    "KLWLAIWKK": {
+        "1": [
+            "klwlaiwkk",
+            "klwlaIWKK",
+            "KLWLAiWKK",
+            "KlWLAIwKK",
+            "klWLAIWKK",
+            "KLwlaIWKK"
+        ],
+        "0": [
+            "KLWLAIwKK",
+            "KlwlAIWKK",
+            "KLWLaiWKK",
+            "kLWlAIWKK",
+            "kLWLalWKK"
+        ]
+    },
+    "LKWLKKL": {
+        "1": [
+            "lkwlkkl",
+            "LKWlKKL",
+            "lKwLKKl",
+            "LkWLKKl",
+            "LKWlkkl",
+            "lkwLKkl"
+        ],
+        "0": [
+            "LkWLkkL",
+            "lKwlKKl",
+            "LKWLKKl",
+            "lkwlKKL",
+            "lKWLKkL",
+            "LkWLKkl",
+            "lKwLkKL"
+        ]
+    },
+    "LRWLRRL": {
+        "1": [
+            "lrwlrrl",
+            "lRwlrrl",
+            "lrwlRrl",
+            "lrwlrRl",
+            "lRwlRrl",
+            "lRwlrRl"
+        ],
+        "0": [
+            "LrWLrrL",
+            "lRwlRRl",
+            "Lrwlrrl",
+            "lrWlrrl",
+            "lrwLrrl",
+            "lrwlrrL",
+            "LrwLrrl"
+        ]
+    },
+    "FLKLLKKLLFLKLLKKLL": {
+        "1": [
+            "fLKLLKKLLfLKLLKKLL",
+            "fLKLlKKLLfLKLLKKLL",
+            "FLkLLKKLLflKLLKKLL",
+            "flKLlKKLLfLKLLKKLL",
+            "fLKLLkkLLfLKLLKKLL",
+            "fLKLLKKLLfLkLLKKLL"
+        ],
+        "0": [
+            "FLklLKKLLFLKLLKKLL",
+            "FLKLLKKllFLKLLKKLL",
+            "FlkLLKKLLFLKLLKKLL",
+            "FLKllKKLLfLKLLKKLL",
+            "FLKLLKKLLFLKLLkKLL"
+        ]
+    },
+    "VDKPPYLPRPRPPRRIYNR": {
+        "1": [
+            "VDKPPYLPRPRPPRriynr",
+            "VDKPPYLPRPRpprriynr",
+            "VDKPPYLPRPrpprriynr",
+            "VDKPPYLPRPRpPRRIYNR",
+            "VDKPPYLPRPrPPRRIYNR",
+            "VDKPPyLPRPRPPRRIYNR",
+            "VDKPPYLPRPRPPrriynr",
+            "VDKPPYLPRpRPPrriynr",
+            "VDKPPYLPRPRPprriynr",
+            "VDKPPyLPRPRPPrRIYNR",
+            "VDKPPYLPrpRPPRRIYNR"
+        ],
+        "0": [
+            "VDKPPYLPRpRPPRRIYNR",
+            "VDKPPYLPrPRPPRRIYNR",
+            "VDKPPYLpRPRPPRRIYNR",
+            "VDKPPYlPRPRPPRRIYNR",
+            "VDKPpYLPRPRPPRRIYNR",
+            "VDKppYLPRPRPPRRIYNR",
+            "VDKpPYLPRPRPPRRIYNR",
+            "vdkppylprprpprriynr",
+            "vDKPPYLPRPRPPRRIYNR",
+            "VDKpPYLPRPRPPRRIYNr",
+            "VDKPpyLPRPRPPRRIYNR",
+            "VDKPPYLpRPRPpRRIYNR",
+            "VDKPPYLPRpRPPRRIYnR"
+        ]
+    },
+    "VRLIVAVRIWRR": {
+        "1": [
+            "VRLIVAVRIWRR",
+            "vRLIVAVRIWRR",
+            "VRlIVAVRIWRR",
+            "VRLIvAVRIWRR",
+            "VRLIVAvRIWRR"
+        ],
+        "0": [
+            "vrlivavriwrr",
+            "Vrlivavriwrr",
+            "vRlivavriwrr",
+            "vrLivavriwrr",
+            "vrlIvavriwrr",
+            "vrliVavriwrr"
+        ]
+    },
+    "VRLRWWRRRWRR": {
+        "1": [
+            "vRLRWWRRRWRR",
+            "VRlRWWRRRWRR",
+            "VRLRwWRRRWRR",
+            "VRLRWwRRRWRR",
+            "vRlRwwRRRWRR"
+        ],
+        "0": [
+            "vrlrwwrrrwrr",
+            "vrlrwwrrrwrR",
+            "vrlrwwrrrwRr",
+            "Vrlrwwrrrwrr",
+            "VRlrwwrrrwrr",
+            "VrLrWWrrrWrr"
+        ]
+    },
+    "RRW": {
+        "1": [],
+        "0": [
+            "rRW",
+            "RrW",
+            "RRw",
+            "rrW",
+            "Rrw",
+            "rRw",
+            "rrw"
+        ]
+    },
+    "FLGTVLKVAAKVLPAALCQIFKKC": {
+        "1": [
+            "FlGTVlKVAAKVlPAAlCQIFKKC",
+            "FlGTVlKVAAKVlPAALCQIFKKC",
+            "FlGTVlKVAAKVLPAAlCQIFKKC",
+            "FlGTVLKVAAKVlPAAlCQIFKKC",
+            "FLGTVlKVAAKVlPAAlCQIFKKC",
+            "FlGTVlKVAAKVLPAALCQIFKKC"
+        ],
+        "0": [
+            "FLGTVLkVAAkVLPAALCQIFkkC",
+            "FLGTVLkVAAkVLPAALCQIFKkC",
+            "FLGTVLkVAAKVLPAALCQIFkkC",
+            "FLGTVLKVAAkVLPAALCQIFkkC",
+            "FLGTVLkVAAkVLPAALCQIFKKC"
+        ]
+    },
+    "FLGTVLKVLAKVLPAALCQIFKKC": {
+        "1": [
+            "FlGTVlKVlAKVlPAAlCQIFKKC",
+            "FLGTVlKVlAKVlPAAlCQIFKKC",
+            "FlGTVLKVlAKVlPAAlCQIFKKC",
+            "FlGTVlKVLAKVlPAAlCQIFKKC",
+            "FlGTVlKVlAKVLPAAlCQIFKKC",
+            "FlGTVlKVlAKVlPAALCQIFKKC"
+        ],
+        "0": [
+            "fLGTVLKVLAKVLPAALCQIFKKC",
+            "FLgTVLKVLAKVLPAALCQIFKKC",
+            "FLGtVLKVLAKVLPAALCQIFKKC",
+            "FLGTvLKVLAKVLPAALCQIFKKC",
+            "FLGTVLkVLAKVLPAALCQIFKKC"
+        ]
+    },
+    "FLGTVLRVAARVLPAALCQIFRRC": {
+        "1": [
+            "FLGtvLRVAARVLPAALCQIFRRC",
+            "FLGTVLRvaarVLPAALCQIFRRC",
+            "FLGTVLRVAARvlpAALCQIFRRC",
+            "fLGTVLRVAARVLPAALcqIFRRC",
+            "FLGTVLrvAARVLPAALCQiFRRC"
+        ],
+        "0": [
+            "FLGTVLrVAArVLPAALCQIFrrC",
+            "FLGTVlrVAARVLPAALCQIFRRC",
+            "FLGTVLrVaaRVLPAALCQIFRRC",
+            "FLGTVLrVAARVLPAALCQIFrRC",
+            "FLGTVLRVAaRVLPAALCQIFrrC",
+            "FLGTVLRVAARVLPAALCqifrRC"
+        ]
+    },
+    "RWKIFKKIEKMGRNIRDGIVKAGPAIQVLGSAKAI": {
+        "1": [
+            "rWKIFKKIEKMGRNIRDGIVKAGPAIQVLGSAKAI",
+            "RWKIFKKIEKMGRNIRDGIVKAGPAIQVLGSAKAi",
+            "RWKIFKKIEKmGRNIRDGIVKAGPAIQVLGSAKAI",
+            "rWKIFKKIEKMGRNIRDGIVKAGPAIQVLGSAKAi"
+        ],
+        "0": [
+            "rwkifkkiekmgrnirdgivkagpaiqvlgsakai",
+            "Rwkifkkiekmgrnirdgivkagpaiqvlgsakai",
+            "rwkifkkiekmgrnirdgivkagpaiqvlgsakaI",
+            "rwkifkkiekMgrnirdgivKagpaiqvlgsakai",
+            "RWKIFKKIEKmgrnirdgivkagpaiqvlgsakai",
+            "RwKiFkKiEkMgRnIrDgIvKaGpAiQvLgSaKaI"
+        ]
+    },
+    "GPLGVRGKRLWDIVRRWVGWL": {
+        "1": [
+            "GPlGvRGKRLWDIVRRWVGWL",
+            "GPlGvRGKRLWDIvRRWVGWL",
+            "GPLGvRGKRlWDIVRRWVGWL",
+            "GPLGVRGKRlWDIVRRWVGWl",
+            "GPLGVRGKRLWDIvRRWvGWL",
+            "GPlGVRGKRlWDIvRRWVgWL"
+        ],
+        "0": [
+            "gPLGVRGKRLWDIVRRWVGWL",
+            "GPLGVRgKRLWDIVRRWVGWL",
+            "GPLGVRGKRLWDIvRrWVGWL",
+            "GPLGVRGKRLWDIVrRwVGWL",
+            "GPLGVRGKRLWDIVRRWVgWl"
+        ]
+    },
+    "RIVQRIKKWLR": {
+        "1": [
+            "rivqrikkwlr",
+            "rIVQRIKKwlr",
+            "riVqRIKKWLR",
+            "RivqRIKKwlr",
+            "rivQRiKKWLR",
+            "RIvQrIKKWLr"
+        ],
+        "0": [
+            "RIVqRIKKWLr",
+            "riVQRiKKwlR",
+            "RiVQRIkKwLr",
+            "rIVQrIKKwLR",
+            "rivQrIKKwLR"
+        ]
+    },
+    "KRIWQRIK": {
+        "1": [
+            "kriwqrik",
+            "KrIWQRIK",
+            "KRIwQRIK",
+            "kRIWQRiK",
+            "kriwqRIK",
+            "KriwqRIk"
+        ],
+        "0": [
+            "KRIWqRIK",
+            "kRIwQRIk",
+            "KRIWqriK",
+            "kRIWqRIK",
+            "KRIWQrIk"
+        ]
+    },
+    "KRIWQRIKDF": {
+        "1": [
+            "kriwqrikdf",
+            "Kriwqrikdf",
+            "krIwqrikdf",
+            "kriwQrikdf",
+            "kriwqrIkdf",
+            "kriwqrikDf"
+        ],
+        "0": [
+            "kRIWQRIKDF",
+            "KRiWQRIKDF",
+            "KRIWQRIKDf",
+            "KrIWqRIKDF",
+            "KRIwQrIKDF"
+        ]
+    },
+    "KYKKALKKLAKLL": {
+        "1": [
+            "kykkalkklakll",
+            "Kykkalkklakll",
+            "kYkkalkklakll",
+            "kyKkalkklakll",
+            "kykKalkklakll",
+            "kykkAlkklakll"
+        ],
+        "0": [
+            "kYKKALKKLAKLL",
+            "KyKKALKKLAKLL",
+            "KYkKALKKLAKLL",
+            "KYKkALKKLAKLL",
+            "KYKKaLKKLAKLL"
+        ]
+    },
+    "VQWRAIRVRVIR": {
+        "1": [
+            "vqwrairvrvir",
+            "vQWRAIRVRVIR",
+            "vqWRAIRVRVIR",
+            "vqwRAIRVRVIR",
+            "vqwrAIRVRVIR",
+            "vqwraIRVRVIR"
+        ],
+        "0": [
+            "vqwraiRVRVIR",
+            "vqwrairVRVIR",
+            "vqwrairvRVIR",
+            "vqwrairvrVIR",
+            "vqwrairvrvIR"
+        ]
+    },
+    "GFAWNVCVYRNGVRVCHRRAN": {
+        "1": [
+            "gFAWNVCVYRNGVRVCHRRAN",
+            "GfAWNVCVYRNGVRVCHRRAN",
+            "GFawNVCVYRNGVRVCHRRAN",
+            "GFAWNVCVYRNGVRVCHRRAn",
+            "GFAWNVCVyRNGVRVCHRRAN"
+        ],
+        "0": [
+            "GfawnvcvyrnGvrvchrran",
+            "gfawnvcvyrngvrvchrran",
+            "Gfawnvcvyrngvrvchrran",
+            "GfawnvcvyrngvrvchrraN",
+            "gfawnvcvyrNgvrvChrran",
+            "gfawnvcvyRNgvrvchrran"
+        ]
+    },
+    "LLGDFFRKSKEKIGKEFKRIVQRIKDFLRNLVPRTES": {
+        "1": [
+            "llgdffrkskekigkefkrivqrikdflrnlvprtes",
+            "LLGDFFRkskeKIGKEFKRIVQRIKDFLRNLVPRTES",
+            "LLGDffrkskeKIGKEFKRIVQRIKDFLRNLVPRTES",
+            "LLGDFFRKSKEKIGKefkrIVQRIKDFLRNLVPRTES",
+            "LLGDFFRKSKEKigkeFKRIvqrikdflrnLVPRTES",
+            "llgdFFRKSKEKIGKEFKRIVQrikdflrnlvprtes"
+        ],
+        "0": [
+            "LLgDfFRKsKEkIgKeFkRiVqRIKdFlRnLvPRtEs",
+            "lLGDFfRkSKeKIGKeFkRIvQRIkDfLrnlVPrTeS",
+            "LlGdFfRkSKEkIGKeFkRIVQRIKdflRNLvPRTeS",
+            "LLgDFFRKSkEkIgKeFKRivQRIkdfLrnlVPrTeS",
+            "LLgDFfRksKekIGkEfKrivQrIKdflRNlVpRtEs"
+        ]
+    },
+    "LLGDFFRKSKEKIGKEFKRIVQRIKDFLRNL": {
+        "1": [
+            "llgdffrkskekigkefkrivqrikdflrnl",
+            "Llgdffrkskekigkefkrivqrikdflrnl",
+            "llgdfFrkskekigkefkrivqrikdflrnl",
+            "llgdffrkskEkigkefkrivqrikdflrnl",
+            "llgdffrkskekigkEfkrivqrikdflrnl",
+            "llgdffrkskekigkefkriVqrikdflrnl"
+        ],
+        "0": [
+            "lLGDFFRKSKEKIGKEFKRIVQRIKDFLRNL",
+            "LLGDFFRKSKeKIGKEFKRIVQRIKDFLRNL",
+            "llgDFFRKSKEKIGKEFKRIVQRIKDFLRNL",
+            "LLGDFFRKSKeKIGKEFKRIvQRIKDFLRNl",
+            "LLGDFFRKSKEKIGKEFKRIVQRIKDFLRNl"
+        ]
+    },
+    "RKRWWRWWKWWKR": {
+        "1": [],
+        "0": [
+            "RKrWWrWwkWWkR"
+        ]
+    },
+    "WRWWKWW": {
+        "1": [
+            "wRWWKWW",
+            "WRwWKWW",
+            "wRWwKWW",
+            "WRWWKwW",
+            "wrWwKWW"
+        ],
+        "0": [
+            "WrWwkWW",
+            "WrWWKWW",
+            "WRWwKWW",
+            "WRWWkWW",
+            "WrWWkWW",
+            "WRWwkWW"
+        ]
+    },
+    "WWRWWKWW": {
+        "1": [
+            "wWRWWKWW",
+            "WwRWWKWW",
+            "WWRwWKWW",
+            "WWRWWKwW",
+            "wWRWWKWw"
+        ],
+        "0": [
+            "WWrWwkWW",
+            "WWrWWKWW",
+            "WWRWwKWW",
+            "WWRWWkWW",
+            "WWrWwKWW",
+            "WWRWwkWW"
+        ]
+    },
+    "RRGKKLLLLLKKKG": {
+        "1": [
+            "rrgkklllllkkkg",
+            "RRGKKlllllKKKG",
+            "rrGKKlllllKKKG",
+            "RRgKKlllllKKKG",
+            "RRGKKlllllkkkG",
+            "RRGkklllllKKKG"
+        ],
+        "0": [
+            "rrgkkLLLLLKKKG",
+            "RRGKKLLLLLkkkg",
+            "rRGKKllllLKKKG",
+            "RrGKKLllllKKKG",
+            "RRGkkLLLllKKKG"
+        ]
+    },
+    "LLWIALRKK": {
+        "1": [
+            "llwialrkk",
+            "llwIaLRKK",
+            "LLwiaLRKK",
+            "LLWIALrKK",
+            "llWialRKK",
+            "LLwiAlrKK"
+        ],
+        "0": [
+            "lLwIALRKK",
+            "LLWiaLrKK",
+            "LLwiALRKK",
+            "llwIaLrkK",
+            "LLWIAlrKk"
+        ]
+    },
+    "PRPRPRP": {
+        "1": [
+            "PrPrPrP",
+            "pRpRpRp",
+            "PRpRPRP",
+            "PrPRPRP",
+            "pRPRPRP"
+        ],
+        "0": [
+            "prprprp",
+            "prPRPRP",
+            "PRprPRP",
+            "PRPRprP",
+            "prprprP",
+            "pRPRPRp"
+        ]
+    },
+    "KWLKKWLKWLKK": {
+        "1": [
+            "kwlKkWLKWLKK",
+            "KWlKkKWLKWLK",
+            "KWLKkwLKWLKk",
+            "KWlKKwLkwLKK",
+            "kWLKKWLKwLkK"
+        ],
+        "0": [
+            "kwLkkwLkwLkk",
+            "kwlkkwLkwLkk",
+            "kwLKKwlkwlkk",
+            "kWLkkwLkWLkk",
+            "kwLKKwLkWLkk",
+            "KwLKKwLkwlkk"
+        ]
+    },
+    "ILRWPWWPWRRK": {
+        "1": [
+            "iLRWPWWPWRRK",
+            "ILrWPWWPWRRK",
+            "ILRwPWWPWRRK",
+            "ILRWPwWPWRRK",
+            "ILRWPWWpWRRK"
+        ],
+        "0": [
+            "ilrwpwwpwrrk",
+            "Ilrwpwwpwrrk",
+            "ilRwpwwpwrrk",
+            "ilrWpwwpwrrk",
+            "ilrwPwwpwrrk",
+            "ilrwpwWpwrrk"
+        ]
+    },
+    "KRKIFLRTKILV": {
+        "1": [
+            "KrKiFlRtKiLv",
+            "KrKiFLRTKILV",
+            "KrKIFlRTKILV",
+            "KRKiFlRTKILV",
+            "KrKIFLRTKILv",
+            "KRKiFLRTKILv"
+        ],
+        "0": [
+            "kRkIfLrTkIlV",
+            "kRKIFLRTKILV",
+            "KRkIFLRTKILV",
+            "kRkIFLRTKILV",
+            "kRKIfLRTKILV",
+            "KRKIFLrTKILV"
+        ]
+    },
+    "VLIKTRLFIKRK": {
+        "1": [
+            "vLiKtRlFiKrK",
+            "vLiKtRLFIKrK",
+            "VLIKtRlFiKrK",
+            "vLIKtRlFiKrK",
+            "VLiKtRLfiKRk",
+            "VLIKTlFirKrk"
+        ],
+        "0": [
+            "VliKTrlfiKRK",
+            "vLIkTrLfIkRK",
+            "VLkTrLFiKrkK",
+            "vlIKtrLFikRk",
+            "VLiKTlFiKrkk"
+        ]
+    },
+    "KWKLFKKIEKVGQNIRDGIIKAGPAVAVVGQATQIAK": {
+        "1": [
+            "kWKLFKKIEKVGQNIRDGIIKAGPAVAVVGQATQIAK",
+            "KwKLFKKIEKVGQNIRDGIIKAGPAVAVVGQATQIAK",
+            "KWkLFKKIEKVGQNIRDGIIKAGPAVAVVGQATQIAK",
+            "kWkLFKKIEKVGQNIRDGIIKAGPAVAVVGQATQIAK",
+            "KWKLFKKIEKVGQNIRDGIIKAGPAVAVVGQATQIAk"
+        ],
+        "0": [
+            "kwklfkkiekvgqnirdgiikagpavavvgqatqiak",
+            "Kwklfkkiekvgqnirdgiikagpavavvgqatqiak",
+            "kwklfkkiekvgqnirdgiikagpavavvgqatqiAk",
+            "kwklfkkiekvgqnirdgiiKAGPAVAVVGQATQIAK",
+            "KwKlFkKiEkVgQnIrDgIiKaGpAvAvVgQaTqIaK"
+        ]
+    },
+    "GIGKFLHSAKKFGKAFVGEIMNS": {
+        "1": [
+            "gigkflhsakkfgkafvgeimns",
+            "gIgKFLHSAKKFGKAFVGEIMNS",
+            "GIgKFLHSAKKFGKAFVGEIMNS",
+            "GIGkfLHSAKKFGKAFVGEIMNS",
+            "GIGKFlHSAKKFGKaFVGEIMNS",
+            "GIGKFLHSaKKFGKAFVGEiMNS"
+        ],
+        "0": [
+            "GIGkfLHSaKKFGKAFVGEIMNS",
+            "GIGKFLHsakkfgKAFVGEIMNS",
+            "GIGKFLHSAKKfGKafvgeimns",
+            "GigkflhsakKfGKAFVGEIMNS",
+            "GIGKFLHSaKKFGKAFVGEImnS"
+        ]
+    },
+    "KWKLFKKIEKVGQGIGAVLKVLTTGL": {
+        "1": [
+            "KWKLfKKIEKVGQGIGAVLKVLTTGL",
+            "kKWLFKKIEKVGQGIGAVLKVLTTGL",
+            "KWKkFKKIEKVGQGIGAVLKVLTTGL",
+            "KWKlFKKiEKVGQGIGAVLKVLTTGL",
+            "KwKLFKKIEkVGQGIGAVLKVLTTGL"
+        ],
+        "0": [
+            "kwklfkkiekvgqgigavlkvlttgl",
+            "kwkLfkkiekvgqgigavlkvlttgl",
+            "kwklfkkiekvgqgigavlkVLttgl",
+            "KwklfkkiekvgqgigavLKVlttgl",
+            "kwklfKkiekvgqgigavlkvlttgl",
+            "kWkLfkkiekvgqgigavlkvlttgl"
+        ]
+    },
+    "KWKLFKKIGIGAVLKVLTTGLPALIS": {
+        "1": [
+            "kwklfkkigigavlkvlttglpalis",
+            "kwklfkkigigavlkvlttgLPALIS",
+            "KWKLFKkigigavlkvlttglpalis",
+            "KwklfkkigIgavlkvlttGlpalis",
+            "kwklFkkigigavlKvlttglpalIs"
+        ],
+        "0": [
+            "kWKLFKKIGIGAVLKVLTTGLPALIS",
+            "kwKLFKKIGIGAVLKVLTTGLPALIS",
+            "KWKLFKKIGiGAVLKVLTTGLPALIS",
+            "KWKLFKKIGIGAVLKVLTTgLPALIS",
+            "KWKLfKKIGIGAVLKVLTTGLPALIS"
+        ]
+    },
+    "KWKLFKKGIGAVLKV": {
+        "1": [
+            "kwklfkkgigavlkv",
+            "kWKLFKKGIGAVLKv",
+            "kwKLFKKGIGAVLKV",
+            "kwkLFKKGIGAVLKV",
+            "KWKLFKKGIGAVLkv",
+            "KWKLFKKgIGAVlkV"
+        ],
+        "0": [
+            "KWKlfKKGIGAVLKV",
+            "KWKLFKKGiGAVLKV",
+            "KWKlFKKGiGAVLKV",
+            "KWKLfKKGiGAVLKV"
+        ]
+    },
+    "KWKLFKKIGAVLKVL": {
+        "1": [
+            "kwklfkkigavlkvl",
+            "kWKLFKKIGAVLKVL",
+            "KwKLFKKIGAVLKVL",
+            "kwKLFKKIGAVLKVL",
+            "KWKLFKKIGAVLKVl",
+            "kWkLfKkIgAvLkVl"
+        ],
+        "0": [
+            "KWklFKKIGAVLKVL",
+            "kwkLFKKIGAVLKVL",
+            "KWKLFKKIGAVLkvl",
+            "KwKlFkKiGaVlKvL",
+            "kwKlFKkIGAvLKVL"
+        ]
+    },
+    "KWKLFKKGAVLKVLT": {
+        "1": [
+            "kwklfkkgavlkvlt",
+            "KWKlfkkgavlkvlt",
+            "kwklFKKgavlkvlt",
+            "Kwklfkkgavlkvlt",
+            "kwklfkkkgVLkvlt"
+        ],
+        "0": [
+            "kWKLFKKGAVLKVLT",
+            "kWKLFKKGAVLKVLt",
+            "kwkLFKKGAVLKVLT",
+            "KWKlfkkGAVLKVLT",
+            "kWkLfKkGaVLKVLT"
+        ]
+    },
+    "KWKLFKKAVLKVLTT": {
+        "1": [
+            "kwklfkkavlkvltt",
+            "Kwklfkkavlkvltt",
+            "kWklfkkavlkvltt",
+            "kwKlfkkavlkvltt",
+            "kwkLfkkavlkvltt",
+            "kwklFkkavlkvltt"
+        ],
+        "0": [
+            "KWKLFkKAVLKVLTT",
+            "KWKLFKkAVLKVLTT",
+            "KWKLFKKaVLKVLTT",
+            "KWKLFKKAvLKVLTT",
+            "KWKLFKKAVlKVLTT"
+        ]
+    },
+    "KWKLFKKVLKVLTTG": {
+        "1": [
+            "kwklfkkvlkvlttg",
+            "Kwklfkkvlkvlttg",
+            "kWklfkkvlkvlttg",
+            "kwKlfkkvlkvlttg",
+            "kwkLfkkvlkvlttg",
+            "kwklFkkvlkvlttg"
+        ],
+        "0": [
+            "kWKLFKKVLKVLTTG",
+            "kwKLFKKVLKVLTTG",
+            "kwkLFKKVLKVLTTG",
+            "kwklFKKVLKVLTTG",
+            "kwklfKKVLKVLTTG"
+        ]
+    },
+    "GSKKPVPIIYCNRRTGKCQRM": {
+        "1": [
+            "GsKKPVPIIYCNRRTGKCQRM",
+            "GSKkpvpiiyCNRRTGKCQRM",
+            "GSKKPVPIIYCNrRTgKCQRM",
+            "gSKKPVPIIYCNRRTGkCQRM",
+            "GSKKPVPIIycnrrTGKCQRM"
+        ],
+        "0": [
+            "gskkpvpiiycnrrtgkcqrm",
+            "gskkpvpiiycnrrtgkCQRM",
+            "gskkpvpiIYCNRRTGKcqrM",
+            "GSKKPVPiiycnrrtgkcqrm",
+            "gskkpVPIIYCNRRTgkcqrm",
+            "gskkPVPIIYcnrrtgkcqrm"
+        ]
+    },
+    "RRWQWRMKK": {
+        "1": [
+            "rrwqwrmkk",
+            "Rrwqwrmkk",
+            "rRwqwrmkk",
+            "rrwQwrmkk",
+            "rrwqWrmkk",
+            "rrwqwRmkk"
+        ],
+        "0": [
+            "rRWQWRMKK",
+            "RrWQWRMKK",
+            "RRwQWRMKK",
+            "RRWqWRMKK",
+            "RRWQwRMKK"
+        ]
+    },
+    "FKCRRWQWRMKKLGA": {
+        "1": [
+            "fkcrrwqwrmkklga",
+            "fkcrrwqwRMKKLGA",
+            "fKcRrWqWrMkKlGa",
+            "FKCRRwQwRMKKLGA",
+            "fkCrrwqwrmkklga"
+        ],
+        "0": [
+            "fKCRRWQWRMKKLGA",
+            "FkCRRWQWRMKKLGA",
+            "FKCrRWQWRMKKLGA",
+            "FKCRRWqWRMKKLGA",
+            "FKCRRWQWRMKKLGa"
+        ]
+    },
+    "PKLLKTFLSKWIG": {
+        "1": [
+            "pKLLKTFLSKWIG",
+            "PKlLKTFLSKWIG",
+            "PKLLkTFLSKWIG",
+            "PKLLKTfLSKWIG",
+            "PKLLKTFLsKWIG"
+        ],
+        "0": [
+            "pkllktflskwig",
+            "pkllktflskwiG",
+            "pkllktflskwIg",
+            "pkllktflsKwIg",
+            "pkllktfLSkwiG",
+            "pkllkTflskwiG"
+        ]
+    },
+    "KLPLIGRVLSGIL": {
+        "1": [
+            "klpligrvlsgil",
+            "KLPLigrvlsgil",
+            "kLPLigrvlsgil",
+            "klPLigrvlsgil",
+            "klpLIGRvlsgil",
+            "klpliGrVLSGIL"
+        ],
+        "0": [
+            "KlPLigrvlsgil",
+            "klpLIGRVlSGIL",
+            "KLPLigRvLSGIl",
+            "klPLIGRvlsgil",
+            "KlpligRVLSGiL"
+        ]
+    },
+    "KKHRKHRKHRKHGGSGGSKNLRRIIRKGIHIIKKYG": {
+        "1": [
+            "KKHRKHRKHRKHGGSGGSKNLRRIIRKGIHIIKKYG",
+            "kKHRKHRKHRKHGGSGGSKNLRRIIRKGIHIIKKYG",
+            "KKHRKHRKHRKHGGSGGSKNLRRIIRKGIHIIKKYg",
+            "KKHRKHRKHRKHGGsGGSKNLRRIIRKGIHIIKKYG"
+        ],
+        "0": [
+            "kkhrkhrkhrkhggsggsknlrriirkgihiikkyg",
+            "kkhrkhrkhrkhggsggsKnlrriirkgihiikkyg",
+            "KKHRKhrkhrkhggsggsknlrriirkgihiikkyg",
+            "KkHrKhRkHrKhGgSgGsKnLrRiIrKgIhIiKkYg",
+            "KKHRKHRKHRKHGGSGGSknlrriirkgihiikkyg"
+        ]
+    },
+    "FKRIVQRIKDFLRNLV": {
+        "1": [
+            "fKRIVQRIKDFLRNLV",
+            "FkRIVQRIKDFLRNLV",
+            "FKrIVQRIKDFLRNLV",
+            "FKRIVQRIKDFLRrLV",
+            "FKRIVQRIKdFLRNLV"
+        ],
+        "0": [
+            "FKRiVQRiKDFlRNLV",
+            "FKRIvQRiKDFlRNLV",
+            "FKRIVQrIKDFLRNlV",
+            "fKRiVQRIKDFLRNLV",
+            "FKRIVQRikDFLRnLV",
+            "FKRIVQRIKDfLRNLV"
+        ]
+    },
+    "GWGSFFKKAAHVGKHVGKAALTHYL": {
+        "1": [
+            "gwgsffkKAAHVGKHVGKAALTHYL",
+            "GWGSFFKKAAhvgkhvgkaalTHYL",
+            "gWgSfFkKaAhVgKhVgKaAlThYl",
+            "GwGSffKKaaHvGKHvGKaalTHyl",
+            "GWGSFFkkAAHVGKHVGKAALTHYL"
+        ],
+        "0": [
+            "gwgsffkkaahvgkhvgkaalthyl",
+            "Gwgsffkkaahvgkhvgkaalthyl",
+            "GWGSFFkkAAHVGkHVGkAALTHYL",
+            "gwgsffkkaahvgKHVGKAALTHYL",
+            "gwgsffkkaahvgKhvgKaalthyl",
+            "GwGsffkkaahvGkhvGkaalthyl"
+        ]
+    },
+    "RRGWVLALVLRYGRR": {
+        "1": [
+            "rRGWVLALVLRYGRR",
+            "RrGWVLALVLRYGRR",
+            "RRgWVLALVLRYGRR",
+            "RRGwVLALVLRYGRR",
+            "RRGWvLALVLRYGRR"
+        ],
+        "0": [
+            "RRGWVLALVlRYGRR",
+            "rRGWVLALVlRYGRR",
+            "RrGWVLALVlRYGRR",
+            "RRgWVLALVlRYGRR",
+            "RRGwVLALVlRYGRR",
+            "RRGWvLALVlRYGRR"
+        ]
+    },
+    "RRGWVLALYLRYGRR": {
+        "1": [
+            "rRGWVLALYLRYGRR",
+            "RRgWVLALYLRYGRR",
+            "RRGWVLalYLRyGRR",
+            "RrGWvLALYlRYgRR"
+        ],
+        "0": [
+            "RRGWVLALYlRYGRR",
+            "RRGWVLALyLrYGRR",
+            "RRGwVLalYLRYGRR",
+            "rRGWVLALYLryGRR",
+            "RRGWvLALyLRYGrR",
+            "RRgWVLalYLRyGRr"
+        ]
+    },
+    "RRGWALRLVLAY": {
+        "1": [
+            "rRGWALRLVLAY",
+            "RrGWALRLVLAY",
+            "RRgWALRLVLAY",
+            "RRGWaLRLVLAY",
+            "RRGWALrLVLAY"
+        ],
+        "0": [
+            "RRGWALRLVlAY",
+            "RRGwALRLVLAY",
+            "RRGWAlRLVLAY",
+            "RRGWALRlVLAY",
+            "RRGWALRLVLaY",
+            "RRGWALRLVLAy"
+        ]
+    },
+    "KWKKLLKKPLLKKLLKKL": {
+        "1": [
+            "kwkkllkkpllkkllkkl",
+            "Kwkkllkkpllkkllkkl",
+            "kWkkllkkpllkkllkkl",
+            "kwkkllkkpLLkkllkkl",
+            "kwkkllkkpllkkllkkL",
+            "KWkkllkkpllkkllkkl"
+        ],
+        "0": [
+            "kWKKLLKKPLLKKLLKKL",
+            "KWKKLLKKPLLKKLLKKl",
+            "KWKKLLKKpLLKKLLKKL",
+            "kwKKLLKKPLLKKLLKKL",
+            "KWKKLLKKPllKKLLKKL"
+        ]
+    },
+    "NKKAGLFVVQFPKKY": {
+        "1": [
+            "nkkaglfvvqfpkky",
+            "nKkAGLFVVQFPKKY",
+            "NkKAGLFVVQFPKKy",
+            "NKkaglfVVQFPKKY",
+            "NkkaGlfVVQFPKKY",
+            "nKKaGLfvvqFPkky"
+        ],
+        "0": [
+            "NKkAGlFVVQfPKKy",
+            "NkkAglFvvQFPKkY",
+            "nkkaglFVVqfpKKY",
+            "NKKAGLfVvQfPKkY",
+            "nkKAGlFvVQFPkKy"
+        ]
+    },
+    "LVKKLLKLAMGFG": {
+        "1": [
+            "lvkkllklamgfg",
+            "Lvkkllklamgfg",
+            "lVkkllklamgfg",
+            "lvKkllklamgfg",
+            "lvkKllklamgfg",
+            "lvkkLlklamgfg"
+        ],
+        "0": [
+            "VkKKLLKLAMGFG",
+            "LvKLLKLAMGFGg",
+            "LVKKllKLaMGFG",
+            "LVKKLLkLAmGfG"
+        ]
+    },
+    "WLRRIKAWLRRIKA": {
+        "1": [
+            "wlrrikawlrrika",
+            "Wlrrikawlrrika",
+            "wLrrikawlrrika",
+            "wlRrikawlrrika",
+            "wlrRikawlrrika",
+            "WLrrikawlrrika"
+        ],
+        "0": [
+            "wLRRIKAWLRRIKA",
+            "WlRRIKAWLRRIKA",
+            "WLrRIKAWLRRIKA",
+            "WLRrIKAWLRRIKA",
+            "WLrrIKAWLRRIKA"
+        ]
+    },
+    "RRGWARRLAFAFGRR": {
+        "1": [
+            "rrgwarrlafafgrr",
+            "Rrgwarrlafafgrr",
+            "rrgwarRLafafgrr",
+            "RrGWarRLafafgrr",
+            "rrgWarrlafafgRR",
+            "rrgwarrlafafgRr"
+        ],
+        "0": [
+            "rRGWARRLAFAFGRR",
+            "rrGWARRLAFAFGRR",
+            "RRGWARRLAFaFGRR",
+            "RRGWaRRLAFAFGRR",
+            "RRGWARRLafAFGRR"
+        ]
+    }
+}

finetune.py

@@ -0,0 +1,201 @@
+import argparse
+import json
+import logging
+import os
+import time
+
+from dataset import PeptidePairDataset, PeptidePairPicDataset, SimplePairClsDataset
+from network import DMutaPeptide, DMutaPeptideCNN#, DMutaPeptideWiden
+from sklearn.model_selection import KFold
+from train import train_cls
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader, WeightedRandomSampler, RandomSampler, Subset
+import numpy as np
+from loss import MLCE, SuperLoss, LogCoshLoss, BMCLoss
+from utils import set_seed
+
+
+parser = argparse.ArgumentParser(description='resnet26')
+# model setting
+parser.add_argument('--model', type=str, default='resnet34',
+                    help='resnet34 resnet50 densenet')
+parser.add_argument('--q-encoder', dest='q_encoder', type=str, default='cnn',
+                    help='lstm mamba mla')
+parser.add_argument("--side-enc", dest='side_enc', type=str, default='lstm',
+                    help="use side features")
+parser.add_argument('--channels', type=int, default=16)
+parser.add_argument('--fusion', type=str, default='att',
+                    help='mlp att diff')
+parser.add_argument('--glob-feat', dest='glob_feat', action='store_true', default=False,
+                    help="use global features")
+parser.add_argument('--non-siamese', dest='non_siamese', action='store_true', default=False,
+                    help="use non-siamese architecture")
+parser.add_argument('--widen', action='store_true', default=False,
+                    help='use widen non-siamese architecture')
+
+# task & dataset setting
+parser.add_argument('--task', type=str, default='cls',
+                    help='reg or cls')
+parser.add_argument('--pdb-src', type=str, dest='pdb_src', default='af',
+                    help='af or hf')
+parser.add_argument('--data-ver', type=str, dest='data_ver', default='250228',
+                    help='data version')
+parser.add_argument('--one-way', action='store_true', dest='one_way', default=True,
+                    help='use one-way constructed dataset')
+parser.add_argument('--max-length', dest='max_length', type=int, default=30,
+                    help='Max length for sequence filtering')
+parser.add_argument('--split', type=int, default=5,
+                    help="Split k fold in cross validation (default: 5)")
+parser.add_argument('--run-folds', type=int, dest='run_folds', nargs='+', default=-1,
+                    help='specify which folds to run')
+parser.add_argument('--seed', type=int, default=1,
+                    help="Seed (default: 1)")
+parser.add_argument('--pcs', action='store_true', default=False,
+                    help='Consider protease cut site')
+parser.add_argument('--mix-pcs', dest='mix_pcs', action='store_true', default=False,
+                    help='Consider protease cut site')
+parser.add_argument('--resize', type=int, default=[768], nargs='+',
+                    help='resize the image')
+parser.add_argument('--llm-data', action='store_true', default=False,
+                    help='Use LLM augmentation data')
+
+# training setting
+parser.add_argument('--gpu', type=int, default=0,
+                    help='GPU index to use, -1 for CPU (default: 0)')
+parser.add_argument('--batch-size', type=int, dest='batch_size', default=32,
+                    help='input batch size for training (default: 128)')
+parser.add_argument('--epochs', type=int, default=50,
+                    help='number of epochs to train (default: 100)')
+parser.add_argument('--lr', type=float, default=0.001,
+                    help='learning rate (default: 0.001)')
+parser.add_argument('--decay', type=float, default=0.0005,
+                    help='weight decay (default: 0.0005)')
+parser.add_argument('--warm-steps', type=int, dest='warm_steps', default=0,
+                    help='number of warm start steps for learning rate (default: 10)')
+parser.add_argument('--patience', type=int, default=10,
+                    help='patience for early stopping (default: 10)')
+parser.add_argument('--pretrain', type=str, dest='pretrain', default='',
+                    help='path of the pretrain model')
+parser.add_argument('--metric-avg', type=str, dest='metric_avg', default='macro',
+                    help='metric average type')
+
+parser.add_argument('--loss', type=str, default='ce',
+                    help='loss function')
+parser.add_argument('--dir', action='store_true', default=False,
+                    help='use DIR')
+
+parser.add_argument('--bias-curri', dest='bias_curri', action='store_true', default=False,
+                    help='directly use loss as the training data (biased) or not (unbiased)')
+parser.add_argument('--anti-curri', dest='anti_curri', action='store_true', default=False,
+                    help='easy to hard (curri), hard to easy (anti)')
+parser.add_argument('--std-coff', dest='std_coff', type=float, default=1,
+                    help='the hyper-parameter of std')
+
+parser.add_argument('--ft-epochs', dest='ft_epochs', type=int, default=15,
+                    help='fine-tune epochs')
+parser.add_argument('--ft-lr', dest='ft_lr', type=float, default=0.0002,
+                    help='fine-tune learning rate')
+
+parser.add_argument('--simple', dest='simple', action='store_true', default=False)
+
+args = parser.parse_args()
+
+if args.llm_data:
+    args.simple = True
+
+if args.simple:
+    args.one_way = True
+
+if args.run_folds == -1:
+    args.run_folds = list(range(args.split))
+
+def main():
+    set_seed(args.seed)
+    if args.task == 'reg':
+        args.classes = 1
+        if args.loss == "mse" or args.loss in ['ce']:
+            args.loss = 'mse'
+            criterion = nn.MSELoss()
+        elif args.loss == "smoothl1":
+            criterion = nn.SmoothL1Loss()
+        elif args.loss == "super":
+            criterion = SuperLoss()
+        elif args.loss in ["bmc", "bmc_ln"]:
+            criterion = BMCLoss()
+        else:
+            raise NotImplementedError("unimplemented regression task loss function")
+    elif args.task == 'cls':
+        args.classes = 2
+        if args.loss == 'ce' or args.loss in ['mse', 'smoothl1', 'super']:
+            args.loss = 'ce'
+            criterion = nn.CrossEntropyLoss()
+        else:
+            raise NotImplementedError("unimplemented classification task loss function")
+    else:
+        raise NotImplementedError("unimplemented task")
+    
+    if args.q_encoder in ['cnn', 'rn18']:
+        weight_dir = f'./run-{args.task}/{args.q_encoder}{f"-non-siamese" if args.non_siamese else ""}-{args.fusion}-{args.channels}{f"-{args.side_enc}" if args.side_enc else ""}{"-mixpcs" if args.mix_pcs else ""}{"-pcs" if args.pcs==True else ""}{"-simple" if args.simple else ""}{"-llm" if args.llm_data else ""}{"-" + "x".join(str(n) for n in args.resize) if args.resize else ""}{"-gf" if args.glob_feat else ""}{"-oneway" if args.one_way else ""}-{args.loss + "-dir" if args.dir else args.loss}-{str(args.batch_size)}-{str(args.lr)}-{str(args.epochs)}'
+    else:
+        weight_dir = f'./run-{args.task}/{args.q_encoder}{f"-non-siamese" if args.non_siamese else ""}-{args.fusion}-{args.channels}{"-simple" if args.simple else ""}{"-llm" if args.llm_data else ""}{"-gf" if args.glob_feat else ""}{"-oneway" if args.one_way else ""}-{args.loss + "-dir" if args.dir else args.loss}-{str(args.batch_size)}-{str(args.lr)}-{str(args.epochs)}'
+    
+    logging.basicConfig(handlers=[
+        logging.FileHandler(filename=os.path.join(weight_dir, "finetune.log"), encoding='utf-8', mode='w+'),
+        logging.StreamHandler()],
+        format="%(asctime)s: %(message)s", datefmt="%F %T", level=logging.INFO)
+    
+    logging.info(f'Finetuning: {weight_dir}')
+
+    device = torch.device("cpu" if args.gpu == -1 or not torch.cuda.is_available() else f"cuda:{args.gpu}")
+
+    logging.info(f'Loading Training Dataset')
+    train_set = SimplePairClsDataset(pad_length=args.max_length, ftr2=True, gf=args.glob_feat, q_encoder=args.q_encoder, side_enc=args.side_enc, pcs=args.pcs, resize=args.resize)
+
+    logging.info('Loading Test Dataset')
+    if args.q_encoder in ['cnn', 'rn18']:
+        test_set = PeptidePairPicDataset(mode='r2_case', pad_length=args.max_length, task=args.task, gf=args.glob_feat, side_enc=args.side_enc, pcs=args.pcs, resize=args.resize)
+    else:
+        test_set = PeptidePairDataset(mode='r2_case', pad_length=args.max_length, task=args.task, gf=args.glob_feat)
+
+    train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True, drop_last=True, num_workers=8, pin_memory=True)
+    test_loader = DataLoader(test_set, batch_size=args.batch_size, shuffle=False, num_workers=8, pin_memory=True)
+
+    best_perform_list = [[] for i in range(5)]
+
+    for fold in range(args.split):
+        logging.info(f'Finetuning Fold {fold}')
+        logging.info(f'Fold {fold}  Train set:{len(train_set)}, Test set: {len(test_set)}')
+        # if args.widen:
+        #     model = DMutaPeptideWiden(q_encoder=args.q_encoder, classes=args.classes, channels=args.channels, dir=args.dir, gf=args.glob_feat, fusion=args.fusion, side_enc=args.side_enc)
+        # else:
+        if args.q_encoder in ['cnn', 'rn18']:
+            model = DMutaPeptideCNN(q_encoder=args.q_encoder, classes=args.classes, channels=args.channels, dir=args.dir, gf=args.glob_feat, side_enc=args.side_enc, fusion=args.fusion, non_siamese=args.non_siamese)
+        else:
+            model = DMutaPeptide(q_encoder=args.q_encoder, classes=args.classes, channels=args.channels, dir=args.dir, gf=args.glob_feat, fusion=args.fusion, non_siamese=args.non_siamese)
+
+        weights_path = f"{weight_dir}/model_{fold}.pth"
+
+        model.to(device)
+        # model.load_state_dict(torch.load(weights_path.replace('.pth', '_test.pth'), map_location=device), strict=False)
+        model.load_state_dict(torch.load(weights_path, map_location=device), strict=False)
+
+        optimizer = torch.optim.AdamW(model.parameters(), lr=args.ft_lr)
+
+        best_metric = -float('inf')
+
+        if args.task == 'cls':
+            for epoch in range(1, args.ft_epochs + 1):
+                train_loss, ap, auc, f1, acc = train_cls(args, epoch, model, train_loader, test_loader, device, criterion, optimizer)
+                logging.info(f'Epoch: {epoch:03d} Train Loss: {train_loss:.3f}, ap: {ap:.3f}, auc: {auc:.3f}, f1: {f1:.3f}, acc: {acc:.3f}')
+                avg_metric = ap + auc #+ f1 + acc
+                if avg_metric > best_metric:
+                    logging.info(f'Epoch: {epoch:03d} New best VALIDATION metrics')
+                    best_metric = avg_metric
+                    best_perform_list[fold] = np.asarray([ap, auc, f1, acc])
+                    torch.save(model.state_dict(), weights_path.replace('.pth', '_ft.pth'))
+
+
+
+if __name__ == "__main__":
+    main()

gradcam.py

@@ -0,0 +1,407 @@
+import torch
+import torch.nn.functional as F
+import numpy as np
+from PIL import Image
+import matplotlib.pyplot as plt
+from matplotlib.colors import ListedColormap
+from torchvision import transforms
+from network import DMutaPeptideCNN
+from dataset import draw_peptide, encode_sequence
+
+class GradCAMMulti:
+    def __init__(self, model):
+        self.model = model
+        self.has_side_enc = hasattr(model, 'side_encoder') and model.side_encoder is not None
+
+    def generate(self, img1, img2, seq1=None, seq2=None, target_class=1):
+        self.model.eval()
+        
+        # 先计算两个图的原始CAM（未归一化）
+        cam1_raw = self._compute_cam_for_input(img1, img2, seq1, seq2, target_class, analyze_idx=0, normalize=False)
+        cam2_raw = self._compute_cam_for_input(img1, img2, seq1, seq2, target_class, analyze_idx=1, normalize=False)
+        
+        # 使用全局最大最小值进行归一化
+        global_min = min(cam1_raw.min(), cam2_raw.min())
+        global_max = max(cam1_raw.max(), cam2_raw.max())
+        
+        hm_cnn1 = self._normalize_cam(cam1_raw, global_min, global_max)
+        hm_cnn2 = self._normalize_cam(cam2_raw, global_min, global_max)
+        
+        if not self.has_side_enc:
+            return hm_cnn1, hm_cnn2
+        
+        # 序列热力图也使用相同的策略
+        seq1_raw = self._compute_seq_cam_for_input(img1, img2, seq1, seq2, target_class, analyze_idx=0, normalize=False)
+        seq2_raw = self._compute_seq_cam_for_input(img1, img2, seq1, seq2, target_class, analyze_idx=1, normalize=False)
+        
+        seq_global_min = min(seq1_raw.min(), seq2_raw.min())
+        seq_global_max = max(seq1_raw.max(), seq2_raw.max())
+        
+        hm_seq1 = self._normalize_cam(seq1_raw, seq_global_min, seq_global_max)
+        hm_seq2 = self._normalize_cam(seq2_raw, seq_global_min, seq_global_max)
+        
+        return hm_cnn1, hm_cnn2, hm_seq1, hm_seq2
+    
+    def _normalize_cam(self, cam, global_min, global_max):
+        """使用全局最大最小值归一化"""
+        cam_norm = (cam - global_min) / (global_max - global_min + 1e-8)
+        return np.uint8(cam_norm * 255)
+
+    def _compute_cam_for_input(self, img1, img2, seq1, seq2, target_class, analyze_idx, normalize=True):
+        """
+        analyze_idx: 0 分析 img1, 1 分析 img2
+        normalize: 是否在此函数内归一化（False时返回原始numpy数组）
+        """
+        if analyze_idx == 0:
+            img_analyze = img1.clone().requires_grad_(True)
+            img_other = img2.detach()
+        else:
+            img_analyze = img2.clone().requires_grad_(True)
+            img_other = img1.detach()
+        
+        activations = []
+        gradients = []
+        
+        def fwd_hook(mod, inp, out):
+            activations.append(out)
+            return out
+        
+        def bwd_hook(mod, grad_in, grad_out):
+            gradients.append(grad_out[0])
+        
+        last_conv = self.model.q_encoder[7][-1].conv2
+        fwd_h = last_conv.register_forward_hook(fwd_hook)
+        bwd_h = last_conv.register_full_backward_hook(bwd_hook)
+        
+        try:
+            if self.has_side_enc:
+                if analyze_idx == 0:
+                    inputs = ((img_analyze, seq1), (img_other, seq2))
+                else:
+                    inputs = ((img_other, seq1), (img_analyze, seq2))
+            else:
+                if analyze_idx == 0:
+                    inputs = (img_analyze, img_other)
+                else:
+                    inputs = (img_other, img_analyze)
+            
+            logits = self.model(inputs)
+            if isinstance(logits, tuple):
+                logits = logits[0]
+            score = logits[0, target_class]
+            
+            self.model.zero_grad()
+            score.backward()
+            
+            act = activations[analyze_idx]
+            grad = gradients[-(analyze_idx + 1)]
+            
+            if grad is None:
+                cam = np.zeros((img1.shape[2], img1.shape[3]), dtype=np.float32)
+                return np.uint8(cam * 255) if normalize else cam
+            
+            # 使用梯度的绝对值来计算权重
+            α = grad.abs().mean(dim=(2, 3), keepdim=True)
+            cam = (α * act).sum(dim=1, keepdim=True)
+            cam = cam.abs()  # 取绝对值
+            
+            cam = F.interpolate(cam, size=img1.shape[2:], mode='bilinear', align_corners=False)
+            cam = cam.squeeze().detach().cpu().numpy()
+            
+            if normalize:
+                cam = (cam - cam.min()) / (cam.max() - cam.min() + 1e-8)
+                return np.uint8(cam * 255)
+            else:
+                return cam  # 返回原始float数组
+            
+        finally:
+            fwd_h.remove()
+            bwd_h.remove()
+
+    def _compute_seq_cam_for_input(self, img1, img2, seq1, seq2, target_class, analyze_idx, normalize=True):
+        """序列CAM计算"""
+        if analyze_idx == 0:
+            seq_analyze = seq1.clone().requires_grad_(True)
+            seq_other = seq2.detach()
+        else:
+            seq_analyze = seq2.clone().requires_grad_(True)
+            seq_other = seq1.detach()
+        
+        activations = []
+        gradients = []
+        
+        def fwd_hook(mod, inp, out):
+            activations.append(out)
+            return out
+        
+        def bwd_hook(mod, grad_in, grad_out):
+            gradients.append(grad_out[0])
+        
+        fwd_h = self.model.side_encoder.mamba.register_forward_hook(fwd_hook)
+        bwd_h = self.model.side_encoder.mamba.register_full_backward_hook(bwd_hook)
+        
+        try:
+            if analyze_idx == 0:
+                inputs = ((img1.detach(), seq_analyze), (img2.detach(), seq_other))
+            else:
+                inputs = ((img1.detach(), seq_other), (img2.detach(), seq_analyze))
+            
+            logits = self.model(inputs)
+            if isinstance(logits, tuple):
+                logits = logits[0]
+            score = logits[0, target_class]
+            
+            self.model.zero_grad()
+            score.backward()
+            
+            act = activations[analyze_idx]
+            grad = gradients[-(analyze_idx + 1)]
+            
+            if grad is None:
+                cam_seq = np.zeros(seq1.shape[1], dtype=np.float32)
+                return np.uint8(cam_seq * 255) if normalize else cam_seq
+            
+            # 使用绝对值
+            α = grad.abs().mean(dim=1, keepdim=True)
+            cam_seq = (act * α).sum(dim=2).abs()
+            
+            cam_seq = cam_seq.squeeze().detach().cpu().numpy()
+            
+            if normalize:
+                cam_seq = (cam_seq - cam_seq.min()) / (cam_seq.max() - cam_seq.min() + 1e-8)
+                return np.uint8(cam_seq * 255)
+            else:
+                return cam_seq  # 返回原始float数组
+            
+        finally:
+            fwd_h.remove()
+            bwd_h.remove()
+
+
+def plot_seq_heat_tailpad(
+    seq: str,
+    heatmap: np.ndarray,
+    keep_pad: int = 2,
+    ax=None,
+    cmap='Oranges',
+    border_width: float = 2.0,
+    figsize_per_base: float = 0.3
+):
+    """
+    seq:             原始氨基酸序列，不含 padding
+    heatmap:         np.uint8 数组，长度 N = L + padding_length
+    keep_pad:        在末端保留的 padding 方块数
+    ax:              matplotlib Axes
+    cmap:            配色方案
+    border_width:    最外圈边框宽度
+    figsize_per_base: 每个位置宽度，用于自动计算 figsize
+    """
+    N = len(heatmap)
+    L = len(seq)
+    # 实际要显示的长度：0 ~ end_pos
+    end_pos = min(L + keep_pad, N)
+    data = heatmap[:end_pos].astype(np.float32) / 255.0  # 归一化到 [0,1]
+    M = end_pos
+
+    # 构造 x 轴标签：前 L 位显示字母，后面 keep_pad 位留空
+    xticks = [seq[i] if i < L else '' for i in range(M)]
+
+    if ax is None:
+        fig, ax = plt.subplots(
+            figsize=(figsize_per_base * M, 1.5),
+            dpi=100
+        )
+    im = ax.imshow(
+        data[np.newaxis, :],      # 变为 shape (1, M)
+        cmap=cmap,
+        aspect='auto',
+        interpolation='nearest',
+        vmin=0, vmax=1
+    )
+
+    # x 轴在顶部显示
+    ax.set_xticks(np.arange(M))
+    ax.set_xticklabels(xticks, fontsize=12)
+    ax.xaxis.set_ticks_position('top')
+    ax.xaxis.set_label_position('top')
+
+    # 隐藏 y 轴
+    ax.set_yticks([])
+
+    # 四周画一圈粗边框
+    for spine in ax.spines.values():
+        spine.set_visible(True)
+        spine.set_linewidth(border_width)
+        spine.set_edgecolor('black')
+
+    return im, ax
+
+
+def inv_norm(tensor: torch.Tensor, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]):
+    tensor = tensor.clone()
+    for t, m, s in zip(tensor, mean, std):
+        t.mul_(s).add_(m)
+    return -tensor
+
+
+def diff_hm(hm1, hm2):
+    diff = hm2.astype(np.float32) - hm1.astype(np.float32) + 127.
+    return np.clip(diff, 0, 255).astype(np.uint8)
+
+def get_resnet18_last_conv(model):
+    """
+    获取 ResNet18 的最后一个卷积层
+    从打印的结构可知：
+    - model.q_encoder[7] 是 layer4 (Sequential with 2 BasicBlocks)
+    - model.q_encoder[7][-1] 是最后一个 BasicBlock
+    - model.q_encoder[7][-1].conv2 是最后一个卷积层
+    """
+    return model.q_encoder[7][-1].conv2
+
+def add_alpha_to_cmap(base_cmap='RdBu_r', name='RdBu_alpha', center_alpha=0.0):
+    """
+    给已有的colormap添加alpha通道
+    
+    Args:
+        base_cmap: 基础colormap名称
+        name: 新colormap名称
+        center_alpha: 中心透明度
+    """
+    from matplotlib import colormaps as cm
+    
+    # 获取基础colormap
+    base = cm.get_cmap(base_cmap)
+    
+    # 创建新的颜色数组
+    n = 256
+    colors = base(np.linspace(0, 1, n))
+    
+    # 修改alpha通道：中心透明，两端不透明
+    alpha_values = np.abs(np.linspace(-1, 1, n))  # V型曲线
+    alpha_values = alpha_values ** 0.7  # 调整曲线形状
+    alpha_values = alpha_values * (1 - center_alpha) + center_alpha
+    
+    colors[:, 3] = alpha_values
+    
+    return ListedColormap(colors, name=name)
+
+def main(sequence1, sequence2, model):
+    img1 = draw_peptide(sequence1, pcs=True)
+    img2 = draw_peptide(sequence2, pcs=True)
+    img1_raw = transforms.ToPILImage()(inv_norm(img1))
+    img2_raw = transforms.ToPILImage()(inv_norm(img2))
+
+    # img1_raw.save('./gradcam/img1.png')
+
+    img1 = img1.unsqueeze(0).to(torch.device('cuda'))
+    img2 = img2.unsqueeze(0).to(torch.device('cuda'))
+
+    has_side_enc = hasattr(model, 'side_enc') and model.side_enc
+    
+    if has_side_enc:
+        # 假设序列已 one-hot 或 embedding，直接作 tensor
+        seq1 = encode_sequence(sequence1, 30).unsqueeze(0).to(torch.device('cuda'))
+        seq2 = encode_sequence(sequence2, 30).unsqueeze(0).to(torch.device('cuda'))
+
+        # 挂 hook - ResNet18 的最后一个卷积层
+        cam = GradCAMMulti(model)
+
+        # 生成热力图
+        hm_c1, hm_c2, hm_s1, hm_s2 = cam.generate(
+            img1, img2, seq1, seq2,
+            target_class=1
+        )
+    else:
+        seq1 = seq2 = None
+        cam = GradCAMMulti(model)
+        
+        hm_c1, hm_c2 = cam.generate(
+            img1, img2, seq1, seq2,
+            target_class=1
+        )
+
+    # 可视化 CNN 热力图
+    def show_img_heat(img_pil, hm, name, cmap='jet', alpha=0.4):
+        plt.figure(figsize=(5, 5))
+        img = np.array(img_pil.resize(hm.shape[::-1]))
+        plt.imshow(img, alpha=0.8)
+        plt.imshow(hm, cmap=cmap, alpha=alpha)
+        plt.axis('off')
+        plt.savefig(f'{name}.png',
+                    bbox_inches='tight',
+                    pad_inches=0,
+                    dpi=200)
+        plt.close()
+
+    diff_cmap = add_alpha_to_cmap()
+
+    hm_diff = diff_hm(hm_c1, hm_c2)
+
+    show_img_heat(img1_raw, hm_c1, f'./gradcam/{sequence1}-temp')
+    show_img_heat(img2_raw, hm_c2, f'./gradcam/{sequence2}-muta')
+    show_img_heat(img2_raw, hm_diff, f'./gradcam/{sequence2}-diff', cmap=diff_cmap, alpha=0.8)
+
+    # 可视化序列热力图（如果有）
+    if has_side_enc:
+        fig, axes = plt.subplots(
+            2, 1,
+            figsize=(len(sequence1) * 0.3, 1.25),
+            constrained_layout=True
+        )
+        plot_seq_heat_tailpad(
+            sequence1, hm_s1,
+            keep_pad=0,
+            ax=axes[0],
+            cmap='jet'
+        )
+        plot_seq_heat_tailpad(
+            sequence2, hm_s2,
+            keep_pad=0,
+            ax=axes[1],
+            cmap='jet'
+        )
+        plt.savefig(f'./gradcam/{sequence1}_seq.svg')
+        plt.close()
+
+        fig, ax = plt.subplots(
+            1, 1,
+            figsize=(len(sequence1) * 0.3, 0.625),
+            constrained_layout=True
+        )
+        plot_seq_heat_tailpad(
+            sequence2, diff_hm(hm_s1, hm_s2),
+            keep_pad=0,
+            ax=ax,
+            cmap=diff_cmap
+        )
+        plt.savefig(f'./gradcam/{sequence2}_diff.svg')
+        plt.close()
+
+
+# —— 使用示例 —— #
+if __name__ == "__main__":
+    # 1) load model
+    model = DMutaPeptideCNN(
+        q_encoder='rn18',
+        classes=2,
+        channels=16,
+        dir=False,
+        gf=False,
+        side_enc='mamba',
+        fusion='diff'
+    )
+    model.eval().to(torch.device('cuda'))
+    model.load_state_dict(
+        torch.load("run-cls/rn18-diff-16-mamba-pcs-768-ce-32-0.001-50/model_0.pth",
+                   map_location=torch.device('cuda')),
+        strict=True
+    )
+
+    # 2) 准备数据
+    sequence1 = "KWKIKWPVKWFKML"
+    sequence2 = "KWKIKWPVKWfKML"
+    main(sequence1, sequence2, model)
+    
+    sequence1 = "KKLFKKILKYL"
+    sequence2 = "KKLFKKiLKYL"
+    main(sequence1, sequence2, model)

infer.py

@@ -0,0 +1,201 @@
+import argparse
+from dataset import PeptidePairDataset, PeptidePairPicDataset
+from network import DMutaPeptide, DMutaPeptideCNN
+from train import move_to_device
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader
+import numpy as np
+from utils import set_seed
+import pandas as pd
+from torchmetrics import MeanAbsoluteError, RelativeSquaredError, PearsonCorrCoef, KendallRankCorrCoef, F1Score, Accuracy, AveragePrecision, AUROC
+
+parser = argparse.ArgumentParser(description='resnet26')
+# model setting
+parser.add_argument('--model', type=str, default='resnet34',
+                    help='resnet34 resnet50 densenet')
+parser.add_argument('--q-encoder', dest='q_encoder', type=str, default='cnn',
+                    help='lstm mamba mla')
+parser.add_argument('--channels', type=int, default=16)
+parser.add_argument("--side-enc", dest='side_enc', type=str, default=None,
+                    help="use side features")
+parser.add_argument('--fusion', type=str, default='mlp',
+                    help='mlp att')
+parser.add_argument('--glob-feat', dest='glob_feat', action='store_true', default=False,
+                    help="use global features")
+parser.add_argument('--non-siamese', dest='non_siamese', action='store_true', default=False,
+                    help="use non-siamese architecture")
+
+# task & dataset setting
+parser.add_argument('--task', type=str, default='cls',
+                    help='reg or cls')
+parser.add_argument('--pdb-src', type=str, dest='pdb_src', default='af',
+                    help='af or hf')
+parser.add_argument('--data-ver', type=str, dest='data_ver', default='250228',
+                    help='data version')
+parser.add_argument('--one-way', action='store_true', dest='one_way', default=False,
+                    help='use one-way constructed dataset')
+parser.add_argument('--max-length', dest='max_length', type=int, default=30,
+                    help='Max length for sequence filtering')
+parser.add_argument('--resize', type=int, default=[768], nargs='+',
+                    help='resize the image')
+parser.add_argument('--split', type=int, default=5,
+                    help="Split k fold in cross validation (default: 5)")
+parser.add_argument('--seed', type=int, default=1,
+                    help="Seed (default: 1)")
+parser.add_argument('--pcs', action='store_true', default=False,
+                    help='Consider protease cut site')
+parser.add_argument('--mix-pcs', dest='mix_pcs', action='store_true', default=False,
+                    help='Consider protease cut site')
+
+# training setting
+parser.add_argument('--gpu', type=int, default=0,
+                    help='GPU index to use, -1 for CPU (default: 0)')
+parser.add_argument('--batch-size', type=int, dest='batch_size', default=32,
+                    help='input batch size for training (default: 128)')
+parser.add_argument('--epochs', type=int, default=50,
+                    help='number of epochs to train (default: 100)')
+parser.add_argument('--lr', type=float, default=0.001,
+                    help='learning rate (default: 0.001)')
+parser.add_argument('--decay', type=float, default=0.0005,
+                    help='weight decay (default: 0.0005)')
+parser.add_argument('--warm-steps', type=int, dest='warm_steps', default=0,
+                    help='number of warm start steps for learning rate (default: 10)')
+parser.add_argument('--patience', type=int, default=10,
+                    help='patience for early stopping (default: 10)')
+parser.add_argument('--pretrain', type=str, dest='pretrain', default='',
+                    help='path of the pretrain model')  # /home/duadua/Desktop/fetal/3dpretrain/runs/e50.pth
+parser.add_argument('--metric-avg', type=str, dest='metric_avg', default='macro',
+                    help='metric average type')
+
+parser.add_argument('--loss', type=str, default='ce',
+                    help='loss function')
+parser.add_argument('--dir', action='store_true', default=False,
+                    help='use DIR')
+
+parser.add_argument('--simple', dest='simple', action='store_true', default=False)
+parser.add_argument('--llm-data', dest='llm_data', action='store_true', default=False)
+parser.add_argument('--uda', type=str, default=None)
+
+args = parser.parse_args()
+
+if args.llm_data:
+    args.simple = True
+
+if args.simple:
+    args.one_way = True
+
+if args.mix_pcs:
+    args.pcs = 'mix'
+
+if args.q_encoder in ['cnn', 'rn18']:
+    weight_dir = f'./run-{args.task}/{f"non-siamese-" if args.non_siamese else ""}{args.q_encoder}-{args.fusion}-{args.channels}{f"-{args.side_enc}" if args.side_enc else ""}{"-mixpcs" if args.mix_pcs else ""}{"-pcs" if args.pcs==True else ""}{"-simple" if args.simple else ""}{"-llm" if args.llm_data else ""}{"-" + "x".join(str(n) for n in args.resize) if args.resize else ""}{"-gf" if args.glob_feat else ""}{"-oneway" if args.one_way else ""}-{args.loss + "-dir" if args.dir else args.loss}-{str(args.batch_size)}-{str(args.lr)}-{str(args.epochs)}'
+else:
+    weight_dir = f'./run-{args.task}/{f"non-siamese-" if args.non_siamese else ""}{args.q_encoder}-{args.fusion}-{args.channels}{"-simple" if args.simple else ""}{"-llm" if args.llm_data else ""}{"-gf" if args.glob_feat else ""}{"-oneway" if args.one_way else ""}-{args.loss + "-dir" if args.dir else args.loss}-{str(args.batch_size)}-{str(args.lr)}-{str(args.epochs)}'
+
+if args.uda:
+    weight_dir += f'/uda_{args.uda}'
+
+print(weight_dir)
+
+def metrics(preds, gt, task):
+    avg = 'marco'
+    device = preds.device
+    if task == 'cls':
+        metric_1 = AveragePrecision(average=avg, task='binary').to(device)
+        metric_2 = AUROC(average=avg, task='binary').to(device)
+        metric_3 = F1Score(average=avg, task='binary').to(device)
+        metric_4 = Accuracy(average=avg, task='binary').to(device)
+        all_metrics = [metric_1(preds, gt).item(), 
+                metric_2(preds, gt).item(), 
+                metric_3(preds, gt).item(), 
+                metric_4(preds, gt).item()]
+    
+    elif task == 'reg':
+        metric_1 = MeanAbsoluteError().to(device)
+        metric_2 = RelativeSquaredError(num_outputs=1).to(device)
+        metric_3 = PearsonCorrCoef(num_outputs=1).to(device)
+        metric_4 = KendallRankCorrCoef(num_outputs=1).to(device)
+        all_metrics = [metric_1(preds, gt).item(), 
+                metric_2(preds, gt).item(), 
+                metric_3(preds.squeeze(), gt.squeeze()).mean().item(), 
+                metric_4(preds.squeeze(), gt.squeeze()).mean().item()]
+
+    return [f'{i * 100:.2f}' for i in all_metrics]
+    
+
+def main(dataset):
+    set_seed(args.seed)
+    if args.task == 'reg':
+        args.classes = 1
+    elif args.task == 'cls':
+        args.classes = 2
+    else:
+        raise NotImplementedError("unimplemented task")
+
+    device = torch.device("cpu" if args.gpu == -1 or not torch.cuda.is_available() else f"cuda:{args.gpu}")
+    
+    if args.q_encoder in ['cnn', 'rn18']:
+        model = DMutaPeptideCNN(q_encoder=args.q_encoder, classes=args.classes, channels=args.channels, dir=args.dir, gf=args.glob_feat, side_enc=args.side_enc, fusion=args.fusion, non_siamese=args.non_siamese).to(device).eval()
+        test_set = PeptidePairPicDataset(mode=dataset, pad_length=args.max_length, task=args.task, gf=args.glob_feat, side_enc=args.side_enc, pcs=args.pcs, resize=args.resize)
+    else:
+        model = DMutaPeptide(q_encoder=args.q_encoder, classes=args.classes, channels=args.channels, dir=args.dir, gf=args.glob_feat, fusion=args.fusion, non_siamese=args.non_siamese).to(device).eval()
+        test_set = PeptidePairDataset(mode=dataset, pad_length=args.max_length, task=args.task, gf=args.glob_feat)
+    
+    test_loader = DataLoader(test_set, batch_size=args.batch_size, shuffle=False)
+
+    df = pd.DataFrame()
+    raw_preds = []
+    ckpt_names = ['model_uda_teacher'] if args.uda else [f'model_{i}_test' for i in range(5)]
+    for i in ckpt_names:
+        model.load_state_dict(torch.load(f'{weight_dir}/{i}.pth', map_location=device))
+        preds = []
+        gt_list_valid = []
+        with torch.no_grad():
+            for data in test_loader:
+                x, gt = data
+                gt_list_valid.append(gt.to(device))
+                out = model(move_to_device(x, device))
+                if args.dir:
+                    out, _ = out
+                preds.append(out)
+        r_pred = torch.cat(preds, dim=0)
+        if args.task == 'reg':
+            preds = r_pred.cpu().numpy()
+        elif args.task == 'cls':
+            preds = torch.softmax(r_pred, dim=-1)[:, 1].cpu().numpy()
+        gt_tensor = torch.cat(gt_list_valid, dim=0)
+        gt_list_valid = gt_tensor.cpu().numpy()
+        df[f'{i}'] = preds
+        raw_preds.append(r_pred)
+    if args.task == 'cls':
+        preds_tensor = torch.softmax(torch.stack(raw_preds, 0).mean(0), dim=-1)[:, 1]
+    elif args.task == 'reg':
+        preds_tensor = torch.stack(raw_preds, 0).mean(0)
+    df['fusion'] = preds_tensor.cpu().numpy()
+    df['gt'] = gt_list_valid
+    df.to_csv(f'{weight_dir}/preds_{dataset}.csv', index=False)
+    return metrics(preds_tensor, gt_tensor, args.task)
+
+
+if __name__ == '__main__':
+    if args.task == 'cls':
+        df = pd.DataFrame(columns=['dataset', 'AUPRC', 'AUROC', 'F1', 'ACC'])
+    elif args.task == 'reg':
+        df = pd.DataFrame(columns=['dataset', 'MAE', 'RSE', 'PCC', 'KCC'])
+
+    datasets = [
+        'r2_case',
+        # 'r2_case_'
+        "test", 
+        # "mhb", 
+        # "nacl", 
+        # "125fbs", 
+        # "25fbs",
+        ]
+
+    for dataset in datasets:
+        results = main(dataset)
+        df.loc[len(df) + 1] = [dataset] + results
+    df.to_csv(f'{weight_dir}/inference_results.csv', index=False)
+    print(df)

infer_case.py

@@ -0,0 +1,245 @@
+import argparse
+import time
+from dataset import PeptidePairPicCaseDataset, encode_sequence
+from network import DMutaPeptideCNN
+from train import move_to_device
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader
+import numpy as np
+from utils import set_seed
+import pandas as pd
+
+parser = argparse.ArgumentParser(description='resnet26')
+# model setting
+parser.add_argument('--model', type=str, default='resnet34',
+                    help='resnet34 resnet50 densenet')
+parser.add_argument('--q-encoder', dest='q_encoder', type=str, default='cnn',
+                    help='lstm mamba mla')
+parser.add_argument('--channels', type=int, default=16)
+parser.add_argument("--side-enc", dest='side_enc', type=str, default='lstm',
+                    help="use side features")
+parser.add_argument('--fusion', type=str, default='att',
+                    help='mlp att')
+parser.add_argument('--glob-feat', dest='glob_feat', action='store_true', default=False,
+                    help="use global features")
+parser.add_argument('--non-siamese', dest='non_siamese', action='store_true', default=False,
+                    help="use non-siamese architecture")
+
+# task & dataset setting
+parser.add_argument('--task', type=str, default='cls',
+                    help='reg or cls')
+parser.add_argument('--one-way', action='store_true', dest='one_way', default=False,
+                    help='use one-way constructed dataset')
+parser.add_argument('--max-length', dest='max_length', type=int, default=30,
+                    help='Max length for sequence filtering')
+parser.add_argument('--resize', type=int, default=[768], nargs='+',
+                    help='resize the image')
+parser.add_argument('--split', type=int, default=5,
+                    help="Split k fold in cross validation (default: 5)")
+parser.add_argument('--seed', type=int, default=1,
+                    help="Seed for model initialization (default: 1)")
+parser.add_argument('--pcs', action='store_true', default=False,
+                    help='Consider protease cut site')
+parser.add_argument('--mix-pcs', dest='mix_pcs', action='store_true', default=False,
+                    help='Consider protease cut site')
+
+# training setting
+parser.add_argument('--gpu', type=int, default=0,
+                    help='GPU index to use, -1 for CPU (default: 0)')
+parser.add_argument('--batch-size', type=int, dest='batch_size', default=32,
+                    help='input batch size for training (default: 128)')
+parser.add_argument('--epochs', type=int, default=50,
+                    help='number of epochs to train (default: 100)')
+parser.add_argument('--lr', type=float, default=0.001,
+                    help='learning rate (default: 0.001)')
+parser.add_argument('--decay', type=float, default=0.0005,
+                    help='weight decay (default: 0.0005)')
+parser.add_argument('--pretrain', type=str, dest='pretrain', default='',
+                    help='path of the pretrain model')
+parser.add_argument('--metric-avg', type=str, dest='metric_avg', default='macro',
+                    help='metric average type')
+
+parser.add_argument('--loss', type=str, default='ce',
+                    help='loss function')
+parser.add_argument('--dir', action='store_true', default=False,
+                    help='use DIR')
+
+parser.add_argument('--simple', dest='simple', action='store_true', default=False)
+parser.add_argument('--llm-data', dest='llm_data', action='store_true', default=False)
+
+# Case Study Specific
+parser.add_argument('--case', type=str, default='r2',
+                    help='case to infer')
+parser.add_argument('--use-ft', dest='use_ft', type=str, default='')
+
+args = parser.parse_args()
+
+if args.llm_data:
+    args.simple = True
+
+if args.simple:
+    args.one_way = True
+
+if args.mix_pcs:
+    args.pcs = 'mix'
+
+if args.gpu != -1:
+    torch.backends.cudnn.benchmark = True
+    torch.set_float32_matmul_precision('high')
+
+
+class FasterModelForCase(DMutaPeptideCNN):
+    def cache_temp_vector(self, seq):
+        if self.side_enc:
+            seq_seq = seq[1]
+            seq = seq[0]
+            if self.side_encoder.__class__.__name__ == 'MambaModel':
+                self.temp_seq_vector = self.norm(self.side_encoder(seq_seq))
+            else:
+                self.temp_seq_vector = self.norm(self.side_encoder(seq_seq)[0][:, -1, :])
+        self.temp_vector = self.norm(self.q_encoder(seq))
+
+    def forward(self, x, labels=None, epoch=0):
+        seq2 = x
+
+        if self.side_enc:
+            seq2_seq = seq2[1]
+            seq2 = seq2[0]
+
+        batch_size = seq2.shape[0]
+
+        fusion = []
+
+        # 获取两个序列的编码结果
+        fusion.append(self.temp_vector.expand(batch_size, -1))
+        fusion.append(self.norm(self.q_encoder_2(seq2)))
+        if self.side_enc:
+            fusion.append(self.temp_seq_vector.expand(batch_size, -1))
+            if self.side_encoder.__class__.__name__ == 'MambaModel':
+                fusion.append(self.norm(self.side_encoder_2(seq2_seq)))
+            else:
+                fusion.append(self.norm(self.side_encoder_2(seq2_seq)[0][:, -1, :]))
+
+        # 根据 fusion_method 决定融合方式
+        if self.fusion_method == 'mlp':
+            # 维持原有行为：拼接两个向量
+            fusion = torch.cat(fusion, dim=-1)
+        elif self.fusion_method == 'diff':
+            if not self.side_enc:
+                fusion = torch.cat([fusion[1] - fusion[0]] + fusion[2:], dim=-1)
+            else:
+                fusion = torch.cat([fusion[1] - fusion[0], fusion[3] - fusion[2]] + fusion[4:], dim=-1)
+        elif self.fusion_method == 'att':
+            # 使用 attention 融合:
+            # 先将两个向量堆叠成“tokens”，形状：(batch, 2, embed_dim)
+            tokens = torch.stack(fusion, dim=1)  # embed_dim 应该为 final_dim//2
+            # 利用 MultiheadAttention 进行自注意力计算
+            # 注意：因为采用 batch_first=True，所以输入形状为 (batch, seq_len, embed_dim)
+            attn_output, _ = self.attn(tokens, tokens, tokens)
+            # 将 attention 输出展平，得到形状 (batch, 2 * embed_dim)，即 (batch, final_dim)
+            fusion = attn_output.reshape(attn_output.size(0), -1)
+        else:
+            raise ValueError("Invalid fusion method: choose either 'mse' or 'att'.")
+
+        # 如果启用 DIR 模块，保留传入 FDS 前的特征表示
+        if self.DIR:
+            features = fusion
+            fusion = self.FDS.smooth(fusion, labels, epoch)
+        
+        pred = self.fc(fusion)
+
+        if self.DIR:
+            return pred, features
+        else:
+            return pred
+
+class CustomDataset(PeptidePairPicCaseDataset):
+    def __getitem__(self, idx):
+        variant  = self.variants[idx]
+        seq2, label = variant, variant
+        img2 = self.read_img(variant)
+
+        if self.side_enc:
+            img2 = (img2, encode_sequence(seq2, self.pad_length))
+
+        return img2, label
+
+def load_model(args, weight_path, device, temp_batch):
+    model = FasterModelForCase(q_encoder=args.q_encoder, classes=args.classes, channels=args.channels, dir=args.dir, gf=args.glob_feat, side_enc=args.side_enc, fusion=args.fusion, non_siamese=args.non_siamese).to(device).eval()
+    model.load_state_dict(torch.load(weight_path, map_location=device), strict=False)
+    model.cache_temp_vector(move_to_device(temp_batch, device))
+    model.compile()
+    return model
+
+
+def main():
+    set_seed(args.seed)
+    if args.task == 'reg':
+        args.classes = 1
+    elif args.task == 'cls':
+        args.classes = 2
+    else:
+        raise NotImplementedError("unimplemented task")
+    weight_dir = f'./run-{args.task}/{args.q_encoder}{f"-non-siamese" if args.non_siamese else ""}-{args.fusion}-{args.channels}{f"-{args.side_enc}" if args.side_enc else ""}{"-mixpcs" if args.mix_pcs else ""}{"-pcs" if args.pcs==True else ""}{"-simple" if args.simple else ""}{"-llm" if args.llm_data else ""}{"-" + "x".join(str(n) for n in args.resize) if args.resize else ""}{"-gf" if args.glob_feat else ""}{"-oneway" if args.one_way else ""}-{args.loss + "-dir" if args.dir else args.loss}-{str(args.batch_size)}-{str(args.lr)}-{str(args.epochs)}'
+    device = torch.device("cpu" if args.gpu == -1 or not torch.cuda.is_available() else f"cuda:{args.gpu}")
+    print(weight_dir)
+    print(device)
+    
+    test_set = CustomDataset(case=args.case, pad_length=args.max_length, side_enc=args.side_enc, pcs=True, resize=args.resize, gf=args.glob_feat)
+    test_loader = DataLoader(test_set, batch_size=192, shuffle=False, num_workers=16, pin_memory=True)
+    # test_loader = DataLoader(test_set, batch_size=192, shuffle=False, num_workers=8)
+
+    temp_batch = test_set.template_pic.unsqueeze(0)
+    if args.side_enc:
+        temp_batch = (temp_batch, test_set.template_seq.unsqueeze(0))
+    
+    models = [load_model(args, f'{weight_dir}/model_{i}{f"_{args.use_ft}" if args.use_ft else ""}.pth', device, temp_batch) for i in range(args.split)]
+    # models = [load_model(args, f'{weight_dir}/model_{i}{"_ft" if args.use_ft else ""}.pth', device, temp_batch) for i in [0]]
+
+    all_seqs = []
+    logits_batches = []  # 存放每个 batch 的 [m,B,2] avg_logits (CPU 上)
+
+    start_time = time.time()
+
+    with torch.no_grad():
+        for x, gt in test_loader:
+            # x: [B, ...] on CPU pin memory，gt: tuple of B strings
+            x = move_to_device(x, device, non_blocking=True)
+            # x = move_to_device(x, device)
+
+            # 1) 记录 5 个模型的 logits
+            # logits: [m,B,2]
+            logits = torch.zeros(len(models), len(gt), args.classes, device=device)
+            for i, m in enumerate(models):
+                logits[i] = m(x)
+            # avg_logits = sum_logits.div_(len(models))
+
+            # 3) 立刻搬到 CPU（pin_memory 下可以 non_blocking）
+            logits_batches.append(logits.cpu())
+            all_seqs.extend(gt)
+
+    # 拼接成 [n,2]，n = sum(batch_size)
+    all_logits = torch.cat(logits_batches, dim=1)  # [m,n,2]
+    
+    if args.task == 'reg':
+        preds = all_logits.mean(0).squeeze().tolist()
+    elif args.task == 'cls':
+        # 最后一次性 softmax，取正类概率
+        preds = torch.softmax(all_logits, dim=-1)[:, :, 1].mean(0).tolist()
+
+    consumed_time = time.time() - start_time
+    print(f'total consumed time: {consumed_time} s')
+    print(f'time per sample: {consumed_time / len(test_set)} s')
+
+    # 保存到 CSV
+    df = pd.DataFrame({
+        "seq":  all_seqs,
+        "pred": preds,
+    })
+
+    df.to_csv(f'{weight_dir}/preds_case{f"_{args.use_ft}" if args.use_ft else ""}.csv', index=False)
+
+
+if __name__ == '__main__':
+    main()

infer_case_feature.py

@@ -0,0 +1,223 @@
+import argparse
+import time
+from dataset import PeptidePairPicCaseDataset, encode_sequence
+from network import DMutaPeptideCNN
+from train import move_to_device
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader
+import numpy as np
+from utils import set_seed
+import pandas as pd
+
+parser = argparse.ArgumentParser(description='resnet26')
+# model setting
+parser.add_argument('--model', type=str, default='resnet34',
+                    help='resnet34 resnet50 densenet')
+parser.add_argument('--q-encoder', dest='q_encoder', type=str, default='cnn',
+                    help='lstm mamba mla')
+parser.add_argument('--channels', type=int, default=16)
+parser.add_argument("--side-enc", dest='side_enc', type=str, default=None,
+                    help="use side features")
+parser.add_argument('--fusion', type=str, default='att',
+                    help='mlp att')
+parser.add_argument('--glob-feat', dest='glob_feat', action='store_true', default=False,
+                    help="use global features")
+parser.add_argument('--non-siamese', dest='non_siamese', action='store_true', default=False,
+                    help="use non-siamese architecture")
+
+# task & dataset setting
+parser.add_argument('--task', type=str, default='cls',
+                    help='reg or cls')
+parser.add_argument('--one-way', action='store_true', dest='one_way', default=False,
+                    help='use one-way constructed dataset')
+parser.add_argument('--max-length', dest='max_length', type=int, default=30,
+                    help='Max length for sequence filtering')
+parser.add_argument('--resize', type=int, default=[768], nargs='+',
+                    help='resize the image')
+parser.add_argument('--split', type=int, default=5,
+                    help="Split k fold in cross validation (default: 5)")
+parser.add_argument('--seed', type=int, default=1,
+                    help="Seed for model initialization (default: 1)")
+parser.add_argument('--pcs', action='store_true', default=False,
+                    help='Consider protease cut site')
+parser.add_argument('--mix-pcs', dest='mix_pcs', action='store_true', default=False,
+                    help='Consider protease cut site')
+
+# training setting
+parser.add_argument('--gpu', type=int, default=0,
+                    help='GPU index to use, -1 for CPU (default: 0)')
+parser.add_argument('--batch-size', type=int, dest='batch_size', default=32,
+                    help='input batch size for training (default: 128)')
+parser.add_argument('--epochs', type=int, default=50,
+                    help='number of epochs to train (default: 100)')
+parser.add_argument('--lr', type=float, default=0.001,
+                    help='learning rate (default: 0.001)')
+parser.add_argument('--decay', type=float, default=0.0005,
+                    help='weight decay (default: 0.0005)')
+parser.add_argument('--pretrain', type=str, dest='pretrain', default='',
+                    help='path of the pretrain model')
+parser.add_argument('--metric-avg', type=str, dest='metric_avg', default='macro',
+                    help='metric average type')
+
+parser.add_argument('--loss', type=str, default='ce',
+                    help='loss function')
+parser.add_argument('--dir', action='store_true', default=False,
+                    help='use DIR')
+
+parser.add_argument('--simple', dest='simple', action='store_true', default=False)
+parser.add_argument('--llm-data', dest='llm_data', action='store_true', default=False)
+
+# Case Study Specific
+parser.add_argument('--case', type=str, default='r2',
+                    help='case to infer')
+parser.add_argument('--uda', action='store_true', default=False)
+
+args = parser.parse_args()
+
+if args.llm_data:
+    args.simple = True
+
+if args.simple:
+    args.one_way = True
+
+if args.mix_pcs:
+    args.pcs = 'mix'
+
+if args.gpu != -1:
+    torch.backends.cudnn.benchmark = True
+    torch.set_float32_matmul_precision('high')
+
+
+class FasterModelForCase(DMutaPeptideCNN):
+    def cache_temp_vector(self, seq):
+        if self.side_enc:
+            seq_seq = seq[1]
+            seq = seq[0]
+            if self.side_encoder.__class__.__name__ == 'MambaModel':
+                self.temp_seq_vector = self.norm(self.side_encoder(seq_seq))
+            else:
+                self.temp_seq_vector = self.norm(self.side_encoder(seq_seq)[0][:, -1, :])
+        self.temp_vector = self.norm(self.q_encoder(seq))
+
+    def forward(self, x, labels=None, epoch=0):
+        seq2 = x
+
+        if self.side_enc:
+            seq2_seq = seq2[1]
+            seq2 = seq2[0]
+
+        batch_size = seq2.shape[0]
+
+        fusion = []
+
+        # 获取两个序列的编码结果
+        fusion.append(self.temp_vector.expand(batch_size, -1))
+        fusion.append(self.norm(self.q_encoder_2(seq2)))
+        if self.side_enc:
+            fusion.append(self.temp_seq_vector.expand(batch_size, -1))
+            if self.side_encoder.__class__.__name__ == 'MambaModel':
+                fusion.append(self.norm(self.side_encoder_2(seq2_seq)))
+            else:
+                fusion.append(self.norm(self.side_encoder_2(seq2_seq)[0][:, -1, :]))
+
+        # 根据 fusion_method 决定融合方式
+        if self.fusion_method == 'mlp':
+            # 维持原有行为：拼接两个向量
+            fusion = torch.cat(fusion, dim=-1)
+        elif self.fusion_method == 'diff':
+            if not self.side_enc:
+                fusion = torch.cat([fusion[1] - fusion[0]] + fusion[2:], dim=-1)
+            else:
+                fusion = torch.cat([fusion[1] - fusion[0], fusion[3] - fusion[2]] + fusion[4:], dim=-1)
+        elif self.fusion_method == 'att':
+            # 使用 attention 融合:
+            # 先将两个向量堆叠成“tokens”，形状：(batch, 2, embed_dim)
+            tokens = torch.stack(fusion, dim=1)  # embed_dim 应该为 final_dim//2
+            # 利用 MultiheadAttention 进行自注意力计算
+            # 注意：因为采用 batch_first=True，所以输入形状为 (batch, seq_len, embed_dim)
+            attn_output, _ = self.attn(tokens, tokens, tokens)
+            # 将 attention 输出展平，得到形状 (batch, 2 * embed_dim)，即 (batch, final_dim)
+            fusion = attn_output.reshape(attn_output.size(0), -1)
+        else:
+            raise ValueError("Invalid fusion method: choose either 'mse' or 'att'.")
+        
+        feature = self.fc[:-1](fusion)
+        pred = self.fc[-1](feature)
+
+        return pred, feature
+
+class CustomDataset(PeptidePairPicCaseDataset):
+    def __getitem__(self, idx):
+        variant  = self.variants[idx]
+        seq2, label = variant, variant
+        img2 = self.read_img(variant)
+
+        if self.side_enc:
+            img2 = (img2, encode_sequence(seq2, self.pad_length))
+
+        return img2, label
+
+def load_model(args, weight_path, device, temp_batch):
+    model = FasterModelForCase(q_encoder=args.q_encoder, classes=args.classes, channels=args.channels, dir=args.dir, gf=args.glob_feat, side_enc=args.side_enc, fusion=args.fusion, non_siamese=args.non_siamese).to(device).eval()
+    model.load_state_dict(torch.load(weight_path, map_location=device), strict=False)
+    model.cache_temp_vector(move_to_device(temp_batch, device))
+    model.compile()
+    return model
+
+
+def main():
+    set_seed(args.seed)
+    if args.task == 'reg':
+        args.classes = 1
+    elif args.task == 'cls':
+        args.classes = 2
+    else:
+        raise NotImplementedError("unimplemented task")
+    weight_dir = f'./run-{args.task}/{args.q_encoder}{f"-non-siamese" if args.non_siamese else ""}-{args.fusion}-{args.channels}{f"-{args.side_enc}" if args.side_enc else ""}{"-mixpcs" if args.mix_pcs else ""}{"-pcs" if args.pcs==True else ""}{"-simple" if args.simple else ""}{"-llm" if args.llm_data else ""}{"-" + "x".join(str(n) for n in args.resize) if args.resize else ""}{"-gf" if args.glob_feat else ""}{"-oneway" if args.one_way else ""}-{args.loss + "-dir" if args.dir else args.loss}-{str(args.batch_size)}-{str(args.lr)}-{str(args.epochs)}'
+    if args.uda:
+        weight_dir += f'/uda_{args.case}'
+    device = torch.device("cpu" if args.gpu == -1 or not torch.cuda.is_available() else f"cuda:{args.gpu}")
+    print(weight_dir)
+    print(device)
+    
+    test_set = CustomDataset(case=args.case, pad_length=args.max_length, side_enc=args.side_enc, pcs=True, resize=args.resize, gf=args.glob_feat)
+    test_loader = DataLoader(test_set, batch_size=192, shuffle=False, num_workers=16, pin_memory=True)
+    # test_loader = DataLoader(test_set, batch_size=192, shuffle=False, num_workers=8)
+
+    temp_batch = test_set.template_pic.unsqueeze(0)
+    if args.side_enc:
+        temp_batch = (temp_batch, test_set.template_seq.unsqueeze(0))
+
+    pth_path = f'{weight_dir}/model_uda_teacher.pth' if args.uda else f'{weight_dir}/model_0_test.pth'
+    
+    model = load_model(args, pth_path, device, temp_batch)
+    # models = [load_model(args, f'{weight_dir}/model_{i}{"_ft" if args.use_ft else ""}.pth', device, temp_batch) for i in [0]]
+
+    all_features = {}
+    all_preds = {}
+
+    start_time = time.time()
+
+    with torch.no_grad():
+        for x, gt in test_loader:
+            # x: [B, ...] on CPU pin memory，gt: tuple of B strings
+            x = move_to_device(x, device, non_blocking=True)
+            preds, feats = model(x)
+            if args.task == 'cls':
+                preds = torch.softmax(preds, dim=1)[:, 1]
+            for name, feat, pred in zip(gt, feats, preds):
+                all_features[name] = feat.cpu()
+                all_preds[name] = pred.item()
+
+    consumed_time = time.time() - start_time
+    print(f'total consumed time: {consumed_time} s')
+    print(f'time per sample: {consumed_time / len(test_set)} s')
+
+    torch.save(all_features, f'{weight_dir}/features.pth')
+
+    df = pd.DataFrame(list(all_preds.items()), columns=['seq', 'pred'])
+    df.to_csv(f'{weight_dir}/feature_preds.csv', index=False)
+
+if __name__ == '__main__':
+    main()

infer_case_uda.py

@@ -0,0 +1,247 @@
+import argparse
+import time
+from dataset import PeptidePairPicCaseDataset, encode_sequence
+from network import DMutaPeptideCNN
+from train import move_to_device
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader
+import numpy as np
+from utils import set_seed
+import pandas as pd
+
+parser = argparse.ArgumentParser(description='resnet26')
+# model setting
+parser.add_argument('--model', type=str, default='resnet34',
+                    help='resnet34 resnet50 densenet')
+parser.add_argument('--q-encoder', dest='q_encoder', type=str, default='cnn',
+                    help='lstm mamba mla')
+parser.add_argument('--channels', type=int, default=16)
+parser.add_argument("--side-enc", dest='side_enc', type=str, default=None,
+                    help="use side features")
+parser.add_argument('--fusion', type=str, default='att',
+                    help='mlp att')
+parser.add_argument('--glob-feat', dest='glob_feat', action='store_true', default=False,
+                    help="use global features")
+parser.add_argument('--non-siamese', dest='non_siamese', action='store_true', default=False,
+                    help="use non-siamese architecture")
+
+# task & dataset setting
+parser.add_argument('--task', type=str, default='cls',
+                    help='reg or cls')
+parser.add_argument('--one-way', action='store_true', dest='one_way', default=False,
+                    help='use one-way constructed dataset')
+parser.add_argument('--max-length', dest='max_length', type=int, default=30,
+                    help='Max length for sequence filtering')
+parser.add_argument('--resize', type=int, default=[768], nargs='+',
+                    help='resize the image')
+parser.add_argument('--split', type=int, default=5,
+                    help="Split k fold in cross validation (default: 5)")
+parser.add_argument('--seed', type=int, default=1,
+                    help="Seed for model initialization (default: 1)")
+parser.add_argument('--pcs', action='store_true', default=False,
+                    help='Consider protease cut site')
+parser.add_argument('--mix-pcs', dest='mix_pcs', action='store_true', default=False,
+                    help='Consider protease cut site')
+
+# training setting
+parser.add_argument('--gpu', type=int, default=0,
+                    help='GPU index to use, -1 for CPU (default: 0)')
+parser.add_argument('--batch-size', type=int, dest='batch_size', default=32,
+                    help='input batch size for training (default: 128)')
+parser.add_argument('--epochs', type=int, default=50,
+                    help='number of epochs to train (default: 100)')
+parser.add_argument('--lr', type=float, default=0.001,
+                    help='learning rate (default: 0.001)')
+parser.add_argument('--decay', type=float, default=0.0005,
+                    help='weight decay (default: 0.0005)')
+parser.add_argument('--pretrain', type=str, dest='pretrain', default='',
+                    help='path of the pretrain model')
+parser.add_argument('--metric-avg', type=str, dest='metric_avg', default='macro',
+                    help='metric average type')
+
+parser.add_argument('--loss', type=str, default='ce',
+                    help='loss function')
+parser.add_argument('--dir', action='store_true', default=False,
+                    help='use DIR')
+
+parser.add_argument('--simple', dest='simple', action='store_true', default=False)
+parser.add_argument('--llm-data', dest='llm_data', action='store_true', default=False)
+
+# Case Study Specific
+parser.add_argument('--case', type=str, default='r2',
+                    help='case to infer')
+parser.add_argument('--use-variant', dest='use_variant', type=str, default='')
+
+args = parser.parse_args()
+
+if args.llm_data:
+    args.simple = True
+
+if args.simple:
+    args.one_way = True
+
+if args.mix_pcs:
+    args.pcs = 'mix'
+
+if args.gpu != -1:
+    torch.backends.cudnn.benchmark = True
+    torch.set_float32_matmul_precision('high')
+
+
+class FasterModelForCase(DMutaPeptideCNN):
+    def cache_temp_vector(self, seq):
+        if self.side_enc:
+            seq_seq = seq[1]
+            seq = seq[0]
+            if self.side_encoder.__class__.__name__ == 'MambaModel':
+                self.temp_seq_vector = self.norm(self.side_encoder(seq_seq))
+            else:
+                self.temp_seq_vector = self.norm(self.side_encoder(seq_seq)[0][:, -1, :])
+        self.temp_vector = self.norm(self.q_encoder(seq))
+
+    def forward(self, x, labels=None, epoch=0):
+        seq2 = x
+
+        if self.side_enc:
+            seq2_seq = seq2[1]
+            seq2 = seq2[0]
+
+        batch_size = seq2.shape[0]
+
+        fusion = []
+
+        # 获取两个序列的编码结果
+        fusion.append(self.temp_vector.expand(batch_size, -1))
+        fusion.append(self.norm(self.q_encoder_2(seq2)))
+        if self.side_enc:
+            fusion.append(self.temp_seq_vector.expand(batch_size, -1))
+            if self.side_encoder.__class__.__name__ == 'MambaModel':
+                fusion.append(self.norm(self.side_encoder_2(seq2_seq)))
+            else:
+                fusion.append(self.norm(self.side_encoder_2(seq2_seq)[0][:, -1, :]))
+
+        # 根据 fusion_method 决定融合方式
+        if self.fusion_method == 'mlp':
+            # 维持原有行为：拼接两个向量
+            fusion = torch.cat(fusion, dim=-1)
+        elif self.fusion_method == 'diff':
+            if not self.side_enc:
+                fusion = torch.cat([fusion[1] - fusion[0]] + fusion[2:], dim=-1)
+            else:
+                fusion = torch.cat([fusion[1] - fusion[0], fusion[3] - fusion[2]] + fusion[4:], dim=-1)
+        elif self.fusion_method == 'att':
+            # 使用 attention 融合:
+            # 先将两个向量堆叠成“tokens”，形状：(batch, 2, embed_dim)
+            tokens = torch.stack(fusion, dim=1)  # embed_dim 应该为 final_dim//2
+            # 利用 MultiheadAttention 进行自注意力计算
+            # 注意：因为采用 batch_first=True，所以输入形状为 (batch, seq_len, embed_dim)
+            attn_output, _ = self.attn(tokens, tokens, tokens)
+            # 将 attention 输出展平，得到形状 (batch, 2 * embed_dim)，即 (batch, final_dim)
+            fusion = attn_output.reshape(attn_output.size(0), -1)
+        else:
+            raise ValueError("Invalid fusion method: choose either 'mse' or 'att'.")
+
+        # 如果启用 DIR 模块，保留传入 FDS 前的特征表示
+        if self.DIR:
+            features = fusion
+            fusion = self.FDS.smooth(fusion, labels, epoch)
+        
+        pred = self.fc(fusion)
+
+        if self.DIR:
+            return pred, features
+        else:
+            return pred
+
+
+class CustomDataset(PeptidePairPicCaseDataset):
+    def __getitem__(self, idx):
+        variant  = self.variants[idx]
+        seq2, label = variant, variant
+        img2 = self.read_img(variant)
+
+        if self.side_enc:
+            img2 = (img2, encode_sequence(seq2, self.pad_length))
+
+        return img2, label
+
+
+def load_model(args, weight_path, device, temp_batch):
+    model = FasterModelForCase(q_encoder=args.q_encoder, classes=args.classes, channels=args.channels, dir=args.dir, gf=args.glob_feat, side_enc=args.side_enc, fusion=args.fusion, non_siamese=args.non_siamese).to(device).eval()
+    model.load_state_dict(torch.load(weight_path, map_location=device), strict=False)
+    model.cache_temp_vector(move_to_device(temp_batch, device))
+    model.compile()
+    return model
+
+
+def main():
+    set_seed(args.seed)
+    if args.task == 'reg':
+        args.classes = 1
+    elif args.task == 'cls':
+        args.classes = 2
+    else:
+        raise NotImplementedError("unimplemented task")
+    weight_dir = f'./run-{args.task}/{args.q_encoder}{f"-non-siamese" if args.non_siamese else ""}-{args.fusion}-{args.channels}{f"-{args.side_enc}" if args.side_enc else ""}{"-mixpcs" if args.mix_pcs else ""}{"-pcs" if args.pcs==True else ""}{"-simple" if args.simple else ""}{"-llm" if args.llm_data else ""}{"-" + "x".join(str(n) for n in args.resize) if args.resize else ""}{"-gf" if args.glob_feat else ""}{"-oneway" if args.one_way else ""}-{args.loss + "-dir" if args.dir else args.loss}-{str(args.batch_size)}-{str(args.lr)}-{str(args.epochs)}/uda_{args.case}'
+    device = torch.device("cpu" if args.gpu == -1 or not torch.cuda.is_available() else f"cuda:{args.gpu}")
+    print(weight_dir)
+    print(device)
+    
+    test_set = CustomDataset(case=args.case, pad_length=args.max_length, side_enc=args.side_enc, pcs=True, resize=args.resize, gf=args.glob_feat)
+    test_loader = DataLoader(test_set, batch_size=192, shuffle=False, num_workers=16, pin_memory=True)
+    # test_loader = DataLoader(test_set, batch_size=192, shuffle=False, num_workers=8)
+
+    temp_batch = test_set.template_pic.unsqueeze(0)
+    if args.side_enc:
+        temp_batch = (temp_batch, test_set.template_seq.unsqueeze(0))
+    
+    models = [load_model(args, f'{weight_dir}/model_uda_{role}{f"_{args.use_variant}" if args.use_variant else ""}.pth', device, temp_batch) for role in ('teacher',)]
+    # models = [load_model(args, f'{weight_dir}/model_{i}{"_ft" if args.use_ft else ""}.pth', device, temp_batch) for i in [0]]
+
+    all_seqs = []
+    logits_batches = []  # 存放每个 batch 的 [m,B,2] avg_logits (CPU 上)
+
+    start_time = time.time()
+
+    with torch.no_grad():#, torch.autocast(device_type=device.type):
+        for x, gt in test_loader:
+            # x: [B, ...] on CPU pin memory，gt: tuple of B strings
+            x = move_to_device(x, device, non_blocking=True)
+            # x = move_to_device(x, device)
+
+            # 1) 记录 5 个模型的 logits
+            # logits: [m,B,2]
+            logits = torch.zeros(len(models), len(gt), args.classes, device=device)
+            for i, m in enumerate(models):
+                logits[i] = m(x)
+            # avg_logits = sum_logits.div_(len(models))
+
+            # 3) 立刻搬到 CPU（pin_memory 下可以 non_blocking）
+            logits_batches.append(logits.cpu())
+            all_seqs.extend(gt)
+
+    # 拼接成 [n,2]，n = sum(batch_size)
+    all_logits = torch.cat(logits_batches, dim=1)  # [m,n,2]
+    
+    if args.task == 'reg':
+        preds = all_logits.mean(0).squeeze().tolist()
+    elif args.task == 'cls':
+        # 最后一次性 softmax，取正类概率
+        preds = torch.softmax(all_logits, dim=-1)[:, :, 1].mean(0).tolist()
+
+    consumed_time = time.time() - start_time
+    print(f'total consumed time: {consumed_time} s')
+    print(f'time per sample: {consumed_time / len(test_set)} s')
+
+    # 保存到 CSV
+    df = pd.DataFrame({
+        "seq":  all_seqs,
+        "pred": preds,
+    })
+
+    df.to_csv(f'{weight_dir}/preds_case{f"_{args.use_variant}" if args.use_variant else ""}.csv', index=False)
+
+
+if __name__ == '__main__':
+    main()

infer_case_unoptimized.py

@@ -0,0 +1,164 @@
+import argparse
+import time
+from dataset import PeptidePairPicCaseDataset, encode_sequence
+from network import DMutaPeptideCNN
+from train import move_to_device
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader
+import numpy as np
+from utils import set_seed
+import pandas as pd
+
+parser = argparse.ArgumentParser(description='resnet26')
+# model setting
+parser.add_argument('--model', type=str, default='resnet34',
+                    help='resnet34 resnet50 densenet')
+parser.add_argument('--q-encoder', dest='q_encoder', type=str, default='cnn',
+                    help='lstm mamba mla')
+parser.add_argument('--channels', type=int, default=16)
+parser.add_argument("--side-enc", dest='side_enc', type=str, default='lstm',
+                    help="use side features")
+parser.add_argument('--fusion', type=str, default='att',
+                    help='mlp att')
+parser.add_argument('--glob-feat', dest='glob_feat', action='store_true', default=False,
+                    help="use global features")
+parser.add_argument('--non-siamese', dest='non_siamese', action='store_true', default=False,
+                    help="use non-siamese architecture")
+
+# task & dataset setting
+parser.add_argument('--task', type=str, default='cls',
+                    help='reg or cls')
+parser.add_argument('--one-way', action='store_true', dest='one_way', default=False,
+                    help='use one-way constructed dataset')
+parser.add_argument('--max-length', dest='max_length', type=int, default=30,
+                    help='Max length for sequence filtering')
+parser.add_argument('--resize', type=int, default=[768], nargs='+',
+                    help='resize the image')
+parser.add_argument('--split', type=int, default=5,
+                    help="Split k fold in cross validation (default: 5)")
+parser.add_argument('--seed', type=int, default=1,
+                    help="Seed for model initialization (default: 1)")
+parser.add_argument('--pcs', action='store_true', default=False,
+                    help='Consider protease cut site')
+parser.add_argument('--mix-pcs', dest='mix_pcs', action='store_true', default=False,
+                    help='Consider protease cut site')
+
+# training setting
+parser.add_argument('--gpu', type=int, default=0,
+                    help='GPU index to use, -1 for CPU (default: 0)')
+parser.add_argument('--batch-size', type=int, dest='batch_size', default=32,
+                    help='input batch size for training (default: 128)')
+parser.add_argument('--epochs', type=int, default=50,
+                    help='number of epochs to train (default: 100)')
+parser.add_argument('--lr', type=float, default=0.001,
+                    help='learning rate (default: 0.001)')
+parser.add_argument('--decay', type=float, default=0.0005,
+                    help='weight decay (default: 0.0005)')
+parser.add_argument('--pretrain', type=str, dest='pretrain', default='',
+                    help='path of the pretrain model')
+parser.add_argument('--metric-avg', type=str, dest='metric_avg', default='macro',
+                    help='metric average type')
+
+parser.add_argument('--loss', type=str, default='ce',
+                    help='loss function')
+parser.add_argument('--dir', action='store_true', default=False,
+                    help='use DIR')
+
+parser.add_argument('--simple', dest='simple', action='store_true', default=False)
+parser.add_argument('--llm-data', dest='llm_data', action='store_true', default=False)
+
+# Case Study Specific
+parser.add_argument('--case', type=str, default='r2',
+                    help='case to infer')
+parser.add_argument('--use-ft', dest='use_ft', action='store_true', default=False)
+
+args = parser.parse_args()
+
+if args.llm_data:
+    args.simple = True
+
+if args.simple:
+    args.one_way = True
+
+if args.mix_pcs:
+    args.pcs = 'mix'
+
+if args.gpu != -1:
+    torch.backends.cudnn.benchmark = True
+    torch.set_float32_matmul_precision('high')
+
+
+def load_model(args, weight_path, device):
+    model = DMutaPeptideCNN(q_encoder=args.q_encoder, classes=args.classes, channels=args.channels, dir=args.dir, gf=args.glob_feat, side_enc=args.side_enc, fusion=args.fusion, non_siamese=args.non_siamese).to(device).eval()
+    model.load_state_dict(torch.load(weight_path, map_location=device), strict=False)
+    model.compile()
+    return model
+
+
+def main():
+    set_seed(args.seed)
+    if args.task == 'reg':
+        args.classes = 1
+    elif args.task == 'cls':
+        args.classes = 2
+    else:
+        raise NotImplementedError("unimplemented task")
+    weight_dir = f'./run-{args.task}/{args.q_encoder}{f"-non-siamese" if args.non_siamese else ""}-{args.fusion}-{args.channels}{f"-{args.side_enc}" if args.side_enc else ""}{"-mixpcs" if args.mix_pcs else ""}{"-pcs" if args.pcs==True else ""}{"-simple" if args.simple else ""}{"-llm" if args.llm_data else ""}{"-" + "x".join(str(n) for n in args.resize) if args.resize else ""}{"-gf" if args.glob_feat else ""}{"-oneway" if args.one_way else ""}-{args.loss + "-dir" if args.dir else args.loss}-{str(args.batch_size)}-{str(args.lr)}-{str(args.epochs)}'
+    device = torch.device("cpu" if args.gpu == -1 or not torch.cuda.is_available() else f"cuda:{args.gpu}")
+    print(weight_dir)
+    print(device)
+    
+    test_set = PeptidePairPicCaseDataset(case=args.case, pad_length=args.max_length, side_enc=args.side_enc, pcs=True, resize=args.resize, gf=args.glob_feat)
+    test_loader = DataLoader(test_set, batch_size=128, shuffle=False, num_workers=16, pin_memory=True)
+    # test_loader = DataLoader(test_set, batch_size=192, shuffle=False, num_workers=8)
+    
+    models = [load_model(args, f'{weight_dir}/model_{i}{"_ft" if args.use_ft else ""}.pth', device) for i in range(args.split)]
+
+    all_seqs = []
+    logits_batches = []  # 存放每个 batch 的 [m,B,2] avg_logits (CPU 上)
+
+    start_time = time.time()
+
+    with torch.no_grad():
+        for x, gt in test_loader:
+            # x: [B, ...] on CPU pin memory，gt: tuple of B strings
+            x = move_to_device(x, device, non_blocking=True)
+            # x = move_to_device(x, device)
+
+            # 1) 记录 5 个模型的 logits
+            # logits: [m,B,2]
+            logits = torch.zeros(len(models), len(gt), args.classes, device=device)
+            for i, m in enumerate(models):
+                logits[i] = m(x)
+
+            # 3) 立刻搬到 CPU（pin_memory 下可以 non_blocking）
+            logits_batches.append(logits.cpu())
+            all_seqs.extend(gt)
+
+    # 拼接成 [n,2]，n = sum(batch_size)
+    all_logits = torch.cat(logits_batches, dim=1)  # [m,n,2]
+    
+    if args.task == 'reg':
+        preds = all_logits.mean(0).squeeze().tolist()
+    elif args.task == 'cls':
+        # 最后一次性 softmax，取正类概率
+        preds = torch.softmax(all_logits, dim=-1)[:, :, 1].mean(0).tolist()
+
+    consumed_time = time.time() - start_time
+    print(f'total consumed time: {consumed_time} s')
+    print(f'time per sample: {consumed_time / len(test_set)} s')
+
+    # 保存到 CSV
+    df = pd.DataFrame({
+        "seq":  all_seqs,
+        "pred": preds,
+    })
+
+    df.to_csv(f'{weight_dir}/preds_case.csv', index=False)
+
+
+
+
+if __name__ == '__main__':
+    main()

infer_cf.py

@@ -0,0 +1,187 @@
+import argparse
+from dataset import PeptidePairDataset, PeptidePairPicDataset
+from network import DMutaPeptide, DMutaPeptideCNN
+from train import move_to_device
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader
+import numpy as np
+from utils import set_seed
+import pandas as pd
+from torchmetrics import MeanAbsoluteError, RelativeSquaredError, PearsonCorrCoef, KendallRankCorrCoef, F1Score, Accuracy, AveragePrecision, AUROC
+
+parser = argparse.ArgumentParser(description='resnet26')
+# model setting
+parser.add_argument('--model', type=str, default='resnet34',
+                    help='resnet34 resnet50 densenet')
+parser.add_argument('--q-encoder', dest='q_encoder', type=str, default='cnn',
+                    help='lstm mamba mla')
+parser.add_argument('--channels', type=int, default=16)
+parser.add_argument("--side-enc", dest='side_enc', type=str, default=None,
+                    help="use side features")
+parser.add_argument('--fusion', type=str, default='diff',
+                    help='mlp att')
+parser.add_argument('--glob-feat', dest='glob_feat', action='store_true', default=False,
+                    help="use global features")
+parser.add_argument('--non-siamese', dest='non_siamese', action='store_true', default=False,
+                    help="use non-siamese architecture")
+
+# task & dataset setting
+parser.add_argument('--task', type=str, default='cls',
+                    help='reg or cls')
+parser.add_argument('--pdb-src', type=str, dest='pdb_src', default='af',
+                    help='af or hf')
+parser.add_argument('--data-ver', type=str, dest='data_ver', default='250228',
+                    help='data version')
+parser.add_argument('--one-way', action='store_true', dest='one_way', default=False,
+                    help='use one-way constructed dataset')
+parser.add_argument('--max-length', dest='max_length', type=int, default=30,
+                    help='Max length for sequence filtering')
+parser.add_argument('--resize', type=int, default=[768], nargs='+',
+                    help='resize the image')
+parser.add_argument('--split', type=int, default=5,
+                    help="Split k fold in cross validation (default: 5)")
+parser.add_argument('--seed', type=int, default=1,
+                    help="Seed (default: 1)")
+parser.add_argument('--pcs', action='store_true', default=False,
+                    help='Consider protease cut site')
+parser.add_argument('--mix-pcs', dest='mix_pcs', action='store_true', default=False,
+                    help='Consider protease cut site')
+
+# training setting
+parser.add_argument('--gpu', type=int, default=0,
+                    help='GPU index to use, -1 for CPU (default: 0)')
+parser.add_argument('--batch-size', type=int, dest='batch_size', default=32,
+                    help='input batch size for training (default: 128)')
+parser.add_argument('--epochs', type=int, default=50,
+                    help='number of epochs to train (default: 100)')
+parser.add_argument('--lr', type=float, default=0.001,
+                    help='learning rate (default: 0.001)')
+parser.add_argument('--decay', type=float, default=0.0005,
+                    help='weight decay (default: 0.0005)')
+parser.add_argument('--warm-steps', type=int, dest='warm_steps', default=0,
+                    help='number of warm start steps for learning rate (default: 10)')
+parser.add_argument('--patience', type=int, default=10,
+                    help='patience for early stopping (default: 10)')
+parser.add_argument('--pretrain', type=str, dest='pretrain', default='',
+                    help='path of the pretrain model')  # /home/duadua/Desktop/fetal/3dpretrain/runs/e50.pth
+parser.add_argument('--metric-avg', type=str, dest='metric_avg', default='macro',
+                    help='metric average type')
+
+parser.add_argument('--loss', type=str, default='ce',
+                    help='loss function')
+parser.add_argument('--dir', action='store_true', default=False,
+                    help='use DIR')
+
+parser.add_argument('--simple', dest='simple', action='store_true', default=False)
+parser.add_argument('--llm-data', dest='llm_data', action='store_true', default=False)
+parser.add_argument('--uda', type=str, default=None)
+
+args = parser.parse_args()
+
+if args.llm_data:
+    args.simple = True
+
+if args.simple:
+    args.one_way = True
+
+if args.mix_pcs:
+    args.pcs = 'mix'
+
+if args.q_encoder in ['cnn', 'rn18']:
+    weight_dir = f'./run-{args.task}/{f"non-siamese-" if args.non_siamese else ""}{args.q_encoder}-{args.fusion}-{args.channels}{f"-{args.side_enc}" if args.side_enc else ""}{"-mixpcs" if args.mix_pcs else ""}{"-pcs" if args.pcs==True else ""}{"-simple" if args.simple else ""}{"-llm" if args.llm_data else ""}{"-" + "x".join(str(n) for n in args.resize) if args.resize else ""}{"-gf" if args.glob_feat else ""}{"-oneway" if args.one_way else ""}-{args.loss + "-dir" if args.dir else args.loss}-{str(args.batch_size)}-{str(args.lr)}-{str(args.epochs)}'
+else:
+    weight_dir = f'./run-{args.task}/{f"non-siamese-" if args.non_siamese else ""}{args.q_encoder}-{args.fusion}-{args.channels}{"-simple" if args.simple else ""}{"-llm" if args.llm_data else ""}{"-gf" if args.glob_feat else ""}{"-oneway" if args.one_way else ""}-{args.loss + "-dir" if args.dir else args.loss}-{str(args.batch_size)}-{str(args.lr)}-{str(args.epochs)}'
+
+if args.uda:
+    weight_dir += f'/uda_{args.uda}'
+
+print(weight_dir)
+
+def metrics(preds, gt, task):
+    avg = 'marco'
+    device = preds.device
+    if task == 'cls':
+        metric_1 = AveragePrecision(average=avg, task='binary').to(device)
+        metric_2 = AUROC(average=avg, task='binary').to(device)
+        metric_3 = F1Score(average=avg, task='binary').to(device)
+        metric_4 = Accuracy(average=avg, task='binary').to(device)
+        all_metrics = [metric_1(preds, gt).item(), 
+                metric_2(preds, gt).item(), 
+                metric_3(preds, gt).item(), 
+                metric_4(preds, gt).item()]
+    
+    elif task == 'reg':
+        metric_1 = MeanAbsoluteError().to(device)
+        metric_2 = RelativeSquaredError(num_outputs=1).to(device)
+        metric_3 = PearsonCorrCoef(num_outputs=1).to(device)
+        metric_4 = KendallRankCorrCoef(num_outputs=1).to(device)
+        all_metrics = [metric_1(preds, gt).item(), 
+                metric_2(preds, gt).item(), 
+                metric_3(preds.squeeze(), gt.squeeze()).mean().item(), 
+                metric_4(preds.squeeze(), gt.squeeze()).mean().item()]
+
+    return [f'{i * 100:.2f}' for i in all_metrics]
+    
+
+def main(dataset):
+    set_seed(args.seed)
+    if args.task == 'reg':
+        args.classes = 1
+    elif args.task == 'cls':
+        args.classes = 2
+    else:
+        raise NotImplementedError("unimplemented task")
+
+    device = torch.device("cpu" if args.gpu == -1 or not torch.cuda.is_available() else f"cuda:{args.gpu}")
+    
+    if args.q_encoder in ['cnn', 'rn18']:
+        model = DMutaPeptideCNN(q_encoder=args.q_encoder, classes=args.classes, channels=args.channels, dir=args.dir, gf=args.glob_feat, side_enc=args.side_enc, fusion=args.fusion, non_siamese=args.non_siamese).to(device).eval()
+        test_set = PeptidePairPicDataset(mode=dataset, pad_length=args.max_length, task=args.task, gf=args.glob_feat, side_enc=args.side_enc, pcs=args.pcs, resize=args.resize)
+    else:
+        model = DMutaPeptide(q_encoder=args.q_encoder, classes=args.classes, channels=args.channels, dir=args.dir, gf=args.glob_feat, fusion=args.fusion, non_siamese=args.non_siamese).to(device).eval()
+        test_set = PeptidePairDataset(mode=dataset, pad_length=args.max_length, task=args.task, gf=args.glob_feat)
+    
+    test_loader = DataLoader(test_set, batch_size=args.batch_size, shuffle=False)
+
+    raw_preds = []
+    ckpt_names = ['model_uda_teacher'] if args.uda else [f'model_{i}_test' for i in range(5)]
+    for i in ckpt_names:
+        model.load_state_dict(torch.load(f'{weight_dir}/{i}.pth', map_location=device))
+        preds = []
+        gt_list_valid = []
+        with torch.no_grad():
+            for data in test_loader:
+                x, gt = data
+                gt_list_valid.append(gt.to(device))
+                out = model(move_to_device(x, device))
+                if args.dir:
+                    out, _ = out
+                preds.append(out)
+        r_pred = torch.cat(preds, dim=0)
+        if args.task == 'reg':
+            preds = r_pred.cpu().numpy()
+        elif args.task == 'cls':
+            preds = torch.softmax(r_pred, dim=-1)[:, 1].cpu().numpy()
+        gt_tensor = torch.cat(gt_list_valid, dim=0)
+        gt_list_valid = gt_tensor.cpu().numpy()
+        raw_preds.append(r_pred)
+    if args.task == 'cls':
+        preds_tensor = torch.softmax(torch.stack(raw_preds, 0), dim=-1)[:, :, 1]
+    elif args.task == 'reg':
+        preds_tensor = torch.stack(raw_preds, 0)
+
+    return [metrics(preds_tensor[i], gt_tensor, args.task) for i in range(len(ckpt_names))]
+
+
+if __name__ == '__main__':
+    if args.task == 'cls':
+        # df = pd.DataFrame(columns=['dataset', 'AUPRC', 'AUROC', 'F1', 'ACC'])
+        print(','.join(['AUPRC', 'AUROC', 'F1', 'ACC']))
+    elif args.task == 'reg':
+        # df = pd.DataFrame(columns=['dataset', 'MAE', 'RSE', 'PCC', 'KCC'])
+        print(','.join(['MAE', 'RSE', 'PCC', 'KCC']))
+
+    results = main('r2_case')
+    for result in results:
+        print(','.join(result))

inferthro.sh

@@ -0,0 +1,13 @@
+# !/bin/bash
+python infer.py --task cls --loss ce --q-encoder lstm --channels 256 --fusion diff
+python infer.py --task cls --loss ce --q-encoder mamba --channels 256 --fusion diff
+python infer.py --task cls --loss ce --q-encoder mha --channels 256 --fusion diff
+python infer.py --task cls --loss ce --q-encoder gru --channels 256 --fusion diff
+python infer.py --task cls --loss ce --q-encoder rn18 --channels 16 --fusion diff --pcs --side-enc mamba
+python infer.py --task cls --loss ce --q-encoder rn18 --channels 16 --fusion diff --pcs --side-enc mamba --uda r2
+python infer.py --task reg --loss mse --q-encoder lstm --channels 256 --fusion diff
+python infer.py --task reg --loss mse --q-encoder mamba --channels 256 --fusion diff
+python infer.py --task reg --loss mse --q-encoder mha --channels 256 --fusion diff
+python infer.py --task reg --loss mse --q-encoder gru --channels 256 --fusion diff
+python infer.py --task reg --loss mse --q-encoder rn18 --channels 16 --fusion diff --pcs --side-enc mamba
+python infer.py --task reg --loss mse --q-encoder rn18 --channels 16 --fusion diff --pcs --side-enc mamba --uda r2

loss.py

@@ -0,0 +1,164 @@
+import torch
+from torch import nn
+from torch.nn.modules.loss import _Loss
+import torch.nn.functional as F
+from math import cos, pi, sin
+import math
+import numpy as np
+from scipy.special import lambertw
+
+
+
+def mixup_criterion(criterion, pred, y_a, y_b, lam, pow=2):
+    y = lam ** pow * y_a + (1 - lam) ** pow * y_b
+    return criterion(pred, y)
+
+
+def mixup_data(v, q, a):
+    '''Returns mixed inputs, pairs of targets, and lambda without organ constraint'''
+    lam = np.random.beta(1, 1)
+
+    batch_size = v.shape[0]
+    index = torch.randperm(batch_size)
+
+    mixed_v = lam * v + (1 - lam) * v[index, :]
+    mixed_q = lam * q + (1 - lam) * q[index, :]
+
+    a_1, a_2 = a, a[index]
+    return mixed_v, mixed_q, a_1, a_2, lam
+
+
+def linear(epoch, nepoch):
+    return 1 - epoch / nepoch
+
+
+def convex(epoch, nepoch):
+    return epoch / (2 - nepoch)
+
+
+def concave(epoch, nepoch):
+    return 1 - sin((epoch / nepoch) * (pi / 2))
+
+
+def composite(epoch, nepoch):
+    return 0.5 * cos((epoch / nepoch) * pi) + 0.5
+
+
+class LogCoshLoss(nn.Module):
+    def __init__(self):
+        super().__init__()
+
+    def forward(self, y_t, y_prime_t):
+        ey_t = y_t - y_prime_t
+        return torch.mean(torch.log(torch.cosh(ey_t + 1e-12)))+F.mse_loss(y_t, y_prime_t)
+    
+
+class WeightedMSELoss(nn.Module):
+    def __init__(self):
+        super().__init__()
+
+    def forward(self, y, y_t, weights=None):
+        loss = (y - y_t) ** 2
+        if weights is not None:
+            loss *= weights.expand_as(loss)
+        return torch.mean(loss)
+
+
+class MLCE(nn.Module):
+    def __init__(self):
+        super(MLCE, self).__init__()
+
+    def _mlcce(self, y_pred, y_true):
+        y_pred = (1 - 2 * y_true) * y_pred
+        y_pred_neg = y_pred - y_true * 1e12
+        y_pred_pos = y_pred - (1 - y_true) * 1e12
+        zeros = torch.zeros_like(y_pred[..., :1])
+        y_pred_neg = torch.cat([y_pred_neg, zeros], dim=-1)
+        y_pred_pos = torch.cat([y_pred_pos, zeros], dim=-1)
+        neg_loss = torch.logsumexp(y_pred_neg, dim=-1)
+        pos_loss = torch.logsumexp(y_pred_pos, dim=-1)
+        loss = torch.mean(neg_loss + pos_loss)
+        return loss
+
+    def __call__(self, y_pred, y_true):
+        return self._mlcce(y_pred, y_true)
+
+
+class SuperLoss(nn.Module):
+    def __init__(self, C=10, lam=1, batch_size=256):
+        super(SuperLoss, self).__init__()
+        self.tau = math.log(C)
+        self.lam = lam  # set to 1 for CIFAR10 and 0.25 for CIFAR100
+        self.batch_size = batch_size
+
+    def forward(self, logits, targets):
+        l_i = F.mse_loss(logits, targets, reduction='none').detach()
+        sigma = self.sigma(l_i)
+        loss = (F.mse_loss(logits, targets, reduction='none') - self.tau) * sigma + self.lam * (
+                torch.log(sigma) ** 2)
+        loss = loss.sum() / self.batch_size
+        return loss
+
+    def sigma(self, l_i):
+        x = torch.ones_like(l_i) * (-2 / math.exp(1.))
+        y = 0.5 * torch.max(x, (l_i - self.tau) / self.lam)
+        y = y.cpu().numpy()
+        sigma = np.exp(-lambertw(y))
+        sigma = sigma.real.astype(np.float32)
+        sigma = torch.from_numpy(sigma).to(l_i.device)
+        return sigma
+
+
+def unbiased_curriculum_loss(out, data, args, epoch, epochs, scheduler='linear'):
+    losses = []
+    scheduler = linear if scheduler == 'linear' else concave
+
+    # calculate difficulty measurement function
+    adjusted_losses = []
+    for idx in range(out.shape[0]):
+        ground_truth = max(1, abs(data[idx].item()))
+        loss = F.mse_loss(out[idx], data[idx])
+        losses.append(loss)
+        adjusted_losses.append(loss.item() / ground_truth)
+
+    mean_loss, std_loss = np.mean(adjusted_losses), np.std(adjusted_losses)
+
+    # re-weight losses
+    total_loss = 0
+    for i, loss in enumerate(losses):
+        if adjusted_losses[i] > mean_loss + 1 * std_loss:
+            schedule_factor = scheduler(epoch, args.epochs)
+            total_loss += schedule_factor * loss
+        else:
+            total_loss += loss
+
+    return total_loss
+
+
+class BMCLoss(_Loss):
+    def __init__(self, init_noise_sigma=1.0):
+        super(BMCLoss, self).__init__()
+        self.noise_sigma = torch.nn.Parameter(torch.tensor(init_noise_sigma))
+
+    def bmc_loss(self, pred, target, noise_var):
+        """Compute the Balanced MSE Loss (BMC) between `pred` and the ground truth `targets`.
+        Args:
+        pred: A float tensor of size [batch, 1].
+        target: A float tensor of size [batch, 1].
+        noise_var: A float number or tensor.
+        Returns:
+        loss: A float tensor. Balanced MSE Loss.
+        """
+        if len(pred.shape) == 1:
+            pred = pred.unsqueeze(1)
+        if len(target.shape) == 1:
+            target = target.unsqueeze(1)
+        logits = - (pred - target.T).pow(2) / (2 * noise_var)   # logit size: [batch, batch]
+        loss = F.cross_entropy(logits, torch.arange(pred.shape[0], device=pred.device))     # contrastive-like loss
+        loss = loss * (2 * noise_var).detach()  # optional: restore the loss scale, 'detach' when noise is learnable 
+
+        return loss
+
+    def forward(self, pred, target):
+        noise_var = self.noise_sigma ** 2
+        return self.bmc_loss(pred, target, noise_var)

main.py

@@ -0,0 +1,245 @@
+import argparse
+import json
+import logging
+import os
+import time
+
+from dataset import PeptidePairDataset, PeptidePairPicDataset
+from network import DMutaPeptide, DMutaPeptideCNN
+from sklearn.model_selection import KFold
+from train import train, train_cls
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader, Subset
+import numpy as np
+from loss import MLCE, SuperLoss, LogCoshLoss, BMCLoss
+from utils import set_seed
+
+
+parser = argparse.ArgumentParser(description='resnet26')
+# model setting
+parser.add_argument('--model', type=str, default='resnet34',
+                    help='resnet34 resnet50 densenet')
+parser.add_argument('--q-encoder', dest='q_encoder', type=str, default='lstm',
+                    help='lstm mamba mla')
+parser.add_argument("--side-enc", dest='side_enc', type=str, default=None,
+                    help="use side features")
+parser.add_argument('--channels', type=int, default=256)
+parser.add_argument('--fusion', type=str, default='att',
+                    help='mlp att diff')
+parser.add_argument('--glob-feat', dest='glob_feat', action='store_true', default=False,
+                    help="use global features")
+parser.add_argument('--non-siamese', dest='non_siamese', action='store_true', default=False,
+                    help="use non-siamese architecture")
+
+# task & dataset setting
+parser.add_argument('--task', type=str, default='reg',
+                    help='reg or cls')
+parser.add_argument('--one-way', action='store_true', dest='one_way', default=False,
+                    help='use one-way constructed dataset')
+parser.add_argument('--max-length', dest='max_length', type=int, default=30,
+                    help='Max length for sequence filtering')
+parser.add_argument('--split', type=int, default=5,
+                    help="Split k fold in cross validation (default: 5)")
+parser.add_argument('--seed', type=int, default=42,
+                    help="Seed (default: 1)")
+parser.add_argument('--pcs', action='store_true', default=False,
+                    help='Consider protease cleavage site')
+parser.add_argument('--mix-pcs', dest='mix_pcs', action='store_true', default=False,
+                    help='Consider protease cleavage site')
+parser.add_argument('--resize', type=int, default=[768], nargs='+',
+                    help='resize the image')
+# parser.add_argument('--llm-data', action='store_true', default=False,
+#                     help='Use LLM augmentation data')
+
+# training setting
+parser.add_argument('--gpu', type=int, default=0,
+                    help='GPU index to use, -1 for CPU (default: 0)')
+parser.add_argument('--batch-size', type=int, dest='batch_size', default=32,
+                    help='input batch size for training (default: 128)')
+parser.add_argument('--epochs', type=int, default=50,
+                    help='number of epochs to train (default: 100)')
+parser.add_argument('--lr', type=float, default=0.001,
+                    help='learning rate (default: 0.001)')
+parser.add_argument('--decay', type=float, default=0.0005,
+                    help='weight decay (default: 0.0005)')
+parser.add_argument('--pretrain', type=str, dest='pretrain', default='',
+                    help='path of the pretrain model')
+parser.add_argument('--metric-avg', type=str, dest='metric_avg', default='macro',
+                    help='metric average type')
+
+parser.add_argument('--loss', type=str, default='mse',
+                    help='loss function')
+parser.add_argument('--dir', action='store_true', default=False,
+                    help='use DIR')
+
+args = parser.parse_args()
+
+if args.mix_pcs:
+    args.pcs = 'mix'
+
+
+def main():
+    set_seed(args.seed)
+    if args.task == 'reg':
+        args.classes = 1
+        trainer = train
+        if args.loss == "mse" or args.loss in ['ce']:
+            args.loss = 'mse'
+            criterion = nn.MSELoss()
+        elif args.loss == "smoothl1":
+            criterion = nn.SmoothL1Loss()
+        elif args.loss == "super":
+            criterion = SuperLoss()
+        elif args.loss in ["bmc", "bmc_ln"]:
+            criterion = BMCLoss()
+        else:
+            raise NotImplementedError("unimplemented regression task loss function")
+    elif args.task == 'cls':
+        trainer = train_cls
+        args.classes = 2
+        if args.loss == 'ce' or args.loss in ['mse', 'smoothl1', 'super']:
+            args.loss = 'ce'
+            criterion = nn.CrossEntropyLoss()
+        else:
+            raise NotImplementedError("unimplemented classification task loss function")
+    else:
+        raise NotImplementedError("unimplemented task")
+    
+    if args.q_encoder in ['cnn', 'rn18']:
+        weight_dir = f'./run-{args.task}/{"non-siamese-" if args.non_siamese else ""}{args.q_encoder}-{args.fusion}-{args.channels}{f"-{args.side_enc}" if args.side_enc else ""}{"-mixpcs" if args.mix_pcs else ""}{"-pcs" if args.pcs==True else ""}{"-" + "x".join(str(n) for n in args.resize) if args.resize else ""}{"-gf" if args.glob_feat else ""}{"-oneway" if args.one_way else ""}-{args.loss + "-dir" if args.dir else args.loss}-{str(args.batch_size)}-{str(args.lr)}-{str(args.epochs)}'
+    else:
+        weight_dir = f'./run-{args.task}/{"non-siamese-" if args.non_siamese else ""}{args.q_encoder}-{args.fusion}-{args.channels}{"-gf" if args.glob_feat else ""}{"-oneway" if args.one_way else ""}-{args.loss + "-dir" if args.dir else args.loss}-{str(args.batch_size)}-{str(args.lr)}-{str(args.epochs)}'
+
+    if not os.path.exists(weight_dir):
+        os.makedirs(weight_dir)
+    
+    logging.basicConfig(handlers=[
+        logging.FileHandler(filename=os.path.join(weight_dir, "training.log"), encoding='utf-8', mode='w+'),
+        logging.StreamHandler()],
+        format="%(asctime)s: %(message)s", datefmt="%F %T", level=logging.INFO)
+
+    logging.info(f'saving_dir: {weight_dir}')
+    
+    with open(os.path.join(weight_dir, "config.json"), "w") as f:
+        f.write(json.dumps(vars(args)))
+
+    device = torch.device("cpu" if args.gpu == -1 or not torch.cuda.is_available() else f"cuda:{args.gpu}")
+
+    if args.q_encoder in ['cnn', 'rn18']:
+        logging.info('Loading Training Dataset')
+        all_set = PeptidePairPicDataset(mode='train', pad_length=args.max_length, task=args.task, one_way=args.one_way, gf=args.glob_feat, side_enc=args.side_enc, pcs=args.pcs, resize=args.resize)
+        logging.info('Loading Test Dataset')
+        test_set = PeptidePairPicDataset(mode='test', pad_length=args.max_length, task=args.task, gf=args.glob_feat, side_enc=args.side_enc, pcs=args.pcs, resize=args.resize)
+    else:
+        logging.info('Loading Train Dataset')
+        all_set = PeptidePairDataset(mode='train', pad_length=args.max_length, task=args.task, one_way=args.one_way, gf=args.glob_feat)
+        logging.info('Loading Test Dataset')
+        test_set = PeptidePairDataset(mode='test', pad_length=args.max_length, task=args.task, gf=args.glob_feat)
+
+    test_loader = DataLoader(test_set, batch_size=args.batch_size, shuffle=False, num_workers=8, pin_memory=True)
+
+    best_perform_list = [[] for i in range(5)]
+    test_perform_list = [[] for i in range(5)]
+
+    kf = KFold(n_splits=5, shuffle=True, random_state=42)
+
+    for fold, (train_idx, val_idx) in enumerate(kf.split(all_set)):
+        train_set= Subset(all_set, train_idx)
+        valid_set = Subset(all_set, val_idx)
+
+        train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True, drop_last=True, num_workers=8, pin_memory=True)
+        valid_loader = DataLoader(valid_set, batch_size=args.batch_size, shuffle=False, num_workers=8, pin_memory=True)
+
+        if args.q_encoder in ['cnn', 'rn18']:
+            model = DMutaPeptideCNN(q_encoder=args.q_encoder, classes=args.classes, channels=args.channels, dir=args.dir, gf=args.glob_feat, side_enc=args.side_enc, fusion=args.fusion, non_siamese=args.non_siamese)
+        else:
+            model = DMutaPeptide(q_encoder=args.q_encoder, classes=args.classes, channels=args.channels, dir=args.dir, gf=args.glob_feat, fusion=args.fusion, non_siamese=args.non_siamese)
+        if len(args.pretrain) != 0: #TODO: load pretrain
+            pass
+        model.to(device)
+        # model.compile()
+        
+        optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=args.decay)
+        # optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.decay)
+
+        # scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[10], gamma=0.5)
+        if args.q_encoder == 'cnn':
+            scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.5)
+        else:
+            scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.5)
+
+        if args.loss == 'bmc_ln':
+            optimizer.add_param_group({'params': criterion.noise_sigma, 'lr': args.lr, 'name': 'noise_sigma'})
+        weights_path = f"{weight_dir}/model_{fold}.pth"
+        # early_stopping = EarlyStopping(patience=args.patience, path=weights_path)
+        logging.info(f'Running Cross Validation {fold}')
+        logging.info(f'Fold {fold}  Train set:{len(train_set)}, Valid set:{len(valid_set)}, Test set: {len(test_set)}')
+        best_metric = -float('inf')
+        best_test = -float('inf')
+        start_time = time.time()
+        if args.task == 'reg':
+            for epoch in range(1, args.epochs + 1):
+                train_loss, mae, rse, pcc, kcc = trainer(args, epoch, model, train_loader, valid_loader, device, criterion, optimizer)
+                logging.info(f'Epoch: {epoch:03d} Train Loss: {train_loss:.3f}, mae: {mae:.3f}, rse: {rse:.3f}, pcc: {pcc:.3f}, kcc: {kcc:.3f}')
+                scheduler.step()
+                avg_metric = (pcc + kcc) - (mae + rse)
+                if avg_metric > best_metric:
+                    logging.info(f'Epoch: {epoch:03d} New best VALIDATION metrics')
+                    torch.save(model.state_dict(), weights_path)
+                    best_metric = avg_metric
+                    best_perform_list[fold] = np.asarray([mae, rse, pcc, kcc])
+                
+                _, test_mae, test_rse, test_pcc, test_kcc = trainer(args, epoch, model, None, test_loader, device, None, None)
+                logging.info(f'Epoch: {epoch:03d} Test results, ap: mae: {test_mae:.3f}, rse: {test_rse:.3f}, pcc: {test_pcc:.3f}, kcc: {test_kcc:.3f}')
+                test_metric = (test_pcc + test_kcc) - (test_mae + test_rse)
+                if test_metric > best_test and epoch > 10:
+                    logging.info(f'Epoch: {epoch:03d} New best TEST metrics')
+                    best_test = test_metric
+                    test_perform_list[fold] = np.asarray([test_mae, test_rse, test_pcc, test_kcc])
+                    torch.save(model.state_dict(), weights_path.replace('.pth', '_test.pth'))
+        
+        elif args.task == 'cls':
+            for epoch in range(1, args.epochs + 1):
+                train_loss, ap, auc, f1, acc = trainer(args, epoch, model, train_loader, valid_loader, device, criterion, optimizer)
+                logging.info(f'Epoch: {epoch:03d} Train Loss: {train_loss:.3f}, ap: {ap:.3f}, auc: {auc:.3f}, f1: {f1:.3f}, acc: {acc:.3f}')
+                scheduler.step()
+                avg_metric = ap + auc #+ f1 + acc
+                if avg_metric > best_metric:
+                    logging.info(f'Epoch: {epoch:03d} New best VALIDATION metrics')
+                    torch.save(model.state_dict(), weights_path)
+                    best_metric = avg_metric
+                    best_perform_list[fold] = np.asarray([ap, auc, f1, acc])
+                
+                _, test_ap, test_auc, test_f1, test_acc = trainer(args, epoch, model, None, test_loader, device, None, None)
+                logging.info(f'Epoch: {epoch:03d} Test results, ap: {test_ap:.3f}, auc: {test_auc:.3f}, f1: {test_f1:.3f}, acc: {test_acc:.3f}')
+                test_metric = test_ap + test_auc #+ test_f1 + test_acc
+                if test_metric > best_test and epoch > 10:
+                    logging.info(f'Epoch: {epoch:03d} New best TEST metrics')
+                    best_test = test_metric
+                    test_perform_list[fold] = np.asarray([test_ap, test_auc, test_f1, test_acc])
+                    torch.save(model.state_dict(), weights_path.replace('.pth', '_test.pth'))
+        
+        torch.save(model.state_dict(), weights_path.replace('.pth', '_last.pth'))
+        logging.info(f'used time {(time.time()-start_time)/3600:.2f}h')
+
+    logging.info(f'Cross Validation Finished!')
+    best_perform_list = np.asarray(best_perform_list)
+    test_perform_list = np.asarray(test_perform_list)
+    logging.info('Best validation perform list\n%s', best_perform_list)
+    logging.info('mean: %s', np.round(np.mean(best_perform_list, 0), 3))
+    logging.info('std: %s', np.round(np.std(best_perform_list, 0), 3))
+    logging.info('Best test perform list\n%s', test_perform_list)
+    logging.info('mean: %s', np.round(np.mean(test_perform_list, 0), 3))
+    logging.info('std: %s', np.round(np.std(test_perform_list, 0), 3))
+    perform = open(weight_dir+'/result.txt', 'w')
+    perform.write('Valid\n')
+    perform.write(','.join([str(i) for i in np.mean(best_perform_list, 0)])+'\n')
+    perform.write(','.join([str(i) for i in np.std(best_perform_list, 0)])+'\n')
+    perform.write('Test\n')
+    perform.write(','.join([str(i) for i in np.mean(test_perform_list, 0)])+'\n')
+    perform.write(','.join([str(i) for i in np.std(test_perform_list, 0)])+'\n')
+
+
+if __name__ == "__main__":
+    main()

main_aug.py

@@ -0,0 +1,412 @@
+import argparse
+import json
+import logging
+import os
+import time
+
+from dataset import PeptidePairDataset, PeptidePairPicDataset
+from network import DMutaPeptide, DMutaPeptideCNN
+from sklearn.model_selection import KFold
+from torchmetrics import MeanAbsoluteError, RelativeSquaredError, PearsonCorrCoef, KendallRankCorrCoef, F1Score, Accuracy, AveragePrecision, AUROC
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader, Subset
+import torchvision.transforms.v2 as T
+import numpy as np
+from loss import MLCE, SuperLoss, LogCoshLoss, BMCLoss
+from utils import set_seed
+
+
+parser = argparse.ArgumentParser(description='resnet26')
+# model setting
+parser.add_argument('--model', type=str, default='resnet34',
+                    help='resnet34 resnet50 densenet')
+parser.add_argument('--q-encoder', dest='q_encoder', type=str, default='lstm',
+                    help='lstm mamba mla')
+parser.add_argument("--side-enc", dest='side_enc', type=str, default=None,
+                    help="use side features")
+parser.add_argument('--channels', type=int, default=256)
+parser.add_argument('--fusion', type=str, default='att',
+                    help='mlp att diff')
+parser.add_argument('--glob-feat', dest='glob_feat', action='store_true', default=False,
+                    help="use global features")
+parser.add_argument('--non-siamese', dest='non_siamese', action='store_true', default=False,
+                    help="use non-siamese architecture")
+
+# task & dataset setting
+parser.add_argument('--task', type=str, default='reg',
+                    help='reg or cls')
+parser.add_argument('--one-way', action='store_true', dest='one_way', default=False,
+                    help='use one-way constructed dataset')
+parser.add_argument('--max-length', dest='max_length', type=int, default=30,
+                    help='Max length for sequence filtering')
+parser.add_argument('--split', type=int, default=5,
+                    help="Split k fold in cross validation (default: 5)")
+parser.add_argument('--seed', type=int, default=42,
+                    help="Seed (default: 1)")
+parser.add_argument('--pcs', action='store_true', default=False,
+                    help='Consider protease cleavage site')
+parser.add_argument('--mix-pcs', dest='mix_pcs', action='store_true', default=False,
+                    help='Consider protease cleavage site')
+parser.add_argument('--resize', type=int, default=[768], nargs='+',
+                    help='resize the image')
+# parser.add_argument('--llm-data', action='store_true', default=False,
+#                     help='Use LLM augmentation data')
+
+# training setting
+parser.add_argument('--gpu', type=int, default=0,
+                    help='GPU index to use, -1 for CPU (default: 0)')
+parser.add_argument('--batch-size', type=int, dest='batch_size', default=32,
+                    help='input batch size for training (default: 128)')
+parser.add_argument('--epochs', type=int, default=50,
+                    help='number of epochs to train (default: 100)')
+parser.add_argument('--lr', type=float, default=0.001,
+                    help='learning rate (default: 0.001)')
+parser.add_argument('--decay', type=float, default=0.0005,
+                    help='weight decay (default: 0.0005)')
+parser.add_argument('--pretrain', type=str, dest='pretrain', default='',
+                    help='path of the pretrain model')
+parser.add_argument('--metric-avg', type=str, dest='metric_avg', default='macro',
+                    help='metric average type')
+
+parser.add_argument('--loss', type=str, default='mse',
+                    help='loss function')
+parser.add_argument('--dir', action='store_true', default=False,
+                    help='use DIR')
+
+args = parser.parse_args()
+
+if args.mix_pcs:
+    args.pcs = 'mix'
+
+
+def main():
+    set_seed(args.seed)
+    if args.task == 'reg':
+        args.classes = 1
+        trainer = train
+        if args.loss == "mse" or args.loss in ['ce']:
+            args.loss = 'mse'
+            criterion = nn.MSELoss()
+        elif args.loss == "smoothl1":
+            criterion = nn.SmoothL1Loss()
+        elif args.loss == "super":
+            criterion = SuperLoss()
+        elif args.loss in ["bmc", "bmc_ln"]:
+            criterion = BMCLoss()
+        else:
+            raise NotImplementedError("unimplemented regression task loss function")
+    elif args.task == 'cls':
+        trainer = train_cls
+        args.classes = 2
+        if args.loss == 'ce' or args.loss in ['mse', 'smoothl1', 'super']:
+            args.loss = 'ce'
+            criterion = nn.CrossEntropyLoss()
+        else:
+            raise NotImplementedError("unimplemented classification task loss function")
+    else:
+        raise NotImplementedError("unimplemented task")
+    
+    if args.q_encoder in ['cnn', 'rn18']:
+        weight_dir = f'./run-{args.task}/{"non-siamese-" if args.non_siamese else ""}{args.q_encoder}-{args.fusion}-{args.channels}{f"-{args.side_enc}" if args.side_enc else ""}{"-mixpcs" if args.mix_pcs else ""}{"-pcs" if args.pcs==True else ""}{"-" + "x".join(str(n) for n in args.resize) if args.resize else ""}{"-gf" if args.glob_feat else ""}{"-oneway" if args.one_way else ""}-{args.loss + "-dir" if args.dir else args.loss}-{str(args.batch_size)}-{str(args.lr)}-{str(args.epochs)}_aug'
+    else:
+        weight_dir = f'./run-{args.task}/{"non-siamese-" if args.non_siamese else ""}{args.q_encoder}-{args.fusion}-{args.channels}{"-gf" if args.glob_feat else ""}{"-oneway" if args.one_way else ""}-{args.loss + "-dir" if args.dir else args.loss}-{str(args.batch_size)}-{str(args.lr)}-{str(args.epochs)}_aug'
+
+    if not os.path.exists(weight_dir):
+        os.makedirs(weight_dir)
+    
+    logging.basicConfig(handlers=[
+        logging.FileHandler(filename=os.path.join(weight_dir, "training.log"), encoding='utf-8', mode='w+'),
+        logging.StreamHandler()],
+        format="%(asctime)s: %(message)s", datefmt="%F %T", level=logging.INFO)
+
+    logging.info(f'saving_dir: {weight_dir}')
+    
+    with open(os.path.join(weight_dir, "config.json"), "w") as f:
+        f.write(json.dumps(vars(args)))
+
+    device = torch.device("cpu" if args.gpu == -1 or not torch.cuda.is_available() else f"cuda:{args.gpu}")
+
+    if args.q_encoder in ['cnn', 'rn18']:
+        logging.info('Loading Training Dataset')
+        all_set = PeptidePairPicDataset(mode='train', pad_length=args.max_length, task=args.task, one_way=args.one_way, gf=args.glob_feat, side_enc=args.side_enc, pcs=args.pcs, resize=args.resize)
+        logging.info('Loading Test Dataset')
+        test_set = PeptidePairPicDataset(mode='test', pad_length=args.max_length, task=args.task, gf=args.glob_feat, side_enc=args.side_enc, pcs=args.pcs, resize=args.resize)
+    else:
+        logging.info('Loading Train Dataset')
+        all_set = PeptidePairDataset(mode='train', pad_length=args.max_length, task=args.task, one_way=args.one_way, gf=args.glob_feat)
+        logging.info('Loading Test Dataset')
+        test_set = PeptidePairDataset(mode='test', pad_length=args.max_length, task=args.task, gf=args.glob_feat)
+
+    test_loader = DataLoader(test_set, batch_size=args.batch_size, shuffle=False, num_workers=8, pin_memory=True)
+
+    best_perform_list = [[] for i in range(5)]
+    test_perform_list = [[] for i in range(5)]
+
+    kf = KFold(n_splits=5, shuffle=True, random_state=42)
+
+    for fold, (train_idx, val_idx) in enumerate(kf.split(all_set)):
+        train_set= Subset(all_set, train_idx)
+        valid_set = Subset(all_set, val_idx)
+
+        train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True, drop_last=True, num_workers=8, pin_memory=True)
+        valid_loader = DataLoader(valid_set, batch_size=args.batch_size, shuffle=False, num_workers=8, pin_memory=True)
+
+        if args.q_encoder in ['cnn', 'rn18']:
+            model = DMutaPeptideCNN(q_encoder=args.q_encoder, classes=args.classes, channels=args.channels, dir=args.dir, gf=args.glob_feat, side_enc=args.side_enc, fusion=args.fusion, non_siamese=args.non_siamese)
+        else:
+            model = DMutaPeptide(q_encoder=args.q_encoder, classes=args.classes, channels=args.channels, dir=args.dir, gf=args.glob_feat, fusion=args.fusion, non_siamese=args.non_siamese)
+        if len(args.pretrain) != 0: #TODO: load pretrain
+            pass
+        model.to(device)
+        # model.compile()
+        
+        optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=args.decay)
+        # optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.decay)
+
+        # scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[10], gamma=0.5)
+        if args.q_encoder == 'cnn':
+            scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.5)
+        else:
+            scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.5)
+
+        if args.loss == 'bmc_ln':
+            optimizer.add_param_group({'params': criterion.noise_sigma, 'lr': args.lr, 'name': 'noise_sigma'})
+        weights_path = f"{weight_dir}/model_{fold}.pth"
+        # early_stopping = EarlyStopping(patience=args.patience, path=weights_path)
+        logging.info(f'Running Cross Validation {fold}')
+        logging.info(f'Fold {fold}  Train set:{len(train_set)}, Valid set:{len(valid_set)}, Test set: {len(test_set)}')
+        best_metric = -float('inf')
+        best_test = -float('inf')
+        start_time = time.time()
+        if args.task == 'reg':
+            for epoch in range(1, args.epochs + 1):
+                train_loss, mae, rse, pcc, kcc = trainer(args, epoch, model, train_loader, valid_loader, device, criterion, optimizer)
+                logging.info(f'Epoch: {epoch:03d} Train Loss: {train_loss:.3f}, mae: {mae:.3f}, rse: {rse:.3f}, pcc: {pcc:.3f}, kcc: {kcc:.3f}')
+                scheduler.step()
+                avg_metric = (pcc + kcc) - (mae + rse)
+                if avg_metric > best_metric:
+                    logging.info(f'Epoch: {epoch:03d} New best VALIDATION metrics')
+                    torch.save(model.state_dict(), weights_path)
+                    best_metric = avg_metric
+                    best_perform_list[fold] = np.asarray([mae, rse, pcc, kcc])
+                
+                _, test_mae, test_rse, test_pcc, test_kcc = trainer(args, epoch, model, None, test_loader, device, None, None)
+                logging.info(f'Epoch: {epoch:03d} Test results, ap: mae: {test_mae:.3f}, rse: {test_rse:.3f}, pcc: {test_pcc:.3f}, kcc: {test_kcc:.3f}')
+                test_metric = (test_pcc + test_kcc) - (test_mae + test_rse)
+                if test_metric > best_test and epoch > 10:
+                    logging.info(f'Epoch: {epoch:03d} New best TEST metrics')
+                    best_test = test_metric
+                    test_perform_list[fold] = np.asarray([test_mae, test_rse, test_pcc, test_kcc])
+                    torch.save(model.state_dict(), weights_path.replace('.pth', '_test.pth'))
+        
+        elif args.task == 'cls':
+            for epoch in range(1, args.epochs + 1):
+                train_loss, ap, auc, f1, acc = trainer(args, epoch, model, train_loader, valid_loader, device, criterion, optimizer)
+                logging.info(f'Epoch: {epoch:03d} Train Loss: {train_loss:.3f}, ap: {ap:.3f}, auc: {auc:.3f}, f1: {f1:.3f}, acc: {acc:.3f}')
+                scheduler.step()
+                avg_metric = ap + auc #+ f1 + acc
+                if avg_metric > best_metric:
+                    logging.info(f'Epoch: {epoch:03d} New best VALIDATION metrics')
+                    torch.save(model.state_dict(), weights_path)
+                    best_metric = avg_metric
+                    best_perform_list[fold] = np.asarray([ap, auc, f1, acc])
+                
+                _, test_ap, test_auc, test_f1, test_acc = trainer(args, epoch, model, None, test_loader, device, None, None)
+                logging.info(f'Epoch: {epoch:03d} Test results, ap: {test_ap:.3f}, auc: {test_auc:.3f}, f1: {test_f1:.3f}, acc: {test_acc:.3f}')
+                test_metric = test_ap + test_auc #+ test_f1 + test_acc
+                if test_metric > best_test and epoch > 10:
+                    logging.info(f'Epoch: {epoch:03d} New best TEST metrics')
+                    best_test = test_metric
+                    test_perform_list[fold] = np.asarray([test_ap, test_auc, test_f1, test_acc])
+                    torch.save(model.state_dict(), weights_path.replace('.pth', '_test.pth'))
+        
+        torch.save(model.state_dict(), weights_path.replace('.pth', '_last.pth'))
+        logging.info(f'used time {(time.time()-start_time)/3600:.2f}h')
+
+    logging.info(f'Cross Validation Finished!')
+    best_perform_list = np.asarray(best_perform_list)
+    test_perform_list = np.asarray(test_perform_list)
+    logging.info('Best validation perform list\n%s', best_perform_list)
+    logging.info('mean: %s', np.round(np.mean(best_perform_list, 0), 3))
+    logging.info('std: %s', np.round(np.std(best_perform_list, 0), 3))
+    logging.info('Best test perform list\n%s', test_perform_list)
+    logging.info('mean: %s', np.round(np.mean(test_perform_list, 0), 3))
+    logging.info('std: %s', np.round(np.std(test_perform_list, 0), 3))
+    perform = open(weight_dir+'/result.txt', 'w')
+    perform.write('Valid\n')
+    perform.write(','.join([str(i) for i in np.mean(best_perform_list, 0)])+'\n')
+    perform.write(','.join([str(i) for i in np.std(best_perform_list, 0)])+'\n')
+    perform.write('Test\n')
+    perform.write(','.join([str(i) for i in np.mean(test_perform_list, 0)])+'\n')
+    perform.write(','.join([str(i) for i in np.std(test_perform_list, 0)])+'\n')
+
+
+def move_to_device(batch, device, non_blocking=False):
+    if isinstance(batch, (list, tuple)):
+        return type(batch)(move_to_device(item, device, non_blocking) for item in batch)
+    return batch.to(device, non_blocking=non_blocking)
+
+
+def move_and_aug(batch, device, transforms, non_blocking=False):
+    batch = move_to_device(batch, device, non_blocking)
+    if not isinstance(batch[0][0], (list, tuple)):
+        return batch
+    
+    for i in range(batch[0][0][0].shape[0]):
+        img_pair = torch.stack((batch[0][0][0][i], batch[0][1][0][i]), dim=0)
+        img_pair = transforms(img_pair)
+        batch[0][0][0][i] = img_pair[0]
+        batch[0][1][0][i] = img_pair[1]
+    return batch
+
+
+class GaussianNoise(nn.Module):
+    def __init__(self, mean=0., sigma=0.15):
+        super(GaussianNoise, self).__init__()
+        self.mean = mean
+        self.sigma = sigma
+
+    def forward(self, x):
+        return x + torch.randn_like(x) * self.sigma + self.mean
+
+
+Transforms = T.Compose([
+    T.RandomResizedCrop(args.resize, scale=(0.9, 1.0)),
+    T.RandomRotation(degrees=30),
+    GaussianNoise(0., 0.05),
+])
+
+def train(args, epoch, model, train_loader, valid_loader, device, criterion, optimizer):
+    train_loss = 0
+    num_labels = model.classes
+    metric_mae = MeanAbsoluteError().to(device)
+    metric_rse = RelativeSquaredError(num_outputs=num_labels).to(device)
+    metric_pcc = PearsonCorrCoef(num_outputs=num_labels).to(device)
+    metric_kcc = KendallRankCorrCoef(num_outputs=num_labels).to(device)
+
+    if args.dir:
+        encodings, labels = [], []
+
+    if train_loader is not None:
+        model.train()
+        for data in train_loader:
+            x, gt = data
+            x = move_and_aug(x, device, Transforms)
+            if args.dir:
+                out, features = model(x,
+                                      gt.to(device),
+                                      epoch)
+                encodings.append(features.detach().cpu())
+                labels.append(gt.cpu())
+            else:
+                out = model(x)
+            loss = criterion(out, gt.to(device))
+            loss.backward()
+            optimizer.step()
+            optimizer.zero_grad()
+            train_loss += loss.item()
+        train_loss /= len(train_loader)
+
+        if args.dir:
+            encodings, labels = torch.cat(encodings), torch.cat(labels)
+            model.FDS.update_last_epoch_stats(epoch)
+            model.FDS.update_running_stats(encodings, labels, epoch)
+            encodings, labels = [], []
+            
+
+    model.eval()
+    preds = []
+    gt_list_valid = []
+    with torch.no_grad():
+        for data in valid_loader:
+            x, gt = data
+            x = move_to_device(x, device)
+            gt_list_valid.append(gt.to(device))
+            out = model(x)
+            if args.dir:
+                out, _ = out
+            preds.append(out)
+
+    # calculate metrics
+    preds = torch.cat(preds, dim=0)
+    gt_list_valid = torch.cat(gt_list_valid, dim=0)
+
+    mae = metric_mae(preds, gt_list_valid).item()
+    rse = metric_rse(preds, gt_list_valid).item()
+    pcc = metric_pcc(preds.squeeze(), gt_list_valid.squeeze()).mean().item()
+    kcc = metric_kcc(preds.squeeze(), gt_list_valid.squeeze()).mean().item()
+    return train_loss, mae, rse, pcc, kcc
+
+
+def update_ce_loss_weight(loss_fn: torch.nn.CrossEntropyLoss, gt: torch.Tensor, num_classes: int, device):
+    """
+    根据当前 batch 的 ground truth 标签更新 nn.CrossEntropyLoss 对象中的 weight 缓冲区，
+    使用逆频率方法计算新权重，并通过 register_buffer 进行原地更新。
+    
+    参数:
+      loss_fn (nn.CrossEntropyLoss): 已初始化的 nn.CrossEntropyLoss 对象，
+                                      要求在初始化时已经注册了 weight 缓冲区。
+      gt (torch.Tensor): 当前 batch 的 ground truth 标签，1D整数张量，标签取值范围 [0, num_classes-1]。
+    """
+    class_counts = torch.bincount(gt, minlength=num_classes).float()
+    epsilon = 1e-6
+    new_weights = 1.0 / (class_counts + epsilon)
+    new_weights = new_weights / new_weights.sum() * num_classes
+    # 使用 register_buffer 来更新 loss_fn 内部的 weight 缓冲区
+    loss_fn.register_buffer('weight', new_weights.to(device))
+
+def train_cls(args, epoch, model, train_loader, valid_loader, device, criterion, optimizer):
+    train_loss = 0
+    num_labels = model.classes
+    avg = args.metric_avg
+    if num_labels == 1 or num_labels == 2:
+        task = 'binary'
+    else:
+        task = 'multiclass'
+    metric_acc = Accuracy(average=avg, task=task, num_classes=num_labels).to(device)
+    metric_f1 = F1Score(average=avg, task=task, num_classes=num_labels).to(device)
+    metric_ap = AveragePrecision(average=avg, task=task, num_classes=num_labels).to(device)
+    metric_auc = AUROC(average=avg, task=task, num_classes=num_labels).to(device)
+
+    if train_loader is not None:
+        model.train()
+        for data in train_loader:
+            x, gt = data
+            x = move_to_device(x, device)
+            out = model(x)
+            update_ce_loss_weight(criterion, gt, num_classes=num_labels, device=device)
+            loss = criterion(out, gt.to(device))
+            loss.backward()
+            optimizer.step()
+            optimizer.zero_grad()
+            train_loss += loss.item()
+        train_loss /= len(train_loader)
+
+    model.eval()
+    preds = []
+    gt_list_valid = []
+    with torch.no_grad():
+        for data in valid_loader:
+            x, gt = data
+            x = move_to_device(x, device)
+            gt_list_valid.append(gt.to(device))
+            out = model(x)
+            preds.append(out)
+
+    # calculate metrics
+    preds = torch.softmax(torch.cat(preds, dim=0), dim=-1).squeeze()
+    gt_list_valid = torch.cat(gt_list_valid, dim=0).int().squeeze()
+
+    if num_labels == 2:
+        preds = preds[:, 1]
+
+    ap = metric_ap(preds, gt_list_valid).item()
+    auc = metric_auc(preds, gt_list_valid).item()
+    f1 = metric_f1(preds, gt_list_valid).item()
+    acc = metric_acc(preds, gt_list_valid).item()
+    return train_loss, ap, auc, f1, acc
+
+
+if __name__ == "__main__":
+    main()

main_imagemol.py

@@ -0,0 +1,246 @@
+import argparse
+import json
+import logging
+import os
+import time
+
+from dataset import PeptidePairDataset, PeptidePairPicDataset
+from network import DMutaPeptide, DMutaPeptideCNN
+from sklearn.model_selection import KFold
+from train import train, train_cls
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader, Subset
+import numpy as np
+from loss import MLCE, SuperLoss, LogCoshLoss, BMCLoss
+from utils import set_seed
+
+
+parser = argparse.ArgumentParser(description='resnet26')
+# model setting
+parser.add_argument('--model', type=str, default='resnet34',
+                    help='resnet34 resnet50 densenet')
+parser.add_argument('--q-encoder', dest='q_encoder', type=str, default='rn18',
+                    help='lstm mamba mla')
+parser.add_argument("--side-enc", dest='side_enc', type=str, default=None,
+                    help="use side features")
+parser.add_argument('--channels', type=int, default=256)
+parser.add_argument('--fusion', type=str, default='att',
+                    help='mlp att diff')
+parser.add_argument('--glob-feat', dest='glob_feat', action='store_true', default=False,
+                    help="use global features")
+parser.add_argument('--non-siamese', dest='non_siamese', action='store_true', default=False,
+                    help="use non-siamese architecture")
+
+# task & dataset setting
+parser.add_argument('--task', type=str, default='reg',
+                    help='reg or cls')
+parser.add_argument('--one-way', action='store_true', dest='one_way', default=False,
+                    help='use one-way constructed dataset')
+parser.add_argument('--max-length', dest='max_length', type=int, default=30,
+                    help='Max length for sequence filtering')
+parser.add_argument('--split', type=int, default=5,
+                    help="Split k fold in cross validation (default: 5)")
+parser.add_argument('--seed', type=int, default=42,
+                    help="Seed (default: 1)")
+parser.add_argument('--pcs', action='store_true', default=False,
+                    help='Consider protease cleavage site')
+parser.add_argument('--mix-pcs', dest='mix_pcs', action='store_true', default=False,
+                    help='Consider protease cleavage site')
+parser.add_argument('--resize', type=int, default=[768], nargs='+',
+                    help='resize the image')
+# parser.add_argument('--llm-data', action='store_true', default=False,
+#                     help='Use LLM augmentation data')
+
+# training setting
+parser.add_argument('--gpu', type=int, default=0,
+                    help='GPU index to use, -1 for CPU (default: 0)')
+parser.add_argument('--batch-size', type=int, dest='batch_size', default=32,
+                    help='input batch size for training (default: 128)')
+parser.add_argument('--epochs', type=int, default=50,
+                    help='number of epochs to train (default: 100)')
+parser.add_argument('--lr', type=float, default=0.001,
+                    help='learning rate (default: 0.001)')
+parser.add_argument('--decay', type=float, default=0.0005,
+                    help='weight decay (default: 0.0005)')
+parser.add_argument('--pretrain', type=str, dest='pretrain', default='',
+                    help='path of the pretrain model')
+parser.add_argument('--metric-avg', type=str, dest='metric_avg', default='macro',
+                    help='metric average type')
+
+parser.add_argument('--loss', type=str, default='mse',
+                    help='loss function')
+parser.add_argument('--dir', action='store_true', default=False,
+                    help='use DIR')
+
+args = parser.parse_args()
+
+if args.mix_pcs:
+    args.pcs = 'mix'
+
+
+def main():
+    set_seed(args.seed)
+    if args.task == 'reg':
+        args.classes = 1
+        trainer = train
+        if args.loss == "mse" or args.loss in ['ce']:
+            args.loss = 'mse'
+            criterion = nn.MSELoss()
+        elif args.loss == "smoothl1":
+            criterion = nn.SmoothL1Loss()
+        elif args.loss == "super":
+            criterion = SuperLoss()
+        elif args.loss in ["bmc", "bmc_ln"]:
+            criterion = BMCLoss()
+        else:
+            raise NotImplementedError("unimplemented regression task loss function")
+    elif args.task == 'cls':
+        trainer = train_cls
+        args.classes = 2
+        if args.loss == 'ce' or args.loss in ['mse', 'smoothl1', 'super']:
+            args.loss = 'ce'
+            criterion = nn.CrossEntropyLoss()
+        else:
+            raise NotImplementedError("unimplemented classification task loss function")
+    else:
+        raise NotImplementedError("unimplemented task")
+    
+    if args.q_encoder in ['cnn', 'rn18']:
+        weight_dir = f'./run-{args.task}/{"non-siamese-" if args.non_siamese else ""}{args.q_encoder}-{args.fusion}-{args.channels}{f"-{args.side_enc}" if args.side_enc else ""}{"-mixpcs" if args.mix_pcs else ""}{"-pcs" if args.pcs==True else ""}{"-" + "x".join(str(n) for n in args.resize) if args.resize else ""}{"-gf" if args.glob_feat else ""}{"-oneway" if args.one_way else ""}-{args.loss + "-dir" if args.dir else args.loss}-{str(args.batch_size)}-{str(args.lr)}-{str(args.epochs)}_ImageMol'
+    else:
+        weight_dir = f'./run-{args.task}/{"non-siamese-" if args.non_siamese else ""}{args.q_encoder}-{args.fusion}-{args.channels}{"-gf" if args.glob_feat else ""}{"-oneway" if args.one_way else ""}-{args.loss + "-dir" if args.dir else args.loss}-{str(args.batch_size)}-{str(args.lr)}-{str(args.epochs)}_ImageMol'
+
+    if not os.path.exists(weight_dir):
+        os.makedirs(weight_dir)
+    
+    logging.basicConfig(handlers=[
+        logging.FileHandler(filename=os.path.join(weight_dir, "training.log"), encoding='utf-8', mode='w+'),
+        logging.StreamHandler()],
+        format="%(asctime)s: %(message)s", datefmt="%F %T", level=logging.INFO)
+
+    logging.info(f'saving_dir: {weight_dir}')
+    
+    with open(os.path.join(weight_dir, "config.json"), "w") as f:
+        f.write(json.dumps(vars(args)))
+
+    device = torch.device("cpu" if args.gpu == -1 or not torch.cuda.is_available() else f"cuda:{args.gpu}")
+
+    if args.q_encoder in ['cnn', 'rn18']:
+        logging.info('Loading Training Dataset')
+        all_set = PeptidePairPicDataset(mode='train', pad_length=args.max_length, task=args.task, one_way=args.one_way, gf=args.glob_feat, side_enc=args.side_enc, pcs=args.pcs, resize=args.resize)
+        logging.info('Loading Test Dataset')
+        test_set = PeptidePairPicDataset(mode='test', pad_length=args.max_length, task=args.task, gf=args.glob_feat, side_enc=args.side_enc, pcs=args.pcs, resize=args.resize)
+    else:
+        logging.info('Loading Train Dataset')
+        all_set = PeptidePairDataset(mode='train', pad_length=args.max_length, task=args.task, one_way=args.one_way, gf=args.glob_feat)
+        logging.info('Loading Test Dataset')
+        test_set = PeptidePairDataset(mode='test', pad_length=args.max_length, task=args.task, gf=args.glob_feat)
+
+    test_loader = DataLoader(test_set, batch_size=args.batch_size, shuffle=False, num_workers=8, pin_memory=True)
+
+    best_perform_list = [[] for i in range(5)]
+    test_perform_list = [[] for i in range(5)]
+
+    kf = KFold(n_splits=5, shuffle=True, random_state=42)
+
+    for fold, (train_idx, val_idx) in enumerate(kf.split(all_set)):
+        train_set= Subset(all_set, train_idx)
+        valid_set = Subset(all_set, val_idx)
+
+        train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True, drop_last=True, num_workers=8, pin_memory=True)
+        valid_loader = DataLoader(valid_set, batch_size=args.batch_size, shuffle=False, num_workers=8, pin_memory=True)
+
+        if args.q_encoder in ['cnn', 'rn18']:
+            model = DMutaPeptideCNN(q_encoder=args.q_encoder, classes=args.classes, channels=args.channels, dir=args.dir, gf=args.glob_feat, side_enc=args.side_enc, fusion=args.fusion, non_siamese=args.non_siamese)
+            model.q_encoder.load_state_dict(torch.load('./ImageMolEncoder.pth', map_location=device))
+        else:
+            model = DMutaPeptide(q_encoder=args.q_encoder, classes=args.classes, channels=args.channels, dir=args.dir, gf=args.glob_feat, fusion=args.fusion, non_siamese=args.non_siamese)
+        if len(args.pretrain) != 0: #TODO: load pretrain
+            pass
+        model.to(device)
+        # model.compile()
+        
+        optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=args.decay)
+        # optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.decay)
+
+        # scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[10], gamma=0.5)
+        if args.q_encoder == 'cnn':
+            scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.5)
+        else:
+            scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.5)
+
+        if args.loss == 'bmc_ln':
+            optimizer.add_param_group({'params': criterion.noise_sigma, 'lr': args.lr, 'name': 'noise_sigma'})
+        weights_path = f"{weight_dir}/model_{fold}.pth"
+        # early_stopping = EarlyStopping(patience=args.patience, path=weights_path)
+        logging.info(f'Running Cross Validation {fold}')
+        logging.info(f'Fold {fold}  Train set:{len(train_set)}, Valid set:{len(valid_set)}, Test set: {len(test_set)}')
+        best_metric = -float('inf')
+        best_test = -float('inf')
+        start_time = time.time()
+        if args.task == 'reg':
+            for epoch in range(1, args.epochs + 1):
+                train_loss, mae, rse, pcc, kcc = trainer(args, epoch, model, train_loader, valid_loader, device, criterion, optimizer)
+                logging.info(f'Epoch: {epoch:03d} Train Loss: {train_loss:.3f}, mae: {mae:.3f}, rse: {rse:.3f}, pcc: {pcc:.3f}, kcc: {kcc:.3f}')
+                scheduler.step()
+                avg_metric = (pcc + kcc) - (mae + rse)
+                if avg_metric > best_metric:
+                    logging.info(f'Epoch: {epoch:03d} New best VALIDATION metrics')
+                    torch.save(model.state_dict(), weights_path)
+                    best_metric = avg_metric
+                    best_perform_list[fold] = np.asarray([mae, rse, pcc, kcc])
+                
+                _, test_mae, test_rse, test_pcc, test_kcc = trainer(args, epoch, model, None, test_loader, device, None, None)
+                logging.info(f'Epoch: {epoch:03d} Test results, ap: mae: {test_mae:.3f}, rse: {test_rse:.3f}, pcc: {test_pcc:.3f}, kcc: {test_kcc:.3f}')
+                test_metric = (test_pcc + test_kcc) - (test_mae + test_rse)
+                if test_metric > best_test and epoch > 10:
+                    logging.info(f'Epoch: {epoch:03d} New best TEST metrics')
+                    best_test = test_metric
+                    test_perform_list[fold] = np.asarray([test_mae, test_rse, test_pcc, test_kcc])
+                    torch.save(model.state_dict(), weights_path.replace('.pth', '_test.pth'))
+        
+        elif args.task == 'cls':
+            for epoch in range(1, args.epochs + 1):
+                train_loss, ap, auc, f1, acc = trainer(args, epoch, model, train_loader, valid_loader, device, criterion, optimizer)
+                logging.info(f'Epoch: {epoch:03d} Train Loss: {train_loss:.3f}, ap: {ap:.3f}, auc: {auc:.3f}, f1: {f1:.3f}, acc: {acc:.3f}')
+                scheduler.step()
+                avg_metric = ap + auc #+ f1 + acc
+                if avg_metric > best_metric:
+                    logging.info(f'Epoch: {epoch:03d} New best VALIDATION metrics')
+                    torch.save(model.state_dict(), weights_path)
+                    best_metric = avg_metric
+                    best_perform_list[fold] = np.asarray([ap, auc, f1, acc])
+                
+                _, test_ap, test_auc, test_f1, test_acc = trainer(args, epoch, model, None, test_loader, device, None, None)
+                logging.info(f'Epoch: {epoch:03d} Test results, ap: {test_ap:.3f}, auc: {test_auc:.3f}, f1: {test_f1:.3f}, acc: {test_acc:.3f}')
+                test_metric = test_ap + test_auc #+ test_f1 + test_acc
+                if test_metric > best_test and epoch > 10:
+                    logging.info(f'Epoch: {epoch:03d} New best TEST metrics')
+                    best_test = test_metric
+                    test_perform_list[fold] = np.asarray([test_ap, test_auc, test_f1, test_acc])
+                    torch.save(model.state_dict(), weights_path.replace('.pth', '_test.pth'))
+        
+        torch.save(model.state_dict(), weights_path.replace('.pth', '_last.pth'))
+        logging.info(f'used time {(time.time()-start_time)/3600:.2f}h')
+
+    logging.info(f'Cross Validation Finished!')
+    best_perform_list = np.asarray(best_perform_list)
+    test_perform_list = np.asarray(test_perform_list)
+    logging.info('Best validation perform list\n%s', best_perform_list)
+    logging.info('mean: %s', np.round(np.mean(best_perform_list, 0), 3))
+    logging.info('std: %s', np.round(np.std(best_perform_list, 0), 3))
+    logging.info('Best test perform list\n%s', test_perform_list)
+    logging.info('mean: %s', np.round(np.mean(test_perform_list, 0), 3))
+    logging.info('std: %s', np.round(np.std(test_perform_list, 0), 3))
+    perform = open(weight_dir+'/result.txt', 'w')
+    perform.write('Valid\n')
+    perform.write(','.join([str(i) for i in np.mean(best_perform_list, 0)])+'\n')
+    perform.write(','.join([str(i) for i in np.std(best_perform_list, 0)])+'\n')
+    perform.write('Test\n')
+    perform.write(','.join([str(i) for i in np.mean(test_perform_list, 0)])+'\n')
+    perform.write(','.join([str(i) for i in np.std(test_perform_list, 0)])+'\n')
+
+
+if __name__ == "__main__":
+    main()

main_simple.py

@@ -0,0 +1,208 @@
+import argparse
+import json
+import logging
+import os
+import time
+
+from dataset import PeptidePairDataset, PeptidePairPicDataset, SimplePairClsDataset
+from network import DMutaPeptide, DMutaPeptideCNN
+from sklearn.model_selection import KFold
+from train import train, train_cls
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader, Subset
+import numpy as np
+from loss import MLCE, SuperLoss, LogCoshLoss, BMCLoss
+from utils import set_seed
+
+
+parser = argparse.ArgumentParser(description='resnet26')
+# model setting
+parser.add_argument('--model', type=str, default='resnet34',
+                    help='resnet34 resnet50 densenet')
+parser.add_argument('--q-encoder', dest='q_encoder', type=str, default='lstm',
+                    help='lstm mamba mla')
+parser.add_argument("--side-enc", dest='side_enc', type=str, default=None,
+                    help="use side features")
+parser.add_argument('--channels', type=int, default=256)
+parser.add_argument('--fusion', type=str, default='att',
+                    help='mlp att diff')
+parser.add_argument('--glob-feat', dest='glob_feat', action='store_true', default=False,
+                    help="use global features")
+parser.add_argument('--non-siamese', dest='non_siamese', action='store_true', default=False,
+                    help="use non-siamese architecture")
+
+# task & dataset setting
+parser.add_argument('--task', type=str, default='cls',
+                    help='reg or cls')
+parser.add_argument('--one-way', action='store_true', dest='one_way', default=True,
+                    help='use one-way constructed dataset')
+parser.add_argument('--max-length', dest='max_length', type=int, default=30,
+                    help='Max length for sequence filtering')
+parser.add_argument('--split', type=int, default=5,
+                    help="Split k fold in cross validation (default: 5)")
+parser.add_argument('--seed', type=int, default=1,
+                    help="Seed (default: 1)")
+parser.add_argument('--pcs', action='store_true', default=False,
+                    help='Consider protease cut site')
+parser.add_argument('--mix-pcs', dest='mix_pcs', action='store_true', default=False,
+                    help='Consider protease cut site')
+parser.add_argument('--resize', type=int, default=[768], nargs='+',
+                    help='resize the image')
+parser.add_argument('--llm-data', action='store_true', default=False,
+                    help='Use LLM augmentation data')
+
+# training setting
+parser.add_argument('--gpu', type=int, default=0,
+                    help='GPU index to use, -1 for CPU (default: 0)')
+parser.add_argument('--batch-size', type=int, dest='batch_size', default=32,
+                    help='input batch size for training (default: 128)')
+parser.add_argument('--epochs', type=int, default=50,
+                    help='number of epochs to train (default: 100)')
+parser.add_argument('--lr', type=float, default=0.001,
+                    help='learning rate (default: 0.001)')
+parser.add_argument('--decay', type=float, default=0.0005,
+                    help='weight decay (default: 0.0005)')
+parser.add_argument('--pretrain', type=str, dest='pretrain', default='',
+                    help='path of the pretrain model')
+parser.add_argument('--metric-avg', type=str, dest='metric_avg', default='macro',
+                    help='metric average type')
+
+parser.add_argument('--loss', type=str, default='ce',
+                    help='loss function')
+parser.add_argument('--dir', action='store_true', default=False,
+                    help='use DIR')
+
+args = parser.parse_args()
+
+if args.mix_pcs:
+    args.pcs = 'mix'
+
+
+def main():
+    set_seed(args.seed)
+    if args.task == 'reg':
+        raise NotImplementedError("unimplemented regression task")
+    elif args.task == 'cls':
+        trainer = train_cls
+        args.classes = 2
+        if args.loss == 'ce' or args.loss in ['mse', 'smoothl1', 'super']:
+            args.loss = 'ce'
+            criterion = nn.CrossEntropyLoss()
+        else:
+            raise NotImplementedError("unimplemented classification task loss function")
+    else:
+        raise NotImplementedError("unimplemented task")
+    
+    if args.q_encoder in ['cnn', 'rn18']:
+        weight_dir = f'./run-{args.task}/{"non-siamese-" if args.non_siamese else ""}{args.q_encoder}-{args.fusion}-{args.channels}{f"-{args.side_enc}" if args.side_enc else ""}{"-mixpcs" if args.mix_pcs else ""}{"-pcs" if args.pcs==True else ""}-simple{"-llm" if args.llm_data else ""}{"-" + "x".join(str(n) for n in args.resize) if args.resize else ""}{"-gf" if args.glob_feat else ""}{"-oneway" if args.one_way else ""}-{args.loss + "-dir" if args.dir else args.loss}-{str(args.batch_size)}-{str(args.lr)}-{str(args.epochs)}'
+    else:
+        weight_dir = f'./run-{args.task}/{f"non-siamese-" if args.non_siamese else ""}{args.q_encoder}-{args.fusion}-{args.channels}-simple{"-llm" if args.llm_data else ""}{"-gf" if args.glob_feat else ""}{"-oneway" if args.one_way else ""}-{args.loss + "-dir" if args.dir else args.loss}-{str(args.batch_size)}-{str(args.lr)}-{str(args.epochs)}'
+
+    if not os.path.exists(weight_dir):
+        os.makedirs(weight_dir)
+    
+    logging.basicConfig(handlers=[
+        logging.FileHandler(filename=os.path.join(weight_dir, "training.log"), encoding='utf-8', mode='w+'),
+        logging.StreamHandler()],
+        format="%(asctime)s: %(message)s", datefmt="%F %T", level=logging.INFO)
+
+    logging.info(f'saving_dir: {weight_dir}')
+    
+    with open(os.path.join(weight_dir, "config.json"), "w") as f:
+        f.write(json.dumps(vars(args)))
+
+    device = torch.device("cpu" if args.gpu == -1 or not torch.cuda.is_available() else f"cuda:{args.gpu}")
+
+    logging.info('Loading Training Dataset')
+    all_set = SimplePairClsDataset(pad_length=args.max_length, llm=args.llm_data, gf=args.glob_feat, q_encoder=args.q_encoder, side_enc=args.side_enc, pcs=args.pcs, resize=args.resize)
+
+    logging.info('Loading Test Dataset')
+    if args.q_encoder in ['cnn', 'rn18']:
+        test_set = PeptidePairPicDataset(mode='test', pad_length=args.max_length, task=args.task, gf=args.glob_feat, side_enc=args.side_enc, pcs=args.pcs, resize=args.resize)
+    else:
+        test_set = PeptidePairDataset(mode='test', pad_length=args.max_length, task=args.task, gf=args.glob_feat)
+
+    test_loader = DataLoader(test_set, batch_size=args.batch_size, shuffle=False, num_workers=8, pin_memory=True)
+
+    best_perform_list = [[] for i in range(5)]
+    test_perform_list = [[] for i in range(5)]
+
+    kf = KFold(n_splits=5, shuffle=True, random_state=42)
+
+    for fold, (train_idx, val_idx) in enumerate(kf.split(all_set)):
+        train_set= Subset(all_set, train_idx)
+        valid_set = Subset(all_set, val_idx)
+
+        train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True, drop_last=True, num_workers=8, pin_memory=True)
+        valid_loader = DataLoader(valid_set, batch_size=args.batch_size, shuffle=False, num_workers=8, pin_memory=True)
+
+        if args.q_encoder in ['cnn', 'rn18']:
+            model = DMutaPeptideCNN(q_encoder=args.q_encoder, classes=args.classes, channels=args.channels, dir=args.dir, gf=args.glob_feat, side_enc=args.side_enc, fusion=args.fusion, non_siamese=args.non_siamese)
+        else:
+            model = DMutaPeptide(q_encoder=args.q_encoder, classes=args.classes, channels=args.channels, dir=args.dir, gf=args.glob_feat, fusion=args.fusion, non_siamese=args.non_siamese)
+        if len(args.pretrain) != 0: #TODO: load pretrain
+            pass
+        model.to(device)
+        # model.compile()
+        
+        optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=args.decay)
+
+        if args.q_encoder == 'cnn':
+            scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.5)
+        else:
+            scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.5)
+
+        if args.loss == 'bmc_ln':
+            optimizer.add_param_group({'params': criterion.noise_sigma, 'lr': args.lr, 'name': 'noise_sigma'})
+        weights_path = f"{weight_dir}/model_{fold}.pth"
+        # early_stopping = EarlyStopping(patience=args.patience, path=weights_path)
+        logging.info(f'Running Cross Validation {fold}')
+        logging.info(f'Fold {fold}  Train set:{len(train_set)}, Valid set:{len(valid_set)}, Test set: {len(test_set)}')
+        best_metric = -float('inf')
+        best_test = -float('inf')
+        start_time = time.time()
+        if args.task == 'cls':
+            for epoch in range(1, args.epochs + 1):
+                train_loss, ap, auc, f1, acc = trainer(args, epoch, model, train_loader, valid_loader, device, criterion, optimizer)
+                logging.info(f'Epoch: {epoch:03d} Train Loss: {train_loss:.3f}, ap: {ap:.3f}, auc: {auc:.3f}, f1: {f1:.3f}, acc: {acc:.3f}')
+                scheduler.step()
+                avg_metric = ap + auc #+ f1 + acc
+                if avg_metric > best_metric:
+                    logging.info(f'Epoch: {epoch:03d} New best VALIDATION metrics')
+                    torch.save(model.state_dict(), weights_path)
+                    best_metric = avg_metric
+                    best_perform_list[fold] = np.asarray([ap, auc, f1, acc])
+                
+                _, test_ap, test_auc, test_f1, test_acc = trainer(args, epoch, model, None, test_loader, device, None, None)
+                logging.info(f'Epoch: {epoch:03d} Test results, ap: {test_ap:.3f}, auc: {test_auc:.3f}, f1: {test_f1:.3f}, acc: {test_acc:.3f}')
+                test_metric = test_ap + test_auc #+ test_f1 + test_acc
+                if test_metric > best_test and epoch > 10:
+                    logging.info(f'Epoch: {epoch:03d} New best TEST metrics')
+                    best_test = test_metric
+                    test_perform_list[fold] = np.asarray([test_ap, test_auc, test_f1, test_acc])
+                    torch.save(model.state_dict(), weights_path.replace('.pth', '_test.pth'))
+        
+        torch.save(model.state_dict(), weights_path.replace('.pth', '_last.pth'))
+        logging.info(f'used time {(time.time()-start_time)/3600:.2f}h')
+
+    logging.info(f'Cross Validation Finished!')
+    best_perform_list = np.asarray(best_perform_list)
+    test_perform_list = np.asarray(test_perform_list)
+    logging.info('Best validation perform list\n%s', best_perform_list)
+    logging.info('mean: %s', np.round(np.mean(best_perform_list, 0), 3))
+    logging.info('std: %s', np.round(np.std(best_perform_list, 0), 3))
+    logging.info('Best test perform list\n%s', test_perform_list)
+    logging.info('mean: %s', np.round(np.mean(test_perform_list, 0), 3))
+    logging.info('std: %s', np.round(np.std(test_perform_list, 0), 3))
+    perform = open(weight_dir+'/result.txt', 'w')
+    perform.write('Valid\n')
+    perform.write(','.join([str(i) for i in np.mean(best_perform_list, 0)])+'\n')
+    perform.write(','.join([str(i) for i in np.std(best_perform_list, 0)])+'\n')
+    perform.write('Test\n')
+    perform.write(','.join([str(i) for i in np.mean(test_perform_list, 0)])+'\n')
+    perform.write(','.join([str(i) for i in np.std(test_perform_list, 0)])+'\n')
+
+
+if __name__ == "__main__":
+    main()

network.py

@@ -0,0 +1,586 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from copy import deepcopy
+from mamba_ssm import Mamba
+from utils import FDS
+from torchvision.models import resnet18
+
+class MambaModel(nn.Module):
+    def __init__(self, d_model, max_length=30):
+        super(MambaModel, self).__init__()
+        self.linear = nn.Linear(in_features=21, out_features=d_model)
+        self.pos_encoder = PositionalEncoding(d_model, max_length)
+        self.mamba = Mamba(d_model=d_model, d_state=32, expand=4)
+        self.global_pool = nn.AdaptiveAvgPool1d(1)
+
+    def forward(self, x: torch.Tensor):
+        x = self.pos_encoder(self.linear(x))
+        y = self.mamba(x)
+        y_flip = self.mamba(x.flip([-2])).flip([-2])
+        y = torch.cat((y, y_flip), dim=-1)
+        y = self.global_pool(y.permute(0, 2, 1)).squeeze(-1)
+        return y
+
+
+class MLP(nn.Module):
+    def __init__(self, input_dim, hidden_dim, output_dim, num_layers=3, dropout_rate=0.1):
+        super(MLP, self).__init__()
+        if isinstance(hidden_dim, int):
+            hidden_dim = [hidden_dim] * num_layers
+        
+        layers = []
+        layers.append(nn.Linear(input_dim, hidden_dim[0]))
+        layers.append(nn.ReLU())
+        layers.append(nn.Dropout(dropout_rate))
+        
+        for i in range(len(hidden_dim) - 1):
+            layers.append(nn.Linear(hidden_dim[i], hidden_dim[i + 1]))
+            layers.append(nn.ReLU())
+            layers.append(nn.Dropout(dropout_rate))
+        
+        layers.append(nn.Linear(hidden_dim[-1], output_dim))
+        
+        self.network = nn.Sequential(*layers)
+
+    def forward(self, x):
+        return self.network(x)
+
+
+class PositionalEncoding(nn.Module):
+    def __init__(self, d_model, max_len=50):
+        super(PositionalEncoding, self).__init__()
+        
+        pe = torch.zeros(max_len, d_model)  # (max_len, d_model)
+        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)  # (max_len, 1)
+        div_term = torch.exp(torch.arange(0, d_model, 2).float() * 
+                             (-torch.log(torch.FloatTensor([10000.0])) / d_model))  # (d_model/2,)
+        pe[:, 0::2] = torch.sin(position * div_term)  # 偶数维
+        pe[:, 1::2] = torch.cos(position * div_term)  # 奇数维
+        pe = pe.unsqueeze(0)  # (1, max_len, d_model)
+        self.register_buffer('pe', pe)
+        
+    def forward(self, x):
+        """
+        x: (B, N, d_model)
+        """
+        x = x + self.pe[:, :x.size(1), :]
+        return x
+
+
+class MHAModel(nn.Module):
+    def __init__(self, d_model, max_length=50):
+        super(MHAModel, self).__init__()
+        self.linear = nn.Linear(in_features=21, out_features=d_model)
+        self.pos_encoder = PositionalEncoding(d_model, max_length)
+        self.self_attn = nn.MultiheadAttention(d_model, num_heads=8, batch_first=True)
+        self.global_pool = nn.AdaptiveAvgPool1d(1)
+
+    def forward(self, x: torch.Tensor):
+        # 线性变换 + 位置编码
+        x = self.pos_encoder(self.linear(x))  # [batch, seq_len, d_model]
+        
+        # 正向自注意力
+        y, _ = self.self_attn(x, x, x)  # [batch, seq_len, d_model]
+        
+        # 反向自注意力
+        x_flip = x.flip([-2])  # 沿序列维度翻转
+        y_flip, _ = self.self_attn(x_flip, x_flip, x_flip)
+        y_flip = y_flip.flip([-2])  # 翻转回原顺序
+        
+        # 拼接正反向结果
+        y = torch.cat((y, y_flip), dim=-1)  # [batch, seq_len, 2*d_model]
+        
+        # 全局池化
+        y = self.global_pool(y.permute(0, 2, 1))  # [batch, 2*d_model, 1]
+        return y.squeeze(-1)  # [batch, 2*d_model]  
+    
+
+class MLAModel(nn.Module):
+    def __init__(self, d_model, max_length=50):
+        super(MLAModel, self).__init__()
+        self.linear = nn.Linear(in_features=21, out_features=d_model)
+        self.pos_encoder = PositionalEncoding(d_model, max_length)
+        self.MLA = MLA(d_model, n_heads=8, max_len=max_length)
+        self.global_pool = nn.AdaptiveAvgPool1d(1)
+
+    def forward(self, x: torch.Tensor):
+        x = self.pos_encoder(self.linear(x))
+        y = self.MLA(x)
+        y_flip = self.MLA(x.flip([-2])).flip([-2])
+        y = torch.cat((y, y_flip), dim=-1)
+        y = self.global_pool(y.permute(0, 2, 1)).squeeze(-1)
+        return y
+
+
+class MLA(nn.Module):
+    def __init__(self, d_model, n_heads, max_len=50, rope_theta=10000.0):
+        super().__init__()
+        self.d_model = d_model
+        self.n_heads = n_heads
+        self.dh = d_model // n_heads
+        self.q_proj_dim = d_model // 2
+        self.kv_proj_dim = (2*d_model) // 3
+
+        self.qk_nope_dim = self.dh // 2
+        self.qk_rope_dim = self.dh // 2
+        
+        ## Q projections
+        # Lora
+        self.W_dq = nn.Parameter(0.01*torch.randn((d_model, self.q_proj_dim)))
+        self.W_uq = nn.Parameter(0.01*torch.randn((self.q_proj_dim, self.d_model)))
+        self.q_layernorm = nn.LayerNorm(self.q_proj_dim)
+        
+        ## KV projections
+        # Lora
+        self.W_dkv = nn.Parameter(0.01*torch.randn((d_model, self.kv_proj_dim + self.qk_rope_dim)))
+        self.W_ukv = nn.Parameter(0.01*torch.randn((self.kv_proj_dim,
+                                                          self.d_model + (self.n_heads * self.qk_nope_dim))))
+        self.kv_layernorm = nn.LayerNorm(self.kv_proj_dim)
+        
+        # output projection
+        self.W_o = nn.Parameter(0.01*torch.randn((d_model, d_model)))
+
+        # RoPE
+        self.max_seq_len = max_len
+        self.rope_theta = rope_theta
+
+        # https://github.com/lucidrains/rotary-embedding-torch/tree/main
+        # visualize emb later to make sure it looks ok
+        # we do self.dh here instead of self.qk_rope_dim because its better
+        freqs = 1.0 / (rope_theta ** (torch.arange(0, self.dh, 2).float() / self.dh))
+        emb = torch.outer(torch.arange(self.max_seq_len).float(), freqs)
+        cos_cached = emb.cos()[None, None, :, :]
+        sin_cached = emb.sin()[None, None, :, :]
+
+        # https://pytorch.org/docs/stable/generated/torch.nn.Module.html#torch.nn.Module.register_buffer
+        # This is like a parameter but its a constant so we can use register_buffer
+        self.register_buffer("cos_cached", cos_cached)
+        self.register_buffer("sin_cached", sin_cached)
+    
+    def apply_rope_x(self, x, cos, sin):
+        return (x * cos) + (self.rotate_half(x) * sin)
+    
+    @staticmethod
+    def rotate_half(x):
+        x1, x2 = x.chunk(2, dim=-1)
+        return torch.cat((-x2, x1), dim=-1)
+
+    def forward(self, x, kv_cache=None, past_length=0):
+        B, S, D = x.size()
+
+        # Q Projections
+        compressed_q = x @ self.W_dq
+        compressed_q = self.q_layernorm(compressed_q)
+        Q = compressed_q @ self.W_uq
+        Q = Q.view(B, -1, self.n_heads, self.dh).transpose(1,2)
+        Q, Q_for_rope = torch.split(Q, [self.qk_nope_dim, self.qk_rope_dim], dim=-1)
+
+        # Q Decoupled RoPE
+        cos_q = self.cos_cached[:, :, past_length:past_length+S, :self.qk_rope_dim//2].repeat(1, 1, 1, 2)
+        sin_q = self.sin_cached[:, :, past_length:past_length+S, :self.qk_rope_dim//2].repeat(1, 1, 1, 2)
+        Q_for_rope = self.apply_rope_x(Q_for_rope, cos_q, sin_q)
+
+        # KV Projections
+        if kv_cache is None:
+            compressed_kv = x @ self.W_dkv
+            KV_for_lora, K_for_rope = torch.split(compressed_kv,
+                                                  [self.kv_proj_dim, self.qk_rope_dim],
+                                                  dim=-1)
+            KV_for_lora = self.kv_layernorm(KV_for_lora)
+        else:
+            new_kv = x @ self.W_dkv
+            compressed_kv = torch.cat([kv_cache, new_kv], dim=1)
+            new_kv, new_K_for_rope = torch.split(new_kv,
+                                                 [self.kv_proj_dim, self.qk_rope_dim],
+                                                 dim=-1)
+            old_kv, old_K_for_rope = torch.split(kv_cache,
+                                                 [self.kv_proj_dim, self.qk_rope_dim],
+                                                 dim=-1)
+            new_kv = self.kv_layernorm(new_kv)
+            old_kv = self.kv_layernorm(old_kv)
+            KV_for_lora = torch.cat([old_kv, new_kv], dim=1)
+            K_for_rope = torch.cat([old_K_for_rope, new_K_for_rope], dim=1)
+            
+
+        KV = KV_for_lora @ self.W_ukv
+        KV = KV.view(B, -1, self.n_heads, self.dh+self.qk_nope_dim).transpose(1,2)
+        K, V = torch.split(KV, [self.qk_nope_dim, self.dh], dim=-1)
+        S_full = K.size(2)        
+
+        # K Rope
+        K_for_rope = K_for_rope.view(B, -1, 1, self.qk_rope_dim).transpose(1,2)
+        cos_k = self.cos_cached[:, :, :S_full, :self.qk_rope_dim//2].repeat(1, 1, 1, 2)
+        sin_k = self.sin_cached[:, :, :S_full, :self.qk_rope_dim//2].repeat(1, 1, 1, 2)
+        K_for_rope = self.apply_rope_x(K_for_rope, cos_k, sin_k)
+
+        # apply position encoding to each head
+        K_for_rope = K_for_rope.repeat(1, self.n_heads, 1, 1)
+
+        # split into multiple heads
+        q_heads = torch.cat([Q, Q_for_rope], dim=-1)
+        k_heads = torch.cat([K, K_for_rope], dim=-1)
+        v_heads = V # already reshaped before the split
+
+        # make attention mask
+        mask = torch.ones((S,S_full), device=x.device)
+        mask = torch.tril(mask, diagonal=past_length)
+        mask = mask[None, None, :, :]
+
+        sq_mask = mask == 1
+
+        # attention
+        x = nn.functional.scaled_dot_product_attention(
+            q_heads, k_heads, v_heads,
+            attn_mask=sq_mask
+        )
+
+        x = x.transpose(1, 2).reshape(B, S, D)
+
+        # apply projection
+        x = x @ self.W_o.T
+
+        return x
+    
+
+class DMutaPeptide(nn.Module):
+    def __init__(self, q_encoder='lstm', classes=1, channels=128, dir=False, gf=False, fusion='mlp', non_siamese=False):
+        """
+        参数:
+            q_encoder: 使用的编码器类型，支持 'lstm', 'mamba', 'mla', 'mha'
+            classes: 输出类别数
+            channels: 通道数量，影响隐藏状态维度
+            dir: 是否使用 DIR 模块
+            fusion: 融合方法，可选 'mlp'（默认，直接拼接）或 'att'（使用 attention 融合）
+        """
+        super().__init__()
+        self.classes = classes
+        self.DIR = dir
+        self.gf = gf
+        self.fusion_method = fusion  # 融合方式
+        self.non_siamese = non_siamese
+        # 拼接后维度设定为 channels * 4
+        final_dim = channels * 4
+
+
+        # 初始化编码器
+        if q_encoder == 'lstm':
+            self.q_encoder = nn.LSTM(
+                input_size=21,
+                hidden_size=channels,
+                num_layers=2,
+                batch_first=True,  # 输入和输出均以 (batch, time_step, input_size) 表示
+                dropout=0.1,
+                bidirectional=True
+            )
+        elif q_encoder == 'gru':
+            self.q_encoder = nn.GRU(
+                input_size=21,
+                hidden_size=channels,
+                num_layers=2,
+                batch_first=True,  # 输入和输出均以 (batch, time_step, input_size) 表示
+                dropout=0.1,
+                bidirectional=True
+            )
+        elif q_encoder == 'mamba':
+            self.q_encoder = MambaModel(channels, 30)
+        elif q_encoder == 'mla':
+            self.q_encoder = MLAModel(channels, 30)
+        elif q_encoder == 'mha':
+            self.q_encoder = MHAModel(channels, 30)
+        else:
+            raise NotImplementedError
+        
+        if non_siamese:
+            self.q_encoder_2 = deepcopy(self.q_encoder)
+        else:
+            self.q_encoder_2 = self.q_encoder
+        
+        if self.fusion_method == 'diff':
+            final_dim //= 2
+
+        if gf:
+            self.g_encoder = MLP(1024, [512, 256, 128], channels * 2, dropout_rate=0.3)
+            final_dim += channels * 2
+
+        # 如果 fusion 模式为 'att' ，则使用 MultiheadAttention 对两个向量进行融合
+        if self.fusion_method == 'att':
+            # 假设每个编码器输出的向量维度为 final_dim // 2
+            embed_dim = channels * 2
+            self.attn = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=4 if gf else 2, batch_first=True)
+            
+        if self.DIR:
+            self.FDS = FDS(final_dim)
+
+        self.fc = nn.Sequential(
+            nn.Linear(final_dim, 128),
+            nn.Mish(),
+            nn.Dropout(0.3),
+            nn.Linear(128, 64),
+            nn.Mish(),
+            nn.Dropout(0.3),
+            nn.Linear(64, self.classes)
+        )
+
+    def norm(self, x, dim=-1, p=2):
+        return F.normalize(x, p=p, dim=dim)
+
+    def forward(self, x, labels=None, epoch=0):
+        if self.gf:
+            seq1, seq2, gf = x
+        else:
+            seq1, seq2 = x
+        fusion = []
+
+        # 获取两个序列的编码结果
+        if self.q_encoder.__class__.__name__ in ['LSTM', 'GRU']:
+            # 对于 LSTM, 取序列最后时刻的输出，其维度应为 channels*2 (bidirectional)
+            fusion.append(self.norm(self.q_encoder(seq1)[0][:, -1, :]))
+            fusion.append(self.norm(self.q_encoder_2(seq2)[0][:, -1, :]))
+        # elif self.q_encoder.__class__.__name__ in ['MambaModel', 'MLAModel', 'MHAModel']:
+        else:
+            fusion.append(self.norm(self.q_encoder(seq1)))
+            fusion.append(self.norm(self.q_encoder_2(seq2)))
+        
+        if self.gf:
+            fusion.append(self.g_encoder(gf))
+
+        # 根据 fusion_method 决定融合方式
+        if self.fusion_method == 'mlp':
+            # 维持原有行为：拼接两个向量
+            fusion = torch.cat(fusion, dim=-1)
+        elif self.fusion_method == 'diff':
+            fusion = torch.cat([fusion[1] - fusion[0]] + fusion[2:], dim=-1)
+        elif self.fusion_method == 'att':
+            # 使用 attention 融合:
+            # 先将两个向量堆叠成“tokens”，形状：(batch, 2, embed_dim)
+            tokens = torch.stack(fusion, dim=1)  # embed_dim 应该为 final_dim//2
+            # 利用 MultiheadAttention 进行自注意力计算
+            # 注意：因为采用 batch_first=True，所以输入形状为 (batch, seq_len, embed_dim)
+            attn_output, _ = self.attn(tokens, tokens, tokens)
+            # 将 attention 输出展平，得到形状 (batch, 2 * embed_dim)，即 (batch, final_dim)
+            fusion = attn_output.reshape(attn_output.size(0), -1)
+        else:
+            raise ValueError("Invalid fusion method: choose either 'mse' or 'att'.")
+
+        # 如果启用 DIR 模块，保留传入 FDS 前的特征表示
+        if self.DIR:
+            features = fusion
+            fusion = self.FDS.smooth(fusion, labels, epoch)
+        
+        pred = self.fc(fusion).squeeze(-1)
+
+        if self.DIR:
+            return pred, features
+        else:
+            return pred
+
+
+class CNNEncoder(nn.Module):
+    def __init__(self, feature_dim=256, base_channels=16, in_dim=3):
+        """
+        feature_dim: 输出的一维特征向量维度
+        base_channels: 基础卷积模块的通道数
+        """
+        super(CNNEncoder, self).__init__()
+        
+        # 卷积层
+        self.conv = nn.Sequential(
+            nn.Conv2d(in_dim, base_channels, kernel_size=3, stride=1, padding=1),
+            nn.BatchNorm2d(base_channels),
+            # nn.ReLU(inplace=True),
+            nn.Mish(inplace=True),
+            nn.MaxPool2d(kernel_size=2),
+
+            nn.Conv2d(base_channels, base_channels * 2, kernel_size=3, stride=1, padding=1),
+            nn.BatchNorm2d(base_channels * 2),
+            # nn.ReLU(inplace=True),
+            nn.Mish(inplace=True),
+            nn.MaxPool2d(kernel_size=2),
+            
+            nn.Conv2d(base_channels * 2, base_channels * 4, kernel_size=3, stride=1, padding=1),
+            nn.BatchNorm2d(base_channels * 4),
+            # nn.ReLU(inplace=True),
+            nn.Mish(inplace=True),
+            nn.MaxPool2d(kernel_size=2)
+        )
+        
+        # 自适应池化，得到固定尺寸（1x1）的特征图
+        self.adaptive_pool = nn.AdaptiveAvgPool2d((1, 1))
+        
+        # 全连接层将卷积特征转换为一维特征向量
+        self.fc = nn.Linear(base_channels * 4, feature_dim)
+        
+    def forward(self, img):
+        """
+        img: [B, 3, 1024, 1024] 输入的 RGB 图像张量
+        """
+        # 融合后进一步进行卷积、池化处理
+        fused_conv = self.conv(img)
+        pooled = self.adaptive_pool(fused_conv)  # [B, base_channels*4, 1, 1]
+        
+        # 展平并经过全连接层输出特征向量
+        flattened = pooled.view(pooled.size(0), -1)  # [B, base_channels*4]
+        feature_vector = self.fc(flattened)          # [B, feature_dim]
+        return feature_vector
+
+
+class DMutaPeptideCNN(nn.Module):
+    def __init__(self, q_encoder='cnn', classes=1, channels=16, dir=False, gf=False, side_enc=None, fusion='mlp', non_siamese=False):
+        """
+        参数:
+            q_encoder: 使用的编码器类型，支持 'lstm', 'mamba', 'mla', 'mha'
+            classes: 输出类别数
+            channels: 通道数量，影响隐藏状态维度
+            dir: 是否使用 DIR 模块
+            fusion: 融合方法，可选 'mlp'（默认，直接拼接）或 'att'（使用 attention 融合）
+        """
+        super().__init__()
+        self.classes = classes
+        self.DIR = dir
+        self.gf = gf
+        self.fusion_method = fusion  # 融合方式
+        self.non_siamese = non_siamese
+        # 拼接后维度设定为 channels * 4
+        vector_dim = 512
+        final_dim = vector_dim * 2
+
+        # 初始化编码器
+        if q_encoder == 'cnn':
+            self.q_encoder = CNNEncoder(feature_dim=vector_dim, base_channels=channels)
+        elif q_encoder == 'rn18':
+            self.q_encoder = resnet18_backbone(pretrained=True)
+        if non_siamese:
+            self.q_encoder_2 = deepcopy(self.q_encoder)
+        else:
+            self.q_encoder_2 = self.q_encoder
+
+        if side_enc:
+            self.side_enc = True
+            if side_enc == 'lstm':
+                self.side_encoder = nn.LSTM(
+                    input_size=21,
+                    hidden_size=256,
+                    num_layers=2,
+                    batch_first=True,  # 输入和输出均以 (batch, time_step, input_size) 表示
+                    dropout=0.1,
+                    bidirectional=True
+                )
+            elif side_enc == 'mamba':
+                self.side_encoder = MambaModel(256, 30)
+            else:
+                raise NotImplementedError
+            
+            final_dim += vector_dim * 2
+
+            if non_siamese:
+                self.side_encoder_2 = deepcopy(self.side_encoder)
+            else:
+                self.side_encoder_2 = self.side_encoder
+        else:
+            self.side_enc = False
+        
+        if self.fusion_method == 'diff':
+            final_dim //= 2
+        
+        if gf:
+            self.g_encoder = MLP(1024, [512, 256, 128], vector_dim, dropout_rate=0.3)
+            final_dim += vector_dim
+
+        # 如果 fusion 模式为 'att' ，则使用 MultiheadAttention 对两个向量进行融合
+        if self.fusion_method == 'att':
+            # 假设每个编码器输出的向量维度为 final_dim // 2
+            embed_dim = vector_dim
+            self.attn = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=4 if gf else 2, batch_first=True)
+  
+        if self.DIR:
+            self.FDS = FDS(final_dim)
+
+        self.fc = nn.Sequential(
+            nn.Linear(final_dim, 128),
+            nn.Mish(),
+            nn.Dropout(0.3),
+            nn.Linear(128, 64),
+            nn.Mish(),
+            nn.Dropout(0.3),
+            nn.Linear(64, self.classes)
+        )
+
+    def norm(self, x, dim=-1, p=2):
+        return F.normalize(x, p=p, dim=dim)
+
+    def forward(self, x, labels=None, epoch=0):
+        if self.gf:
+            seq1, seq2, gf = x
+        else:
+            seq1, seq2 = x
+
+        if self.side_enc:
+            seq1_seq = seq1[1]
+            seq1 = seq1[0]
+            seq2_seq = seq2[1]
+            seq2 = seq2[0]
+
+        fusion = []
+
+        # 获取两个序列的编码结果
+        fusion.append(self.norm(self.q_encoder(seq1)))
+        fusion.append(self.norm(self.q_encoder_2(seq2)))
+        if self.side_enc:
+            if self.side_encoder.__class__.__name__ == 'MambaModel':
+                fusion.append(self.norm(self.side_encoder(seq1_seq)))
+                fusion.append(self.norm(self.side_encoder_2(seq2_seq)))
+            # elif self.side_encoder.__class__.__name__ == 'LSTM':
+            else:
+                fusion.append(self.norm(self.side_encoder(seq1_seq)[0][:, -1, :]))
+                fusion.append(self.norm(self.side_encoder_2(seq2_seq)[0][:, -1, :]))
+        
+        if self.gf:
+            fusion.append(self.g_encoder(gf))
+
+        # 根据 fusion_method 决定融合方式
+        if self.fusion_method == 'mlp':
+            # 维持原有行为：拼接两个向量
+            fusion = torch.cat(fusion, dim=-1)
+        elif self.fusion_method == 'diff':
+            if not self.side_enc:
+                fusion = torch.cat([fusion[1] - fusion[0]] + fusion[2:], dim=-1)
+            else:
+                fusion = torch.cat([fusion[1] - fusion[0], fusion[3] - fusion[2]] + fusion[4:], dim=-1)
+        elif self.fusion_method == 'att':
+            # 使用 attention 融合:
+            # 先将两个向量堆叠成“tokens”，形状：(batch, 2, embed_dim)
+            tokens = torch.stack(fusion, dim=1)  # embed_dim 应该为 final_dim//2
+            # 利用 MultiheadAttention 进行自注意力计算
+            # 注意：因为采用 batch_first=True，所以输入形状为 (batch, seq_len, embed_dim)
+            attn_output, _ = self.attn(tokens, tokens, tokens)
+            # 将 attention 输出展平，得到形状 (batch, 2 * embed_dim)，即 (batch, final_dim)
+            fusion = attn_output.reshape(attn_output.size(0), -1)
+        else:
+            raise ValueError("Invalid fusion method: choose either 'mse' or 'att'.")
+
+        # 如果启用 DIR 模块，保留传入 FDS 前的特征表示
+        if self.DIR:
+            features = fusion
+            fusion = self.FDS.smooth(fusion, labels, epoch)
+        
+        pred = self.fc(fusion).squeeze(-1)
+
+        if self.DIR:
+            return pred, features
+        else:
+            return pred
+        
+
+def resnet18_backbone(pretrained=False):
+    weights = None
+    if pretrained:
+        weights = 'IMAGENET1K_V1'
+    model = resnet18(weights=weights, progress=False)
+    return torch.nn.Sequential(*list(model.children())[:-1], nn.Flatten())
+
+
+if __name__ == "__main__":
+    model = resnet18_backbone(pretrained=True)
+    print(model)
+    pass

requirements.txt

@@ -0,0 +1,9 @@
+mamba_ssm==2.2.4
+numpy==1.26.3
+pandas==2.1.4
+rdkit==2024.3.5
+scikit_learn==1.4.1.post1
+scipy==1.13.0
+torch==2.2.0
+torchmetrics==1.3.1
+torchvision==0.17.0