Upload 3 files

detree/utils/dataset.py

@@ -0,0 +1,424 @@
+import json
+import math
+import os
+import random
+import torch
+from torch.utils.data import Dataset
+from .adversarial.alter_number import AlterNumbersAttack
+from .adversarial.alternative_spelling import AlternativeSpellingAttack
+from .adversarial.article_deletion import ArticleDeletionAttack
+from .adversarial.homoglyph import HomoglyphAttack
+from .adversarial.insert_paragraphs import InsertParagraphsAttack
+from .adversarial.misspelling import MisspellingAttack
+from .adversarial.upper_lower import UpperLowerFlipAttack
+from .adversarial.whitespace import WhiteSpaceAttack
+from .adversarial.zero_width_space import ZeroWidthSpaceAttack
+
+model_alias_mapping = {
+    'chatgpt': 'chatgpt',
+    'ChatGPT': 'chatgpt',
+    'chatGPT': 'chatgpt',
+    'gpt-3.5-trubo': 'gpt-3.5-trubo',
+    'GPT4': 'gpt4',
+    'gpt4': 'gpt4',
+    'text-davinci-002': 'text-davinci-002',
+    'text-davinci-003': 'text-davinci-003',
+    'davinci': 'text-davinci',
+    'gpt1': 'gpt1',
+    'gpt2_pytorch': 'gpt2-pytorch',
+    'gpt2_large': 'gpt2-large',
+    'gpt2_small': 'gpt2-small',
+    'gpt2_medium': 'gpt2-medium',
+    'gpt2-xl': 'gpt2-xl',
+    'GPT2-XL': 'gpt2-xl',
+    'gpt2_xl': 'gpt2-xl',
+    'gpt2': 'gpt2-xl',
+    'gpt3': 'gpt3',
+    'GROVER_base': 'grover_base',
+    'grover_base': 'grover_base',
+    'grover_large': 'grover_large',
+    'grover_mega': 'grover_mega',
+    'llama2-fine-tuned': 'llama2',
+    'opt_125m': 'opt_125m',
+    'opt_1.3b': 'opt_1.3b',
+    'opt_2.7b': 'opt_2.7b',
+    'opt_6.7b': 'opt_6.7b',
+    'opt_13b': 'opt_13b',
+    'opt_30b': 'opt_30b',
+    'opt_350m': 'opt_350m',
+    'opt_iml_max_1.3b': 'opt_iml_max_1.3b',
+    'opt_iml_30b': 'opt_iml_30b',
+    'flan_t5_small': 'flan_t5_small',
+    'flan_t5_base': 'flan_t5_base',
+    'flan_t5_large': 'flan_t5_large',
+    'flan_t5_xl': 'flan_t5_xl',
+    'flan_t5_xxl': 'flan_t5_xxl',
+    'flan_t5': 'flan_t5_xxl',
+    'dolly': 'dolly',
+    'GLM130B': 'GLM130B',
+    'bloom_7b': 'bloom_7b',
+    'bloomz': 'bloomz',
+    't0_3b': 't0_3b',
+    't0_11b': 't0_11b',
+    'gpt_neox': 'gpt_neox',
+    'xlm': 'xlm',
+    'xlnet_large': 'xlnet_large',
+    'xlnet_base': 'xlnet_base',
+    'cohere': 'cohere',
+    'ctrl': 'ctrl',
+    'pplm_gpt2': 'pplm_gpt2',
+    'pplm_distil': 'pplm_distil',
+    'fair_wmt19': 'fair_wmt19',
+    'fair_wmt20': 'fair_wmt20',
+    'glm130b': 'GLM130B',
+    'jais-30b': 'jais',
+    'transfo_xl': 'transfo_xl',
+    '7B': '7B',
+    '13B': '13B',
+    '65B': '65B',
+    '30B': '30B',
+    'gpt_j': 'gpt_j',
+    'mpt': 'mpt',
+    'mpt-chat': 'mpt-chat',
+    'llama-chat': 'llama-chat',
+    'mistral': 'mistral',
+    'mistral-chat': 'mistral-chat',
+    'cohere-chat': 'cohere-chat',
+    'human': 'human',
+}
+
+
+def load_datapath(path,include_adversarial=False,dataset_name='all',include_attack=False):
+    data_path = {'train':[],'test':[]}
+    if dataset_name=='all':
+        datasets = os.listdir(path)
+    elif dataset_name=='M4':
+        datasets = ['M4_monolingual','M4_multilingual']
+    elif dataset_name=='RAID_all':
+        datasets = ['RAID','RAID_extra']
+    else:
+        datasets = [dataset_name]
+    for dataset in datasets:
+        dataset_path = os.path.join(path,dataset)
+        for adv in os.listdir(dataset_path):
+            if include_adversarial==False and 'no_attack' not in adv:
+                continue
+            if include_attack==False and ('perplexity_attack' in adv or 'synonym' in adv):
+                continue
+            adv_path = os.path.join(dataset_path,adv)
+            for data in os.listdir(adv_path):
+                if 'train' in data:
+                    data_path['train'].append(os.path.join(adv_path,data))
+                elif 'test' in data:
+                    data_path['test'].append(os.path.join(adv_path,data))
+                elif 'valid' in data:
+                    if 'RAID' in dataset:
+                        data_path['test'].append(os.path.join(adv_path,data))
+                    else:
+                        data_path['train'].append(os.path.join(adv_path,data))
+    
+    return data_path
+
+class TreeDataset(Dataset):
+    def __init__(self,data_path,need_ids=False):
+        self.data_path = data_path
+        self.need_ids=need_ids
+        self.dataset = self.load_data(data_path)
+
+        LLM_name=set()
+        for item in self.dataset:
+            name = model_alias_mapping[item['src']]
+            LLM_name.add(name)
+        self.classes = list(LLM_name)
+        self.classes = sorted(self.classes)
+
+        self.name2id={}
+        for i,name in enumerate(self.classes):
+            self.name2id[name]=i
+        self.human_id = self.name2id['human']
+        
+    def load_jsonl(self,file_path):
+        out = []
+        add = ''
+        if 'paraphrase_by_llm' in file_path:
+            add='-paraphrase-qwen7B'
+        elif 'paraphrase' in file_path:
+            add='-paraphrase-dipper'
+        else:
+            assert 'no_attack' in file_path,file_path+'file path should contain no_attack or paraphrase'
+
+        with open(file_path, mode='r', encoding='utf-8') as jsonl_file:
+            for line in jsonl_file:
+                now = json.loads(line)
+                if add != '':
+                    if 'human' in now['src']:
+                        continue
+                    src = model_alias_mapping[now['src']]+add
+                    if src not in model_alias_mapping:
+                        model_alias_mapping[src]=src
+                    now['src']=src
+                out.append(now)
+        return out
+    
+    def load_data(self,data_path):
+        data = []
+        for path in data_path:
+            if 'no_attack' not in path and 'paraphrase' not in path:
+                continue
+            print(f'loading {path}')
+            data+=self.load_jsonl(path)
+        return data
+
+    def __len__(self):
+        return len(self.dataset)
+
+    def __getitem__(self, idx):
+        data_now = self.dataset[idx]
+        text = data_now['text']
+        label = data_now['label']
+        src = model_alias_mapping[data_now['src']]
+        src_id = self.name2id[src]
+        id = data_now['id']
+        if self.need_ids:
+            return text,int(label),int(src_id),int(id)
+        else:
+            return text,int(label),int(src_id)
+
+class SCLDataset(Dataset):
+    def __init__(self, data_path,fabric,tokenizer,need_ids=False,adv_p=0.5,max_length=530,name2id=None,has_mix=True):
+        self.data_path = data_path
+        self.adv_p = adv_p
+        self.need_ids=need_ids
+        self.tokenizer = tokenizer
+        self.max_length = max_length
+        self.has_mix = has_mix
+
+        self.world_size = fabric.world_size
+        self.global_rank = fabric.global_rank
+        self.LLM_name=set()
+        dataset_len = self.get_data_len(data_path)
+        
+        classes = sorted(list(self.LLM_name))
+        if name2id is None:
+            self.name2id={}
+            for i,name in enumerate(classes):
+                self.name2id[name]=i
+        else:
+            self.name2id = name2id
+            for name in classes:
+                assert name in self.name2id
+        self.classes = classes
+        print(f'there are {len(classes)} classes in dataset')
+        print(f'the classes are {classes}')
+
+        self.num_samples = math.ceil(dataset_len / self.world_size)
+        total_size = self.num_samples * self.world_size
+        indices = list(range(dataset_len))
+        padding_size = total_size - len(indices)
+        indices += indices[:padding_size]
+        assert len(indices) == total_size
+        indices = indices[self.global_rank : total_size : self.world_size]
+        assert len(indices) == self.num_samples
+        self.indices = set(indices)
+
+        data_dict = self.load_data(data_path)
+        self.dataset = [data_dict[i] for i in indices]
+        self.dataset_len = len(self.dataset)
+
+    
+    def get_data_len(self,data_path):
+        total_len = 0
+        for path in data_path:
+            print(f'reading {path}')
+            with open(path, mode='r', encoding='utf-8') as jsonl_file:
+                for line in jsonl_file:
+                    now = json.loads(line)
+                    if now['src'] not in model_alias_mapping:
+                        model_alias_mapping[now['src']]=now['src']
+                    now['src'] = model_alias_mapping[now['src']]
+                    if self.has_mix == False:
+                        if 'human' in now['src'] and now['src'] != 'human':
+                            continue
+                    if now['src'] not in self.LLM_name:
+                        self.LLM_name.add(now['src'])
+                    total_len+=1
+        return total_len
+
+    def truncate_text(self,text):
+    
+        tokens = self.tokenizer.encode(text, truncation=True, max_length=self.max_length)
+        truncated_text = self.tokenizer.decode(tokens, skip_special_tokens=True)
+        return truncated_text
+
+    def merge_dict(self,dict1,dict2):
+        for key in dict2:
+            dict1[key]=dict2[key]
+        return dict1
+
+    def load_jsonl(self,file_path,total_len):
+        out = {}
+        cnt=0
+        with open(file_path, mode='r', encoding='utf-8') as jsonl_file:
+            for line in jsonl_file:
+                now = json.loads(line)
+                if self.has_mix == False:
+                    if 'human' in now['src'] and now['src'] != 'human':
+                        continue
+                if total_len+cnt in self.indices:
+                    out[total_len+cnt]=now
+                cnt+=1
+        return out,cnt
+
+    def load_data(self,data_path):
+        data = {}
+        total_len = 0
+        for path in data_path:
+            print(f'loading {path}')
+            now_data,now_len=self.load_jsonl(path,total_len)
+            data = self.merge_dict(data,now_data)
+            total_len+=now_len
+        return data
+    
+    def __len__(self):
+        return self.dataset_len
+    
+    def __getitem__(self, idx):
+        data = self.dataset[idx]
+        text = data['text']
+        label = data['label']
+        src = self.name2id[model_alias_mapping[data['src']]]
+        id = data['id']
+
+        if random.random()<self.adv_p:
+            text = self.truncate_text(text)
+            attack_method = random.choice([AlterNumbersAttack,AlternativeSpellingAttack,ArticleDeletionAttack,\
+                HomoglyphAttack,InsertParagraphsAttack,MisspellingAttack,UpperLowerFlipAttack,WhiteSpaceAttack,ZeroWidthSpaceAttack])
+            text = attack_method(text)
+        if self.need_ids:
+            return text,int(label),int(src),int(id)
+        return text,int(label),int(src)
+
+
+class SCL_RM_Dataset(Dataset):
+    def __init__(self, data_path,fabric,tokenizer,need_ids=False,adv_p=0.5,max_length=530,name2id=None,has_mix=True,remove_cls=0.9):
+        self.data_path = data_path
+        self.adv_p = adv_p
+        self.need_ids=need_ids
+        self.tokenizer = tokenizer
+        self.max_length = max_length
+        self.has_mix = has_mix
+
+        self.world_size = fabric.world_size
+        self.global_rank = fabric.global_rank
+        self.LLM_name=set()
+        self.remove_cls = remove_cls
+        assert name2id is not None, 'name2id is None, please set name2id'
+        self.remove_name = set()
+        for name in name2id:
+            if random.random()<self.remove_cls and name != 'human':
+                self.remove_name.add(name)
+        dataset_len = self.get_data_len(data_path)
+        
+        classes = sorted(list(self.LLM_name))
+        if name2id is None:
+            self.name2id={}
+            for i,name in enumerate(classes):
+                self.name2id[name]=i
+        else:
+            self.name2id = name2id
+            for name in classes:
+                assert name in self.name2id
+        self.classes = classes
+        print(f'there are {len(classes)} classes in dataset')
+        print(f'the classes are {classes}')
+
+        self.num_samples = math.ceil(dataset_len / self.world_size)
+        total_size = self.num_samples * self.world_size
+        indices = list(range(dataset_len))
+        padding_size = total_size - len(indices)
+        indices += indices[:padding_size]
+        assert len(indices) == total_size
+        indices = indices[self.global_rank : total_size : self.world_size]
+        assert len(indices) == self.num_samples
+        self.indices = set(indices)
+
+        data_dict = self.load_data(data_path)
+        self.dataset = [data_dict[i] for i in indices]
+        self.dataset_len = len(self.dataset)
+
+    
+    def get_data_len(self,data_path):
+        total_len = 0
+        for path in data_path:
+            print(f'reading {path}')
+            with open(path, mode='r', encoding='utf-8') as jsonl_file:
+                for line in jsonl_file:
+                    now = json.loads(line)
+                    if now['src'] not in model_alias_mapping:
+                        model_alias_mapping[now['src']]=now['src']
+                    now['src'] = model_alias_mapping[now['src']]
+                    if self.has_mix == False:
+                        if 'human' in now['src'] and now['src'] != 'human':
+                            continue
+                    if now['src'] in self.remove_name:
+                        continue
+                    if now['src'] not in self.LLM_name:
+                        self.LLM_name.add(now['src'])
+                    total_len+=1
+        return total_len
+
+    def truncate_text(self,text):
+    
+        tokens = self.tokenizer.encode(text, truncation=True, max_length=self.max_length)
+        truncated_text = self.tokenizer.decode(tokens, skip_special_tokens=True)
+        return truncated_text
+
+    def merge_dict(self,dict1,dict2):
+        for key in dict2:
+            dict1[key]=dict2[key]
+        return dict1
+
+    def load_jsonl(self,file_path,total_len):
+        out = {}
+        cnt=0
+        with open(file_path, mode='r', encoding='utf-8') as jsonl_file:
+            for line in jsonl_file:
+                now = json.loads(line)
+                if self.has_mix == False:
+                    if 'human' in now['src'] and now['src'] != 'human':
+                        continue
+                if now['src'] in self.remove_name:
+                        continue
+                if total_len+cnt in self.indices:
+                    out[total_len+cnt]=now
+                cnt+=1
+        return out,cnt
+
+    def load_data(self,data_path):
+        data = {}
+        total_len = 0
+        for path in data_path:
+            print(f'loading {path}')
+            now_data,now_len=self.load_jsonl(path,total_len)
+            data = self.merge_dict(data,now_data)
+            total_len+=now_len
+        return data
+    
+    def __len__(self):
+        return self.dataset_len
+    
+    def __getitem__(self, idx):
+        data = self.dataset[idx]
+        text = data['text']
+        label = data['label']
+        src = self.name2id[model_alias_mapping[data['src']]]
+        id = data['id']
+
+        if random.random()<self.adv_p:
+            text = self.truncate_text(text)
+            attack_method = random.choice([AlterNumbersAttack,AlternativeSpellingAttack,ArticleDeletionAttack,\
+                HomoglyphAttack,InsertParagraphsAttack,MisspellingAttack,UpperLowerFlipAttack,WhiteSpaceAttack,ZeroWidthSpaceAttack])
+            text = attack_method(text)
+        if self.need_ids:
+            return text,int(label),int(src),int(id)
+        return text,int(label),int(src)

detree/utils/index.py

@@ -0,0 +1,105 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+# All rights reserved.
+# 
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import os
+import pickle
+from typing import List, Tuple
+
+import faiss
+import numpy as np
+from tqdm import tqdm
+
+class Indexer(object):
+
+    def __init__(self, vector_sz, n_subquantizers=0, n_bits=16):
+        # if n_subquantizers > 0:
+        #     self.index = faiss.IndexPQ(vector_sz, n_subquantizers, n_bits, faiss.METRIC_INNER_PRODUCT)
+        # else:
+        self.vector_sz = vector_sz
+        self.index = self._create_sharded_index()
+        self.index_id_to_db_id = []
+        self.label_dict = {}
+        # self.index = faiss.IndexFlatIP(vector_sz)
+
+        # self.index = faiss.index_cpu_to_all_gpus(self.index)
+        # #self.index_id_to_db_id = np.empty((0), dtype=np.int64)
+        # self.index_id_to_db_id = []
+        # self.label_dict = {}
+
+    def _create_sharded_index(self):
+        # Determine the number of available GPUs
+        ngpu = faiss.get_num_gpus()
+        # Create an IndexShards object with successive_ids=True to keep ids globally unique
+        index = faiss.IndexShards(self.vector_sz, True, True)
+        # Create a sub-index for each GPU and add it to the IndexShards container
+        for i in range(ngpu):
+            # Create a standard GPU resource object
+            res = faiss.StandardGpuResources()
+            # Configure the GPU index
+            flat_config = faiss.GpuIndexFlatConfig()
+            # flat_config.useFloat16 = True  # enable to reduce memory usage with half precision
+            flat_config.device = i  # assign the GPU device id
+            # Create the GPU index
+            sub_index = faiss.GpuIndexFlatIP(res, self.vector_sz, flat_config)
+            # Add the sub-index into the sharded index
+            index.add_shard(sub_index)
+        return index
+
+    def index_data(self, ids, embeddings):
+        self._update_id_mapping(ids)
+        # embeddings = embeddings
+        # if not self.index.is_trained:
+        #     self.index.train(embeddings)
+        self.index.add(embeddings)
+
+        print(f'Total data indexed {self.index.ntotal}')
+
+    def search_knn(self, query_vectors: np.array, top_docs: int, index_batch_size: int = 8) -> List[Tuple[List[object], List[float]]]:
+        # query_vectors = query_vectors
+        result = []
+        nbatch = (len(query_vectors)-1) // index_batch_size + 1
+        for k in tqdm(range(nbatch)):
+            start_idx = k*index_batch_size
+            end_idx = min((k+1)*index_batch_size, len(query_vectors))
+            q = query_vectors[start_idx: end_idx]
+            scores, indexes = self.index.search(q, top_docs)
+            # convert to external ids
+            db_ids = [[str(self.index_id_to_db_id[i]) for i in query_top_idxs] for query_top_idxs in indexes]
+            db_labels = [[self.label_dict[self.index_id_to_db_id[i]] for i in query_top_idxs] for query_top_idxs in indexes]
+            result.extend([(db_ids[i], scores[i],db_labels[i]) for i in range(len(db_ids))])
+        return result
+
+    def serialize(self, dir_path):
+        index_file = os.path.join(dir_path, 'index.faiss')
+        meta_file = os.path.join(dir_path, 'index_meta.faiss')
+        print(f'Serializing index to {index_file}, meta data to {meta_file}')
+
+        faiss.write_index(self.index, index_file)
+        with open(meta_file, mode='wb') as f:
+            pickle.dump(self.index_id_to_db_id, f)
+
+    def deserialize_from(self, dir_path):
+        index_file = os.path.join(dir_path, 'index.faiss')
+        meta_file = os.path.join(dir_path, 'index_meta.faiss')
+        print(f'Loading index from {index_file}, meta data from {meta_file}')
+
+        self.index = faiss.read_index(index_file)
+        print('Loaded index of type %s and size %d', type(self.index), self.index.ntotal)
+
+        with open(meta_file, "rb") as reader:
+            self.index_id_to_db_id = pickle.load(reader)
+        assert len(
+            self.index_id_to_db_id) == self.index.ntotal, 'Deserialized index_id_to_db_id should match faiss index size'
+
+    def _update_id_mapping(self, db_ids: List):
+        #new_ids = np.array(db_ids, dtype=np.int64)
+        #self.index_id_to_db_id = np.concatenate((self.index_id_to_db_id, new_ids), axis=0)
+        self.index_id_to_db_id.extend(db_ids)
+
+    def reset(self):
+        self.index.reset()
+        self.index_id_to_db_id = []
+        print(f'Index reset, total data indexed {self.index.ntotal}')

detree/utils/utils.py

@@ -0,0 +1,251 @@
+import hashlib
+import os
+import pickle
+import numpy as np
+from sklearn.metrics import precision_recall_curve, auc, roc_auc_score,roc_curve
+
+def stable_long_hash(input_string):
+    hash_object = hashlib.sha256(input_string.encode())
+    hex_digest = hash_object.hexdigest()
+    int_hash = int(hex_digest, 16)
+    long_long_hash = (int_hash & ((1 << 63) - 1))
+    return long_long_hash
+
+def load_pkl(path):
+    with open(path, 'rb') as f:
+        return pickle.load(f)
+
+
+def save_pkl(obj, path):
+    with open(path, 'wb') as f:
+        pickle.dump(obj, f)
+
+
+
+def find_top_n(embeddings,n,index,data):
+    if len(embeddings.shape) == 1:
+        embeddings = embeddings.reshape(1, -1)
+    top_ids_and_scores = index.search_knn(embeddings, n)
+    data_ans=[]
+    for i, (ids, scores) in enumerate(top_ids_and_scores):
+        data_now=[]
+        for id in ids:
+            data_now.append((data[0][int(id)],data[1][int(id)],data[2][int(id)]))
+        data_ans.append(data_now)
+    return data_ans
+
+
+    
+from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
+
+def print_line(class_name, metrics, is_header=False):
+    if is_header:
+        line = f"| {'Class':<10} | " + " | ".join([f"{metric:<10}" for metric in metrics])
+    else:
+        line = f"| {class_name:<10} | " + " | ".join([f"{metrics[metric]:<10.3f}" for metric in metrics])
+    print(line)
+    if is_header:
+        print('-' * len(line))
+
+def calculate_per_class_metrics(classes, ground_truth, predictions):
+    # Convert ground truth and predictions to numeric format
+    gt_numeric = np.array([int(gt) for gt in ground_truth])
+    pred_numeric = np.array([int(pred) for pred in predictions])
+
+    results = {}
+    for i, class_name in enumerate(classes):
+        # For each class, calculate the 'vs rest' binary labels
+        gt_binary = (gt_numeric == i).astype(int)
+        pred_binary = (pred_numeric == i).astype(int)
+
+        # Calculate metrics, handling cases where a class is not present in predictions or ground truth
+        precision = precision_score(gt_binary, pred_binary, zero_division=0)
+        recall = recall_score(gt_binary, pred_binary, zero_division=0)
+        f1 = f1_score(gt_binary, pred_binary, zero_division=0)
+        acc = np.mean(gt_binary == pred_binary)
+        # Calculate recall for all other classes as 'rest'
+        rest_recall = recall_score(1 - gt_binary, 1 - pred_binary, zero_division=0)
+
+        results[class_name] = {
+            'Precision': precision,
+            'Recall': recall,
+            'F1 Score': f1,
+            'Accuracy': acc,
+            'Avg Recall (with rest)': (recall + rest_recall) / 2
+        }
+
+    print_line("Metric", results[classes[0]], is_header=True)
+    for class_name, metrics in results.items():
+        print_line(class_name, metrics)
+    overall_metrics = {metric_name: np.mean([metrics[metric_name] for metrics in results.values()]) for metric_name in results[classes[0]].keys()}
+    print_line("Overall", overall_metrics)
+
+def calculate_metrics(labels, preds):
+    acc = accuracy_score(labels, preds)
+    precision = precision_score(labels, preds, average='macro')
+    recall = recall_score(labels, preds, average='macro')
+    f1 = f1_score(labels, preds, average='macro')
+    return acc, precision, recall, f1
+
+def compute_three_recalls(labels, preds):
+    all_n, all_p, tn, tp = 0, 0, 0, 0
+    for label, pred in zip(labels, preds):
+        if label == '0':
+            all_p += 1
+        if label == '1':
+            all_n += 1
+        # Modified condition to treat None in preds as incorrect prediction
+        if pred is not None and label == pred == '0':
+            tp += 1 
+        # Modified condition to treat None in preds as incorrect prediction
+        if pred is not None and label == pred == '1':
+            tn += 1
+        if pred is None:
+            continue
+    machine_rec , human_rec= tp * 100 / all_p if all_p != 0 else 0, tn * 100 / all_n if all_n != 0 else 0
+    avg_rec = (human_rec + machine_rec) / 2
+    return (human_rec, machine_rec, avg_rec)
+
+
+def compute_metrics(labels, preds,ids=None):
+    # Handling None values in preds as incorrect predictions
+    #preds = ['0' if pred is None else pred for pred in preds]
+    if ids is not None:
+        # Deduplicate labels and predictions for repeated ids
+        dict_labels,dict_preds={},{}
+        for i in range(len(ids)):
+            dict_labels[ids[i]]=labels[i]
+            dict_preds[ids[i]]=preds[i] 
+        labels=list(dict_labels.values())
+        preds=list(dict_preds.values())
+    
+    human_rec, machine_rec, avg_rec = compute_three_recalls(labels, preds)
+    acc = accuracy_score(labels, preds)
+    precision = precision_score(labels, preds, pos_label='1')
+    recall = recall_score(labels, preds, pos_label='1')
+    f1 = f1_score(labels, preds, pos_label='1')
+    # return human_rec, machine_rec, avg_rec
+    return (human_rec, machine_rec, avg_rec, acc, precision, recall, f1)
+
+def evaluate_max_f1_metrics(test_labels, y_score):
+    test_labels = np.array(test_labels)
+    y_score = np.array(y_score)
+
+    auroc = roc_auc_score(test_labels, y_score)
+    precision, recall, thresholds = precision_recall_curve(test_labels, y_score, pos_label=1)
+    pr_auc = auc(recall, precision)
+    epsilon = 1e-6
+    f1_scores = 2 * precision * recall / (precision + recall+epsilon)
+    best_index = f1_scores.argmax()
+    best_f1 = f1_scores[best_index]
+    best_precision = precision[best_index]
+    best_recall = recall[best_index]
+    
+    threshold = thresholds[best_index] if best_index < len(thresholds) else 1.0
+    y_pred_max_f1 = (y_score >= threshold).astype(int)
+    
+    acc = (y_pred_max_f1 == test_labels).mean()
+    tp = sum((y_pred_max_f1 == 1) & (test_labels == 1))
+    fn = sum((y_pred_max_f1 == 0) & (test_labels == 1))
+    fp = sum((y_pred_max_f1 == 1) & (test_labels == 0))
+    tn = sum((y_pred_max_f1 == 0) & (test_labels == 0))
+
+    pos_recall = tp / (tp + fn + epsilon)  # recall for the positive class
+    neg_recall = tn / (tn + fp + epsilon)  # recall for the negative class
+    avg_recall = (pos_recall + neg_recall) / 2  # average recall across classes
+    
+    metric = {'auroc': auroc, 'pr_auc': pr_auc, 'F1': best_f1, 'Precision': best_precision,\
+               'Recall': best_recall, 'threshold': threshold, 'acc': acc, 'avg_recall': avg_recall,\
+                  'pos_recall': pos_recall, 'neg_recall': neg_recall}
+    return metric
+
+def evaluate_metrics(test_labels, y_score, threshold_param=-1,target_fpr = 0.05):
+    if isinstance(test_labels, list):
+        test_labels = np.array(test_labels)
+    if isinstance(y_score, list):
+        y_score = np.array(y_score)
+
+    if threshold_param != -1:
+        if not (0 <= threshold_param <= 1):
+            raise ValueError("Threshold must be between 0 and 1.")
+
+    auroc = roc_auc_score(test_labels, y_score)
+
+    precision, recall, thresholds = precision_recall_curve(test_labels, y_score, pos_label=1)
+    pr_auc = auc(recall, precision)
+
+    epsilon = 1e-6
+    f1_scores = 2 * precision * recall / (precision + recall + epsilon)
+    
+
+    if threshold_param == -1:
+        best_index = f1_scores.argmax()
+        F1 = f1_scores[best_index]
+        Precision = precision[best_index]
+        Recall = recall[best_index]
+        threshold = thresholds[best_index] if best_index < len(thresholds) else 1.0
+    else:
+        threshold = threshold_param
+        index = np.where(thresholds >= threshold)[0][0]
+        Precision = precision[index]
+        Recall = recall[index]
+        F1 = f1_scores[index]
+
+
+    y_pred = (y_score >= threshold).astype(int)
+    acc = (y_pred == test_labels).mean()
+    
+    tp = ((y_pred == 1) & (test_labels == 1)).sum()
+    fn = ((y_pred == 0) & (test_labels == 1)).sum()
+    fp = ((y_pred == 1) & (test_labels == 0)).sum()
+    tn = ((y_pred == 0) & (test_labels == 0)).sum()
+
+    pos_recall = tp / (tp + fn + epsilon)  # TPR
+    neg_recall = tn / (tn + fp + epsilon)  # TNR
+    avg_recall = (pos_recall + neg_recall) / 2
+
+    fpr, tpr, thds = roc_curve(test_labels, y_score)
+    if len(fpr) > 0 and len(tpr) > 0:
+        idx = np.argmin(np.abs(fpr - target_fpr))
+        tpr_at_fpr = tpr[idx]
+        tpr_at_fpr_threshold = thds[idx]
+    else:
+        tpr_at_fpr = 0.0
+
+    metric = {'auroc': auroc, 'pr_auc': pr_auc, 'F1': F1, 'Precision': Precision,'Recall': Recall,\
+               'threshold': threshold, 'acc': acc, 'avg_recall': avg_recall,'pos_recall': pos_recall,\
+                  'neg_recall': neg_recall, 'tpr_at_fpr': tpr_at_fpr, 'tpr_at_fpr_threshold': tpr_at_fpr_threshold}
+    
+    return metric
+    # return (auroc, pr_auc, best_f1, best_precision, best_recall, threshold,
+    #         acc, avg_recall, pos_recall, neg_recall, tpr_at_fpr5)
+
+
+def load_datapath(path,include_adversarial=False,dataset_name='all',attack_type='all'):
+    data_path = {'train':[],'valid':[],'test':[]}
+    if dataset_name=='all':
+        datasets = os.listdir(path)
+    elif dataset_name=='M4':
+        datasets = ['M4_monolingual','M4_multilingual']
+    elif dataset_name=='RAID_all':
+        datasets = ['RAID','RAID_extra']
+    else:
+        datasets = [dataset_name]
+    for dataset in datasets:
+        dataset_path = os.path.join(path,dataset)
+        if attack_type!='all':
+            dataset_path_list = [pth for pth in os.listdir(dataset_path) if attack_type in pth]
+        else:
+            dataset_path_list = os.listdir(dataset_path)
+        for adv in dataset_path_list:
+            if include_adversarial==False and 'no_attack' not in adv:
+                continue
+            adv_path = os.path.join(dataset_path,adv)
+            for data in os.listdir(adv_path):
+                if 'train.' in data:
+                    data_path['train'].append(os.path.join(adv_path,data))
+                elif 'test.' in data:
+                    data_path['test'].append(os.path.join(adv_path,data))
+                elif 'valid.' in data:
+                    data_path['valid'].append(os.path.join(adv_path,data))
+    return data_path