Clean Push to Hugging Face

README.md

@@ -0,0 +1,70 @@
+---
+title: DATE-LM Leaderboard
+emoji: 🏆
+colorFrom: purple
+colorTo: blue
+sdk: gradio
+sdk_version: 5.23.1
+app_file: app.py
+pinned: false
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+# DATE-LM Data Attribution Leaderboards
+
+This repo contains the leaderboard code associated with the DATE-LM Paper. The leaderboards
+are hosted in [this HuggingFace Space](https://huggingface.co/spaces/DataAttributionEval/DATE-LM-Leaderboard).
+
+The leaderboards are split into 2 broad categories: Training Data Selection and Applications.
+Each category contains 3 leaderboards, as indicated below.
+
+- Pre-Training (10K)
+- Pre-Training (30K)
+- Fine-Tuning
+
+| Category                           | Leaderboards                                        |
+| ---------------------------------- | --------------------------------------------------- |
+| Training Data Selection            | Pre-Training (10K), Pre-Training (30K), Fine-Tuning |
+| Applications (Toxicity / Bias)     | Homogeneous, Heterogeneous                          |
+| Applications (Factual Attribution) | Factual Attribution                                 |
+
+Details on the tasks corresponding to each leaderboard as well as their code pipelines
+can be found in the DATE-LM paper and [Github repo](https://github.com/DataAttributionEval/DATE-LM).
+
+## Submission
+
+To submit to the leaderboard: submit via the form in the "Submit Scores" tab on the HuggingFace Space page. This will open up a pull request in this repo. It will need to be merged by a member of the team in order to be displayed in the HuggingFace Space.
+
+Information for Submission include:
+
+- Influence Scores File
+- Paper/Code/Contact Link
+- Method Name and Category
+- Metrics (dependent on leaderboard chosen)
+- and more
+
+## Ranking
+
+Each leaderboard's ranking is based on the values from the metrics, with details specified in the description of each leaderboard. To summarize, the leaderboards are ranked using the following schemes:
+
+| Leaderboard                            | Ranking Metric                                     |
+| -------------------------------------- | -------------------------------------------------- |
+| Pre-Training (10K), Pre-Training (30K) | highest score in **avg** column                    |
+| Fine-Tuning                            | average of **MMLU**, **GSM8K**, and **BBH** scores |
+| Applications (Toxicity / Bias)         | highest score in **AUPRC** column                  |
+| Applications (Factual Attribution)     | average of **Recall@50** and **MRR** scores        |
+
+## Repo Files
+
+Overview of Repo files:
+
+- app file: `app.py`
+  - mappings files: `filePaths.py`, `nameMapping.py`
+  - Github PR creation file: `pr.py`
+- submissions storage: `submissions` folder
+  - Note: each submission has its own dedicated folder containing `metadata.json` and the influence scores
+- leaderboards data: `data` folder
+- github workflow files:
+  - Add submission into leaderboard json upon merge: `merge-data.yml`, `scripts/merge_data.py`
+  - Sync repo with HuggingFace Space: `push-to-hf.yml`

app.py

@@ -0,0 +1,400 @@
+import gradio as gr
+from gradio_leaderboard import Leaderboard
+import pandas as pd
+import json
+import os
+
+import filePaths 
+import nameMapping
+import pr
+
+##################### Leaderboard Paths + Variables #####################
+
+pathLst = filePaths.PATHLIST
+pretrain_10K, pretrain_30K, finetune = pathLst[0], pathLst[1], pathLst[2]
+toxicity_homogeneous, toxicity_heterogeneous, factual = pathLst[3], pathLst[4], pathLst[5]
+
+import nameMapping
+leaderboard_names = nameMapping.LEADERBOARD_NAMES
+trainingNamesSet = nameMapping.TRAINING_LEADERBOARDS
+
+########################## Data Loading ###########################
+
+def load_leaderboard_data(file_path):
+    """
+    Load leaderboard data from JSON file.
+    """
+    if os.path.exists(file_path):
+        with open(file_path, 'r') as f:
+            return json.load(f)
+    return []
+
+def add_ranking_column(data, id):
+    """
+    Add ranking column dynamically for display based on selected metric aggregation.
+    """
+    if id == 'toxicity': # Toxicity: AUPRC
+        key_fn = lambda x: x["AUPRC"]
+    elif id == 'factual': # Factual: Avg of Recall@50 and MRR
+        key_fn = lambda x: (x["Recall@50"]+x["MRR"])/2
+    elif id == 'pretrain': # Pretrain: Avg
+        key_fn = lambda x: x["avg"]
+    else: # FineTune: Avg of Metrics
+        key_fn = lambda x: (x["MMLU"]+x["GSM8K"]+x["BBH"])/3
+
+    sorted_data = sorted(data, key=key_fn, reverse=True)
+    for index, entry in enumerate(sorted_data):
+        entry["Rank"] = index + 1 
+    return sorted_data
+
+def load_data(filePath, id):
+    """
+    Load initial leaderboard data.
+    """
+    return pd.DataFrame(add_ranking_column(load_leaderboard_data(filePath), id))
+
+pretrain_10K_data = load_data(pretrain_10K, "pretrain")
+pretrain_30K_data = load_data(pretrain_30K, "pretrain")
+finetune_data = load_data(finetune, "finetune")
+homogeneous_data = load_data(toxicity_homogeneous, "toxicity")
+heterogeneous_data = load_data(toxicity_heterogeneous, "toxicity")
+factual_data = load_data(factual, "factual")
+
+########################## Leaderboard Columns + Helpers ###########################
+
+def get_leaderboard_columns(leaderboard_name):
+    """
+    Returns the Expected Columns for Leaderboard
+    """
+    leaderboardNameMap = nameMapping.DROPDOWN_NAME_MAPPING
+    if leaderboard_name in leaderboardNameMap["toxicity"]:
+        return nameMapping.TOXICITY_COLS
+    elif leaderboard_name in leaderboardNameMap["factual"]:
+        return nameMapping.FACTUAL_COLS
+    elif leaderboard_name in leaderboardNameMap["finetune"]:    
+        return nameMapping.FINETUNE_COLS
+    else: # pretrain
+        return nameMapping.PRETRAIN_COLS
+
+def get_model_sizes(leaderboard_name):
+    """
+    Returns Model Sizes for Applications Leaderboards
+    """
+    nameFileMapping = {"Homogeneous": toxicity_homogeneous, 
+                       "Heterogeneous": toxicity_heterogeneous,
+                       "Factual Attribution": factual}
+    leaderboardJson = load_leaderboard_data(nameFileMapping[leaderboard_name])
+    modelSizes = set()
+
+    for row in leaderboardJson:
+        modelSizes.add(row["Model Size"])
+    
+    return ['All'] + list(modelSizes)
+
+################### Submission Helper Functions #############################
+
+def update_fields(leaderboard):
+    """
+    Determine visibility of group / display additional metrics in submission area.
+    """
+    nameMap = nameMapping.DROPDOWN_NAME_MAPPING
+    return {
+        pretrain_group: gr.update(visible=(leaderboard in nameMap['pretrain'])),
+        finetune_group: gr.update(visible=(leaderboard in nameMap['finetune'])),
+        toxicity_group: gr.update(visible=(leaderboard in nameMap['toxicity'])),
+        factual_group: gr.update(visible=(leaderboard in nameMap['factual']))
+    }
+
+def validate_inputs(*inputFields):
+    (leaderboard_dropdown, method_name, method_dropdown, model_name, model_size, paper_link, scores, 
+     pre_avg, pre_sciq, pre_arc_easy, pre_arc_chall, pre_logiqa, 
+     pre_boolq, pre_hellaswag, pre_piqa, pre_wino, pre_open, 
+     fine_mmlu, fine_gsm, fine_bbh, 
+     tox_toxicChat, tox_xsTest, tox_jbb, tox_auprc, 
+     fac_recall, fac_mrr) = inputFields
+    
+    if not all([leaderboard_dropdown, model_name, method_name, method_dropdown, model_size]):
+        raise gr.Error("All fields must be filled out and with the correct type.")
+    
+    if not paper_link:
+        raise gr.Error("Please fill in out the Paper/Code/Contact Link info.")
+    
+    if not scores:
+        raise gr.Error("Please upload data attribution scores in .pt file.")
+    
+    # Check Metrics Non-Empty
+    nameMap = nameMapping.DROPDOWN_NAME_MAPPING
+    # nameMap['pretrain'] nameMap['finetune'] nameMap['finetune'] nameMap['factual']
+    if leaderboard_dropdown in nameMap['pretrain']:
+        metricsList = [pre_avg, pre_sciq, pre_arc_easy, pre_arc_chall, pre_logiqa, pre_boolq, pre_hellaswag, pre_piqa, pre_wino, pre_open]
+    elif leaderboard_dropdown in nameMap['finetune']:
+        metricsList = [fine_mmlu, fine_gsm, fine_bbh]
+    elif leaderboard_dropdown in nameMap['toxicity']:
+        metricsList = [tox_toxicChat, tox_xsTest, tox_jbb, tox_auprc]
+    elif leaderboard_dropdown in nameMap['factual']:
+        metricsList = [fac_recall, fac_mrr]
+
+    if not all(metricsList):
+        raise gr.Error("Metrics must be filled out.")
+    if not all(metric > 0 for metric in metricsList):
+        raise gr.Error("Metrics must be positive.")
+
+    
+######## Dynamically Update Ranking when Filtering on Model Size ###############
+
+def update_rankings(filtered_df, id):
+    df_with_rank = filtered_df.copy() # create copy to avoid modifying original
+    
+    if id == 'toxicity': # Toxicity: AUPRC
+        df_with_rank = df_with_rank.sort_values(by="AUPRC", ascending=False)
+    elif id == 'factual': # Factual: Avg of Recall@50 and MRR
+        average_scores  = df_with_rank[["Recall@50", "MRR"]].mean(axis=1)
+        sorted_index = average_scores .sort_values(ascending=False).index
+        df_with_rank = df_with_rank.loc[sorted_index]
+    
+    df_with_rank["Rank"] = range(1, len(df_with_rank) + 1) # Add rank column
+    
+    return df_with_rank
+
+def filter_and_rank(df, filter_value, id):
+    if filter_value == "All":
+        filtered_df = df
+    else:
+        filtered_df = df[df["Model Size"] == filter_value]
+    return update_rankings(filtered_df, id)
+
+def rerank_leaderboard(filter_value, dfPath, idNum):
+    df = load_data(dfPath, idNum)
+    filtered_ranked_df = filter_and_rank(df, filter_value, idNum)
+    return filtered_ranked_df
+
+#################### Leaderboards Code ##############################
+
+with gr.Blocks(css="""
+    body, .gradio-container {
+        font-family: 'roboto';
+    }
+""") as demo:
+    gr.Markdown("""
+    # Data Attribution Methods Leaderboards
+    """)
+    gr.Markdown(f"""
+    Survey and ranking of data attribution methods on data selection and 
+                downstream application tasks for the Date-LM Evaluation paper.
+
+    **Leaderboard Submission**:                        
+    - To submit your team's scores, click on the "Submit Scores" tab.
+                
+    **Data Attribution Method Categories**: 
+    - Gradient (ex. GradDot, GradSim, LESS, DataInf, EKFAC)
+    - Similarity (ex. RepSim)
+    - Modeling (ex. MATES)
+    - Lexical (ex. BM25)
+    - Baseline (ex. GradSafe, OpenAI Moderation, LLM Classifiers)
+    - Other                                
+
+    **Search Feature**: 
+    - Input the name of the method you would like to search / filter for, and
+                then press "Enter". The original row from the leaderboard table will be displayed.   
+    """
+    )
+
+    with gr.Tabs():
+        with gr.TabItem("Training Data Selection"):
+            with gr.Tabs():  # Subtabs container
+                with gr.TabItem("Pre-Training (10K)"):  # Subtab
+                    gr.Markdown("""DATE-LM Task Description: Trained pythia-1B model on Fineweb using 
+                                Lambada reference dataset. Testing results conducted on 10K step checkpoint.
+                                
+                                Ranking Metric: highest score in **avg** column""") # description
+                    l1 = Leaderboard(
+                            value=pd.DataFrame(pretrain_10K_data),
+                            select_columns=get_leaderboard_columns("Pre-Training (10K)"),
+                            search_columns=['Method'],
+                            filter_columns=["Attribution Method Type", "Method", "avg"],
+                        )
+                with gr.TabItem("Pre-Training (30K)"):
+                    gr.Markdown("""DATE-LM Task Description: Trained pythia-1B model on Fineweb using 
+                                Lambada reference dataset. Testing results conducted on 30K step checkpoint.
+                                
+                                Ranking Metric: highest score in **avg** column""")
+                    l2 = Leaderboard(
+                        value=pd.DataFrame(pretrain_30K_data),
+                        select_columns=get_leaderboard_columns("Pre-Training (30K)"),
+                        search_columns=["Method"],
+                        filter_columns=["Attribution Method Type", "Method", "avg"],
+                    )
+                with gr.TabItem("Fine-Tuning"):
+                    gr.Markdown("""DATE-LM Task Description: Targeted instruction tuning setting.
+                                 Given a diverse instruction set and a eval dataset, we select data that would yield 
+                                optimal performance on the eval data. For this task, the training data pool is 
+                                Tulu3 (unfiltered) and the eval data is MMLU, GSM8K, and BBH. 
+                                
+                                Ranking Metric: average of the **MMLU**, **GSM8K**, and **BBH** scores""")
+                    l3 = Leaderboard(
+                        value=pd.DataFrame(finetune_data),
+                        select_columns=get_leaderboard_columns("Fine-Tuning"),
+                        search_columns=["Method"],
+                        filter_columns=["Attribution Method Type", "MMLU", "GSM8K", "BBH"],
+                    )
+        with gr.TabItem("Applications"):
+            with gr.Tabs():
+                with gr.TabItem("Toxicity/Bias"):
+                    with gr.Tabs(): 
+                        with gr.TabItem("Homogeneous"):
+                            gr.Markdown("""DATE-LM Task Description: This leaderboard presents detection AUPRC results of baseline methods and data attribution methods in the homogenous setting 
+                                        (i.e., detecting small amount of toxic/biased data embedded into larger benign data).
+                                        
+                                        Ranking Metric: **AUPRC** (an average of ToxicChat, XSTest-response, JailBreakBench)""")
+                            category_filter4 = gr.Dropdown(
+                                choices=get_model_sizes("Homogeneous"),
+                                value="All",
+                                label="Filter Model Size"
+                            ) # ensures page placement above leaderboard
+                            l4 = Leaderboard(
+                                value=pd.DataFrame(homogeneous_data),
+                                select_columns=get_leaderboard_columns("Homogeneous"),
+                                search_columns=["Method"],
+                                filter_columns=["Attribution Method Type", "Model", "AUPRC"],
+                            )
+                            data_path4 = gr.Textbox(value=toxicity_homogeneous, visible=False)
+                            id_str4 = gr.Textbox(value="toxicity", visible=False)
+                            category_filter4.change(
+                                fn=rerank_leaderboard,
+                                inputs=[category_filter4, data_path4, id_str4],
+                                outputs=[l4]
+                            )
+                        with gr.TabItem("Heterogeneous"):
+                            gr.Markdown("""DATE-LM Task Description: This leaderboard presents detection AUPRC results of baseline methods and data attribution methods in the heterogeneous setting 
+                                        (i.e., safety-aligned examples that resemble unsafe data in format but contain safe responses).
+                                        
+                                        Ranking Metric: **AUPRC** (an average of ToxicChat, XSTest-response, JailBreakBench)""")
+                            category_filter5 = gr.Dropdown(
+                                choices=get_model_sizes("Heterogeneous"),
+                                value="All",
+                                label="Filter Model Size"
+                            )
+                            l5 = Leaderboard(
+                                value=pd.DataFrame(heterogeneous_data),
+                                select_columns=get_leaderboard_columns("Heterogeneous"),
+                                search_columns=["Method"],
+                                filter_columns=["Attribution Method Type", "Model", "AUPRC"]
+                            )
+                            data_path5 = gr.Textbox(value=toxicity_heterogeneous, visible=False)
+                            id_str5 = gr.Textbox(value="toxicity", visible=False)
+                            category_filter5.change(
+                                fn=rerank_leaderboard,
+                                inputs=[category_filter5, data_path5, id_str5],
+                                outputs=[l5]
+                            )
+                with gr.TabItem("Factual Attribution"):
+                    gr.Markdown("""DATE-LM Task Description: Identifying the specific training examples that support a model's generated facts.
+                                        
+                                   Ranking Metric: average of **Recall@50** and **MRR**""")
+                    category_filter6 = gr.Dropdown(
+                        choices=get_model_sizes("Factual Attribution"),
+                        value="All",
+                        label="Filter Model Size"
+                    )
+                    l6 = Leaderboard(
+                        value=pd.DataFrame(factual_data),
+                        select_columns=get_leaderboard_columns("Factual Attribution"),
+                        search_columns=["Method"],
+                        filter_columns=["Attribution Method Type", "Model", "Recall@50", "MRR"],
+                    )
+                    data_path6 = gr.Textbox(value=factual, visible=False)
+                    id_str6 = gr.Textbox(value="factual", visible=False)
+                    category_filter6.change(
+                        fn=rerank_leaderboard,
+                        inputs=[category_filter6, data_path6, id_str6],
+                        outputs=[l6]
+                    )
+        with gr.TabItem("Submit Scores 🚀"):
+            with gr.Column():
+                gr.Markdown("""### Submit Your Score to a Leaderboard
+                            
+                Note: Please first select the leaderboard you would like to submit to. This will display the fields for the 
+                            corresponding metrics that are needed. 
+                """)
+
+                leaderboard_dropdown = gr.Dropdown(
+                    label="Select Leaderboard",
+                    choices=nameMapping.LEADERBOARD_NAMES,
+                    value=None
+                )
+
+                method_name = gr.Textbox(label="Method Name")
+                method_dropdown = gr.Dropdown(
+                    label="Method Type",
+                    choices=["Gradient", "Similarity", "Representation-Based", "Modeling", "Baseline", "Lexical", "Other"],
+                    value=None
+                )
+
+                # model_size = gr.Dropdown(
+                #     label="Model Size",
+                #     choices=["400M", "1B", "3B", "7B"],
+                #     value=None
+                # ) 
+                model_name = gr.Textbox(label="Model Name")
+                model_size = gr.Textbox(label="Model Size (ex. 410M, 1B, 8B)")
+
+                paper_link = gr.Textbox(label="Paper/Code/Contact Link") 
+                
+                scores = gr.File(label='Upload Data Attribution Scores File (.pt)', height=150, file_types=[".pt"])
+                
+                # Dynamically Display Needed Fields for Each Leaderboard Type
+
+                with gr.Column(visible=False) as pretrain_group:
+                    pre_avg = gr.Number(label="Avg")
+                    pre_sciq = gr.Number(label="sciq")
+                    pre_arc_easy = gr.Number(label="arc_easy")
+                    pre_arc_chall = gr.Number(label="arc_challenge")
+                    pre_logiqa = gr.Number(label="logiqa")
+                    pre_boolq = gr.Number(label="boolq")
+                    pre_hellaswag = gr.Number(label="hellaswag")
+                    pre_piqa = gr.Number(label="piqa")
+                    pre_wino = gr.Number(label="winogrande")
+                    pre_open = gr.Number(label="openbookqa")
+
+                with gr.Column(visible=False) as finetune_group:
+                    fine_mmlu = gr.Number(label="MMLU")
+                    fine_gsm = gr.Number(label="GSM8K")
+                    fine_bbh = gr.Number(label="BBH")
+
+                with gr.Column(visible=False) as toxicity_group: 
+                    tox_toxicChat = gr.Number(label="ToxicChat")
+                    tox_xsTest = gr.Number(label="XSTest-response")
+                    tox_jbb = gr.Number(label="JailBreakBench")
+                    tox_auprc = gr.Number(label="AUPRC") 
+
+                with gr.Column(visible=False) as factual_group:
+                    fac_recall = gr.Number(label="Recall@50")
+                    fac_mrr = gr.Number(label="MRR")
+
+                # with gr.Group(visible=False) as training_group:
+                #     acc = gr.Number(label="Accuracy")
+                    
+                # applications_group = gr.Column(visible=False)
+                # with applications_group:
+                #     f1_score = gr.Number(label="F1")
+                #     auprc_score = gr.Number(label="AUPRC")
+                #     acc1 = gr.Number(label="Accuracy")
+                
+                # Submit button
+                submit_button = gr.Button("Submit")
+
+                leaderboard_dropdown.change(update_fields, inputs=[leaderboard_dropdown], outputs=[pretrain_group, finetune_group, toxicity_group, factual_group])
+                
+                # information lists
+                inputsList = [leaderboard_dropdown, method_name, method_dropdown, model_name, model_size, paper_link, scores, \
+                              pre_avg, pre_sciq, pre_arc_easy, pre_arc_chall, pre_logiqa, pre_boolq, pre_hellaswag, pre_piqa, pre_wino, pre_open, \
+                              fine_mmlu, fine_gsm, fine_bbh, \
+                              tox_toxicChat, tox_xsTest, tox_jbb, tox_auprc, \
+                              fac_recall, fac_mrr]
+                
+                submit_button.click(
+                    validate_inputs, inputs=inputsList, outputs=[]
+                ).success(fn=pr.submit_and_open_PR, inputs=inputsList, outputs=[gr.Textbox(label="Opened PR on Github")])
+                
+if __name__ == "__main__":
+    demo.launch(debug=True)

data/Applications/factual.json

@@ -0,0 +1,23 @@
+[
+    {"Rank": 0, "Method": "BM25", "Attribution Method Type": "Lexical", "Model": "Pythia-1b", "Model Size": "1B", "Recall@50": 0.305, "MRR": 0.771, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Rep Sim", "Attribution Method Type": "Similarity", "Model": "Pythia-1b", "Model Size": "1B", "Recall@50": 0.376, "MRR": 0.790, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Dot", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "Recall@50": 0.466, "MRR": 0.768, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Sim", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "Recall@50": 0.493, "MRR": 0.836, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "LESS", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "Recall@50": 0.500, "MRR": 0.772, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "DataInf", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "Recall@50": 0.472, "MRR": 0.765, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "EKFAC", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "Recall@50": 0.465, "MRR": 0.766, "Paper/Code/Contact Link": ""},
+
+    {"Rank": 0, "Method": "BM25", "Attribution Method Type": "Lexical", "Model": "Llama-3.2-1B", "Model Size": "1B", "Recall@50": 0.236, "MRR": 0.683, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Rep Sim", "Attribution Method Type": "Similarity", "Model": "Llama-3.2-1B", "Model Size": "1B", "Recall@50": 0.552, "MRR": 0.758, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Dot", "Attribution Method Type": "Gradient", "Model": "Llama-3.2-1B", "Model Size": "1B", "Recall@50": 0.465, "MRR": 0.786, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Sim", "Attribution Method Type": "Gradient", "Model": "Llama-3.2-1B", "Model Size": "1B", "Recall@50": 0.584, "MRR": 0.839, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "LESS", "Attribution Method Type": "Gradient", "Model": "Llama-3.2-1B", "Model Size": "1B", "Recall@50": 0.573, "MRR": 0.807, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "DataInf", "Attribution Method Type": "Gradient", "Model": "Llama-3.2-1B", "Model Size": "1B", "Recall@50": 0.475, "MRR": 0.785, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "EKFAC", "Attribution Method Type": "Gradient", "Model": "Llama-3.2-1B", "Model Size": "1B", "Recall@50": 0.485, "MRR": 0.881, "Paper/Code/Contact Link": ""},
+
+    {"Rank": 0, "Method": "BM25", "Attribution Method Type": "Lexical", "Model": "Llama-3.1-8B", "Model Size": "8B", "Recall@50": 0.313, "MRR": 0.826, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Rep Sim", "Attribution Method Type": "Similarity", "Model": "Llama-3.1-8B", "Model Size": "8B", "Recall@50": 0.625, "MRR": 0.965, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Dot", "Attribution Method Type": "Gradient", "Model": "Llama-3.1-8B", "Model Size": "8B", "Recall@50": 0.226, "MRR": 0.303, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Sim", "Attribution Method Type": "Gradient", "Model": "Llama-3.1-8B", "Model Size": "8B", "Recall@50": 0.626, "MRR": 0.970, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "LESS", "Attribution Method Type": "Gradient", "Model": "Llama-3.1-8B", "Model Size": "8B", "Recall@50": 0.491, "MRR": 0.991, "Paper/Code/Contact Link": ""}
+]

data/Applications/toxicity-heterogeneous.json

@@ -0,0 +1,27 @@
+[
+    {"Rank": 0, "Method": "GradSafe", "Attribution Method Type": "Baseline", "Model": "N/A", "Model Size": "N/A", "ToxicChat": 0.347, "XSTest-response": 0.491, "JailBreakBench": 0.802, "AUPRC": 0.546, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "OpenAI Moderation", "Attribution Method Type": "Baseline", "Model": "N/A", "Model Size": "N/A", "ToxicChat": 0.214, "XSTest-response": 0.358, "JailBreakBench": 0.185, "AUPRC": 0.253, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Llama-Guard-3-8B", "Attribution Method Type": "Baseline", "Model": "N/A", "Model Size": "N/A", "ToxicChat": 0.423, "XSTest-response": 0.910, "JailBreakBench": 0.966, "AUPRC": 0.766, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Wildguard", "Attribution Method Type": "Baseline", "Model": "N/A", "Model Size": "N/A", "ToxicChat": 0.551, "XSTest-response": 0.928, "JailBreakBench": 0.972, "AUPRC": 0.817, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "ShieldGemma-2b", "Attribution Method Type": "Baseline", "Model": "N/A", "Model Size": "N/A", "ToxicChat": 0.165, "XSTest-response": 0.731, "JailBreakBench": 0.552, "AUPRC": 0.483, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "AEGIS-Defensive", "Attribution Method Type": "Baseline", "Model": "N/A", "Model Size": "N/A", "ToxicChat": 0.376, "XSTest-response": 0.274, "JailBreakBench": 0.294, "AUPRC": 0.314, "Paper/Code/Contact Link": ""},
+
+    {"Rank": 0, "Method": "Rep-Sim", "Attribution Method Type": "Similarity", "Model": "Pythia-1b", "Model Size": "1B", "ToxicChat": 0.335, "XSTest-response": 0.580, "JailBreakBench": 0.578, "AUPRC": 0.498, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Dot", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "ToxicChat": 0.194, "XSTest-response": 0.389, "JailBreakBench": 0.396, "AUPRC": 0.326, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Sim", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "ToxicChat": 0.362, "XSTest-response": 0.601, "JailBreakBench": 0.434, "AUPRC": 0.466, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "LESS", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "ToxicChat": 0.326, "XSTest-response": 0.734, "JailBreakBench": 0.484, "AUPRC": 0.515, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "DataInf", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "ToxicChat": 0.195, "XSTest-response": 0.392, "JailBreakBench": 0.396, "AUPRC": 0.328, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "EKFAC", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "ToxicChat": 0.221, "XSTest-response": 0.344, "JailBreakBench": 0.373, "AUPRC": 0.313, "Paper/Code/Contact Link": ""},
+
+    {"Rank": 0, "Method": "Rep-Sim", "Attribution Method Type": "Similarity", "Model": "Llama-3.2-1B", "Model Size": "1B", "ToxicChat": 0.598, "XSTest-response": 0.733, "JailBreakBench": 0.461, "AUPRC": 0.597, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Dot", "Attribution Method Type": "Gradient", "Model": "Llama-3.2-1B", "Model Size": "1B", "ToxicChat": 0.195, "XSTest-response": 0.341, "JailBreakBench": 0.369, "AUPRC": 0.302, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Sim", "Attribution Method Type": "Gradient", "Model": "Llama-3.2-1B", "Model Size": "1B", "ToxicChat": 0.228, "XSTest-response": 0.772, "JailBreakBench": 0.531, "AUPRC": 0.510, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "LESS", "Attribution Method Type": "Gradient", "Model": "Llama-3.2-1B", "Model Size": "1B", "ToxicChat": 0.230, "XSTest-response": 0.616, "JailBreakBench": 0.596, "AUPRC": 0.481, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "DataInf", "Attribution Method Type": "Gradient", "Model": "Llama-3.2-1B", "Model Size": "1B", "ToxicChat": 0.196, "XSTest-response": 0.347, "JailBreakBench": 0.369, "AUPRC": 0.304, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "EKFAC", "Attribution Method Type": "Gradient", "Model": "Llama-3.2-1B", "Model Size": "1B", "ToxicChat": 0.239, "XSTest-response": 0.398, "JailBreakBench": 0.369, "AUPRC": 0.334, "Paper/Code/Contact Link": ""},
+
+    {"Rank": 0, "Method": "Rep-Sim", "Attribution Method Type": "Similarity", "Model": "Llama-3.1-8B", "Model Size": "8B", "ToxicChat": 0.602, "XSTest-response": 0.638, "JailBreakBench": 0.514, "AUPRC": 0.585, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Dot", "Attribution Method Type": "Gradient", "Model": "Llama-3.1-8B", "Model Size": "8B", "ToxicChat": 0.289, "XSTest-response": 0.328, "JailBreakBench": 0.085, "AUPRC": 0.234, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Sim", "Attribution Method Type": "Gradient", "Model": "Llama-3.1-8B", "Model Size": "8B", "ToxicChat": 0.223, "XSTest-response": 0.703, "JailBreakBench": 0.401, "AUPRC": 0.442, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "LESS", "Attribution Method Type": "Gradient", "Model": "Llama-3.1-8B", "Model Size": "8B", "ToxicChat": 0.258, "XSTest-response": 0.744, "JailBreakBench": 0.114, "AUPRC": 0.372, "Paper/Code/Contact Link": ""}
+]

data/Applications/toxicity-homogeneous.json

@@ -0,0 +1,278 @@
+[
+  {
+    "Rank": 0,
+    "Method": "GradSafe",
+    "Attribution Method Type": "Baseline",
+    "Model": "N/A",
+    "Model Size": "N/A",
+    "ToxicChat": 0.347,
+    "XSTest-response": 0.491,
+    "JailBreakBench": 0.802,
+    "AUPRC": 0.546,
+    "Paper/Code/Contact Link": ""
+  },
+  {
+    "Rank": 0,
+    "Method": "OpenAI Moderation",
+    "Attribution Method Type": "Baseline",
+    "Model": "N/A",
+    "Model Size": "N/A",
+    "ToxicChat": 0.243,
+    "XSTest-response": 0.378,
+    "JailBreakBench": 0.187,
+    "AUPRC": 0.269,
+    "Paper/Code/Contact Link": ""
+  },
+  {
+    "Rank": 0,
+    "Method": "Llama-Guard-3-8B",
+    "Attribution Method Type": "Baseline",
+    "Model": "N/A",
+    "Model Size": "N/A",
+    "ToxicChat": 0.445,
+    "XSTest-response": 0.916,
+    "JailBreakBench": 0.985,
+    "AUPRC": 0.782,
+    "Paper/Code/Contact Link": ""
+  },
+  {
+    "Rank": 0,
+    "Method": "Wildguard",
+    "Attribution Method Type": "Baseline",
+    "Model": "N/A",
+    "Model Size": "N/A",
+    "ToxicChat": 0.56,
+    "XSTest-response": 0.93,
+    "JailBreakBench": 0.989,
+    "AUPRC": 0.827,
+    "Paper/Code/Contact Link": ""
+  },
+  {
+    "Rank": 0,
+    "Method": "ShieldGemma-2b",
+    "Attribution Method Type": "Baseline",
+    "Model": "N/A",
+    "Model Size": "N/A",
+    "ToxicChat": 0.17,
+    "XSTest-response": 0.74,
+    "JailBreakBench": 0.664,
+    "AUPRC": 0.525,
+    "Paper/Code/Contact Link": ""
+  },
+  {
+    "Rank": 0,
+    "Method": "AEGIS-Defensive",
+    "Attribution Method Type": "Baseline",
+    "Model": "N/A",
+    "Model Size": "N/A",
+    "ToxicChat": 0.376,
+    "XSTest-response": 0.274,
+    "JailBreakBench": 0.346,
+    "AUPRC": 0.332,
+    "Paper/Code/Contact Link": ""
+  },
+  {
+    "Rank": 0,
+    "Method": "Rep-Sim",
+    "Attribution Method Type": "Similarity",
+    "Model": "Pythia-1b",
+    "Model Size": "1B",
+    "ToxicChat": 0.374,
+    "XSTest-response": 0.657,
+    "JailBreakBench": 0.986,
+    "AUPRC": 0.672,
+    "Paper/Code/Contact Link": ""
+  },
+  {
+    "Rank": 0,
+    "Method": "Grad Dot",
+    "Attribution Method Type": "Gradient",
+    "Model": "Pythia-1b",
+    "Model Size": "1B",
+    "ToxicChat": 0.084,
+    "XSTest-response": 0.483,
+    "JailBreakBench": 0.999,
+    "AUPRC": 0.522,
+    "Paper/Code/Contact Link": ""
+  },
+  {
+    "Rank": 0,
+    "Method": "Grad Sim",
+    "Attribution Method Type": "Gradient",
+    "Model": "Pythia-1b",
+    "Model Size": "1B",
+    "ToxicChat": 0.106,
+    "XSTest-response": 0.647,
+    "JailBreakBench": 1.0,
+    "AUPRC": 0.584,
+    "Paper/Code/Contact Link": ""
+  },
+  {
+    "Rank": 0,
+    "Method": "LESS",
+    "Attribution Method Type": "Gradient",
+    "Model": "Pythia-1b",
+    "Model Size": "1B",
+    "ToxicChat": 0.388,
+    "XSTest-response": 0.724,
+    "JailBreakBench": 1.0,
+    "AUPRC": 0.704,
+    "Paper/Code/Contact Link": ""
+  },
+  {
+    "Rank": 0,
+    "Method": "DataInf",
+    "Attribution Method Type": "Gradient",
+    "Model": "Pythia-1b",
+    "Model Size": "1B",
+    "ToxicChat": 0.204,
+    "XSTest-response": 0.487,
+    "JailBreakBench": 0.999,
+    "AUPRC": 0.563,
+    "Paper/Code/Contact Link": ""
+  },
+  {
+    "Rank": 0,
+    "Method": "EKFAC",
+    "Attribution Method Type": "Gradient",
+    "Model": "Pythia-1b",
+    "Model Size": "1B",
+    "ToxicChat": 0.216,
+    "XSTest-response": 0.497,
+    "JailBreakBench": 1.0,
+    "AUPRC": 0.571,
+    "Paper/Code/Contact Link": ""
+  },
+  {
+    "Rank": 0,
+    "Method": "Rep-Sim",
+    "Attribution Method Type": "Similarity",
+    "Model": "Llama-3.2-1B",
+    "Model Size": "1B",
+    "ToxicChat": 0.632,
+    "XSTest-response": 0.792,
+    "JailBreakBench": 0.854,
+    "AUPRC": 0.759,
+    "Paper/Code/Contact Link": ""
+  },
+  {
+    "Rank": 0,
+    "Method": "Grad Dot",
+    "Attribution Method Type": "Gradient",
+    "Model": "Llama-3.2-1B",
+    "Model Size": "1B",
+    "ToxicChat": 0.212,
+    "XSTest-response": 0.437,
+    "JailBreakBench": 1.0,
+    "AUPRC": 0.55,
+    "Paper/Code/Contact Link": ""
+  },
+  {
+    "Rank": 0,
+    "Method": "Grad Sim",
+    "Attribution Method Type": "Gradient",
+    "Model": "Llama-3.2-1B",
+    "Model Size": "1B",
+    "ToxicChat": 0.259,
+    "XSTest-response": 0.798,
+    "JailBreakBench": 1.0,
+    "AUPRC": 0.686,
+    "Paper/Code/Contact Link": ""
+  },
+  {
+    "Rank": 0,
+    "Method": "LESS",
+    "Attribution Method Type": "Gradient",
+    "Model": "Llama-3.2-1B",
+    "Model Size": "1B",
+    "ToxicChat": 0.294,
+    "XSTest-response": 0.792,
+    "JailBreakBench": 1.0,
+    "AUPRC": 0.695,
+    "Paper/Code/Contact Link": ""
+  },
+  {
+    "Rank": 0,
+    "Method": "DataInf",
+    "Attribution Method Type": "Gradient",
+    "Model": "Llama-3.2-1B",
+    "Model Size": "1B",
+    "ToxicChat": 0.215,
+    "XSTest-response": 0.442,
+    "JailBreakBench": 1.0,
+    "AUPRC": 0.552,
+    "Paper/Code/Contact Link": ""
+  },
+  {
+    "Rank": 0,
+    "Method": "EKFAC",
+    "Attribution Method Type": "Gradient",
+    "Model": "Llama-3.2-1B",
+    "Model Size": "1B",
+    "ToxicChat": 0.264,
+    "XSTest-response": 0.562,
+    "JailBreakBench": 1.0,
+    "AUPRC": 0.609,
+    "Paper/Code/Contact Link": ""
+  },
+  {
+    "Rank": 0,
+    "Method": "Rep-Sim",
+    "Attribution Method Type": "Similarity",
+    "Model": "Llama-3.1-8B",
+    "Model Size": "8B",
+    "ToxicChat": 0.989,
+    "XSTest-response": 0.999,
+    "JailBreakBench": 0.98,
+    "AUPRC": 0.989,
+    "Paper/Code/Contact Link": ""
+  },
+  {
+    "Rank": 0,
+    "Method": "Grad Dot",
+    "Attribution Method Type": "Gradient",
+    "Model": "Llama-3.1-8B",
+    "Model Size": "8B",
+    "ToxicChat": 0.47,
+    "XSTest-response": 0.368,
+    "JailBreakBench": 0.274,
+    "AUPRC": 0.371,
+    "Paper/Code/Contact Link": ""
+  },
+  {
+    "Rank": 0,
+    "Method": "Grad Sim",
+    "Attribution Method Type": "Gradient",
+    "Model": "Llama-3.1-8B",
+    "Model Size": "8B",
+    "ToxicChat": 0.28,
+    "XSTest-response": 0.603,
+    "JailBreakBench": 0.82,
+    "AUPRC": 0.567,
+    "Paper/Code/Contact Link": ""
+  },
+  {
+    "Rank": 0,
+    "Method": "LESS",
+    "Attribution Method Type": "Gradient",
+    "Model": "Llama-3.1-8B",
+    "Model Size": "8B",
+    "ToxicChat": 0.499,
+    "XSTest-response": 0.615,
+    "JailBreakBench": 0.767,
+    "AUPRC": 0.627,
+    "Paper/Code/Contact Link": ""
+  },
+  {
+    "Rank": 0,
+    "Method": "test",
+    "Attribution Method Type": "Gradient",
+    "Model": "pythia",
+    "Model Size": "1B",
+    "Paper/Code/Contact Link": "test",
+    "ToxicChat": 0.3,
+    "XSTest-response": 0.3,
+    "JailBreakBench": 0.3,
+    "AUPRC": 0.3
+  }
+]

data/DATE-LM-Original-Data/factual.json

@@ -0,0 +1,23 @@
+[
+    {"Rank": 0, "Method": "BM25", "Attribution Method Type": "Lexical", "Model": "Pythia-1b", "Model Size": "1B", "Recall@50": 0.305, "MRR": 0.771, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Rep Sim", "Attribution Method Type": "Similarity", "Model": "Pythia-1b", "Model Size": "1B", "Recall@50": 0.376, "MRR": 0.790, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Dot", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "Recall@50": 0.466, "MRR": 0.768, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Sim", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "Recall@50": 0.493, "MRR": 0.836, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "LESS", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "Recall@50": 0.500, "MRR": 0.772, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "DataInf", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "Recall@50": 0.472, "MRR": 0.765, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "EKFAC", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "Recall@50": 0.465, "MRR": 0.766, "Paper/Code/Contact Link": ""},
+
+    {"Rank": 0, "Method": "BM25", "Attribution Method Type": "Lexical", "Model": "Llama-3.2-1B", "Model Size": "1B", "Recall@50": 0.236, "MRR": 0.683, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Rep Sim", "Attribution Method Type": "Similarity", "Model": "Llama-3.2-1B", "Model Size": "1B", "Recall@50": 0.552, "MRR": 0.758, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Dot", "Attribution Method Type": "Gradient", "Model": "Llama-3.2-1B", "Model Size": "1B", "Recall@50": 0.465, "MRR": 0.786, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Sim", "Attribution Method Type": "Gradient", "Model": "Llama-3.2-1B", "Model Size": "1B", "Recall@50": 0.584, "MRR": 0.839, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "LESS", "Attribution Method Type": "Gradient", "Model": "Llama-3.2-1B", "Model Size": "1B", "Recall@50": 0.573, "MRR": 0.807, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "DataInf", "Attribution Method Type": "Gradient", "Model": "Llama-3.2-1B", "Model Size": "1B", "Recall@50": 0.475, "MRR": 0.785, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "EKFAC", "Attribution Method Type": "Gradient", "Model": "Llama-3.2-1B", "Model Size": "1B", "Recall@50": 0.485, "MRR": 0.881, "Paper/Code/Contact Link": ""},
+
+    {"Rank": 0, "Method": "BM25", "Attribution Method Type": "Lexical", "Model": "Llama-3.1-8B", "Model Size": "8B", "Recall@50": 0.313, "MRR": 0.826, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Rep Sim", "Attribution Method Type": "Similarity", "Model": "Llama-3.1-8B", "Model Size": "8B", "Recall@50": 0.625, "MRR": 0.965, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Dot", "Attribution Method Type": "Gradient", "Model": "Llama-3.1-8B", "Model Size": "8B", "Recall@50": 0.226, "MRR": 0.303, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Sim", "Attribution Method Type": "Gradient", "Model": "Llama-3.1-8B", "Model Size": "8B", "Recall@50": 0.626, "MRR": 0.970, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "LESS", "Attribution Method Type": "Gradient", "Model": "Llama-3.1-8B", "Model Size": "8B", "Recall@50": 0.491, "MRR": 0.991, "Paper/Code/Contact Link": ""}
+]

data/DATE-LM-Original-Data/finetune.json

@@ -0,0 +1,7 @@
+[
+    {"Rank": 0, "Method": "Random", "Attribution Method Type": "Other", "Model": "Llama-3.1-8B", "Model Size": "8B", "MMLU": 0.602, "GSM8K": 0.596, "BBH": 0.653, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "BM25", "Attribution Method Type": "Lexical", "Model": "Llama-3.1-8B", "Model Size": "8B", "MMLU": 0, "GSM8K": 0, "BBH": 0, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Rep Sim", "Attribution Method Type": "Similarity", "Model": "Llama-3.1-8B", "Model Size": "8B", "MMLU": 0, "GSM8K": 0, "BBH": 0, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Sim", "Attribution Method Type": "Gradient", "Model": "Llama-3.1-8B", "Model Size": "8B", "MMLU": 0, "GSM8K": 0, "BBH": 0, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "LESS (optimizer)", "Attribution Method Type": "Gradient", "Model": "Llama-3.1-8B", "Model Size": "8B", "MMLU": 0, "GSM8K": 0, "BBH": 0, "Paper/Code/Contact Link": ""}
+]

data/DATE-LM-Original-Data/pythia1b-10k-lambada.json

@@ -0,0 +1,8 @@
+[
+    {"Rank": 0, "Method": "Random", "Attribution Method Type": "Other", "Model": "Pythia-1b", "Model Size": "1B", "avg": 45.34, "sciq": 0.689, "arc_easy": 0.431, "arc_challenge": 0.244, "logiqa": 0.275, "boolq": 0.520, "hellaswag": 0.407, "piqa": 0.690, "winogrande": 0.535, "openbookqa": 0.290, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "BM25", "Attribution Method Type": "Lexical", "Model": "Pythia-1b", "Model Size": "1B", "avg": 45.72, "sciq": 0.692, "arc_easy": 0.439, "arc_challenge": 0.239, "logiqa": 0.260, "boolq": 0.556, "hellaswag": 0.406, "piqa": 0.696, "winogrande": 0.531, "openbookqa": 0.296, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Sim", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "avg": 45.98, "sciq": 0.689, "arc_easy": 0.440, "arc_challenge": 0.240, "logiqa": 0.272, "boolq": 0.556, "hellaswag": 0.406, "piqa": 0.690, "winogrande": 0.537, "openbookqa": 0.308, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Rep Sim", "Attribution Method Type": "Similarity", "Model": "Pythia-1b", "Model Size": "1B", "avg": 46.00, "sciq": 0.691, "arc_easy": 0.441, "arc_challenge": 0.237, "logiqa": 0.275, "boolq": 0.561, "hellaswag": 0.409, "piqa": 0.695, "winogrande": 0.537, "openbookqa": 0.294, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Mates", "Attribution Method Type": "Modeling", "Model": "Pythia-1b", "Model Size": "1B", "avg": 45.76, "sciq": 0.685, "arc_easy": 0.441, "arc_challenge": 0.241, "logiqa": 0.269, "boolq": 0.563, "hellaswag": 0.408, "piqa": 0.696, "winogrande": 0.523, "openbookqa": 0.292, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Edu", "Attribution Method Type": "Other", "Model": "Pythia-1b", "Model Size": "1B", "avg": 45.83, "sciq": 0.688, "arc_easy": 0.452, "arc_challenge": 0.240, "logiqa": 0.264, "boolq": 0.571, "hellaswag": 0.409, "piqa": 0.689, "winogrande": 0.520, "openbookqa": 0.292, "Paper/Code/Contact Link": ""}
+]

data/DATE-LM-Original-Data/pythia1b-30k-lambada.json

@@ -0,0 +1,7 @@
+[
+    {"Rank": 0, "Method": "Random", "Attribution Method Type": "Other", "Model": "Pythia-1b", "Model Size": "1B", "avg": 47.49, "sciq": 0.695, "arc_easy": 0.456, "arc_challenge": 0.255, "logiqa": 0.276, "boolq": 0.602, "hellaswag": 0.475, "piqa": 0.711, "winogrande": 0.510, "openbookqa": 0.294, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Rep Sim", "Attribution Method Type": "Similarity", "Model": "Pythia-1b", "Model Size": "1B", "avg": 47.83, "sciq": 0.697, "arc_easy": 0.462, "arc_challenge": 0.259, "logiqa": 0.263, "boolq": 0.612, "hellaswag": 0.474, "piqa": 0.712, "winogrande": 0.526, "openbookqa": 0.300, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Sim", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "avg": 47.92, "sciq": 0.711, "arc_easy": 0.465, "arc_challenge": 0.254, "logiqa": 0.273, "boolq": 0.606, "hellaswag": 0.475, "piqa": 0.711, "winogrande": 0.522, "openbookqa": 0.296, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Mates", "Attribution Method Type": "Modeling", "Model": "Pythia-1b", "Model Size": "1B", "avg": 47.64, "sciq": 0.702, "arc_easy": 0.464, "arc_challenge": 0.253, "logiqa": 0.260, "boolq": 0.617, "hellaswag": 0.474, "piqa": 0.708, "winogrande": 0.518, "openbookqa": 0.292, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Edu", "Attribution Method Type": "Other", "Model": "Pythia-1b", "Model Size": "1B", "avg": 48.02, "sciq": 0.700, "arc_easy": 0.471, "arc_challenge": 0.262, "logiqa": 0.267, "boolq": 0.616, "hellaswag": 0.474, "piqa": 0.709, "winogrande": 0.511, "openbookqa": 0.312, "Paper/Code/Contact Link": ""}
+]

data/DATE-LM-Original-Data/toxicity-heterogeneous.json

@@ -0,0 +1,27 @@
+[
+    {"Rank": 0, "Method": "GradSafe", "Attribution Method Type": "Baseline", "Model": "N/A", "Model Size": "N/A", "ToxicChat": 0.347, "XSTest-response": 0.491, "JailBreakBench": 0.802, "AUPRC": 0.546, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "OpenAI Moderation", "Attribution Method Type": "Baseline", "Model": "N/A", "Model Size": "N/A", "ToxicChat": 0.214, "XSTest-response": 0.358, "JailBreakBench": 0.185, "AUPRC": 0.253, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Llama-Guard-3-8B", "Attribution Method Type": "Baseline", "Model": "N/A", "Model Size": "N/A", "ToxicChat": 0.423, "XSTest-response": 0.910, "JailBreakBench": 0.966, "AUPRC": 0.766, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Wildguard", "Attribution Method Type": "Baseline", "Model": "N/A", "Model Size": "N/A", "ToxicChat": 0.551, "XSTest-response": 0.928, "JailBreakBench": 0.972, "AUPRC": 0.817, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "ShieldGemma-2b", "Attribution Method Type": "Baseline", "Model": "N/A", "Model Size": "N/A", "ToxicChat": 0.165, "XSTest-response": 0.731, "JailBreakBench": 0.552, "AUPRC": 0.483, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "AEGIS-Defensive", "Attribution Method Type": "Baseline", "Model": "N/A", "Model Size": "N/A", "ToxicChat": 0.376, "XSTest-response": 0.274, "JailBreakBench": 0.294, "AUPRC": 0.314, "Paper/Code/Contact Link": ""},
+
+    {"Rank": 0, "Method": "Rep-Sim", "Attribution Method Type": "Similarity", "Model": "Pythia-1b", "Model Size": "1B", "ToxicChat": 0.335, "XSTest-response": 0.580, "JailBreakBench": 0.578, "AUPRC": 0.498, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Dot", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "ToxicChat": 0.194, "XSTest-response": 0.389, "JailBreakBench": 0.396, "AUPRC": 0.326, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Sim", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "ToxicChat": 0.362, "XSTest-response": 0.601, "JailBreakBench": 0.434, "AUPRC": 0.466, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "LESS", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "ToxicChat": 0.326, "XSTest-response": 0.734, "JailBreakBench": 0.484, "AUPRC": 0.515, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "DataInf", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "ToxicChat": 0.195, "XSTest-response": 0.392, "JailBreakBench": 0.396, "AUPRC": 0.328, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "EKFAC", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "ToxicChat": 0.221, "XSTest-response": 0.344, "JailBreakBench": 0.373, "AUPRC": 0.313, "Paper/Code/Contact Link": ""},
+
+    {"Rank": 0, "Method": "Rep-Sim", "Attribution Method Type": "Similarity", "Model": "Llama-3.2-1B", "Model Size": "1B", "ToxicChat": 0.598, "XSTest-response": 0.733, "JailBreakBench": 0.461, "AUPRC": 0.597, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Dot", "Attribution Method Type": "Gradient", "Model": "Llama-3.2-1B", "Model Size": "1B", "ToxicChat": 0.195, "XSTest-response": 0.341, "JailBreakBench": 0.369, "AUPRC": 0.302, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Sim", "Attribution Method Type": "Gradient", "Model": "Llama-3.2-1B", "Model Size": "1B", "ToxicChat": 0.228, "XSTest-response": 0.772, "JailBreakBench": 0.531, "AUPRC": 0.510, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "LESS", "Attribution Method Type": "Gradient", "Model": "Llama-3.2-1B", "Model Size": "1B", "ToxicChat": 0.230, "XSTest-response": 0.616, "JailBreakBench": 0.596, "AUPRC": 0.481, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "DataInf", "Attribution Method Type": "Gradient", "Model": "Llama-3.2-1B", "Model Size": "1B", "ToxicChat": 0.196, "XSTest-response": 0.347, "JailBreakBench": 0.369, "AUPRC": 0.304, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "EKFAC", "Attribution Method Type": "Gradient", "Model": "Llama-3.2-1B", "Model Size": "1B", "ToxicChat": 0.239, "XSTest-response": 0.398, "JailBreakBench": 0.369, "AUPRC": 0.334, "Paper/Code/Contact Link": ""},
+
+    {"Rank": 0, "Method": "Rep-Sim", "Attribution Method Type": "Similarity", "Model": "Llama-3.1-8B", "Model Size": "8B", "ToxicChat": 0.602, "XSTest-response": 0.638, "JailBreakBench": 0.514, "AUPRC": 0.585, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Dot", "Attribution Method Type": "Gradient", "Model": "Llama-3.1-8B", "Model Size": "8B", "ToxicChat": 0.289, "XSTest-response": 0.328, "JailBreakBench": 0.085, "AUPRC": 0.234, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Sim", "Attribution Method Type": "Gradient", "Model": "Llama-3.1-8B", "Model Size": "8B", "ToxicChat": 0.223, "XSTest-response": 0.703, "JailBreakBench": 0.401, "AUPRC": 0.442, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "LESS", "Attribution Method Type": "Gradient", "Model": "Llama-3.1-8B", "Model Size": "8B", "ToxicChat": 0.258, "XSTest-response": 0.744, "JailBreakBench": 0.114, "AUPRC": 0.372, "Paper/Code/Contact Link": ""}
+]

data/DATE-LM-Original-Data/toxicity-homogeneous.json

@@ -0,0 +1,27 @@
+[
+    {"Rank": 0, "Method": "GradSafe", "Attribution Method Type": "Baseline", "Model": "N/A", "Model Size": "N/A", "ToxicChat": 0.347, "XSTest-response": 0.491, "JailBreakBench": 0.802, "AUPRC": 0.546, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "OpenAI Moderation", "Attribution Method Type": "Baseline", "Model": "N/A", "Model Size": "N/A", "ToxicChat": 0.243, "XSTest-response": 0.378, "JailBreakBench": 0.187, "AUPRC": 0.269, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Llama-Guard-3-8B", "Attribution Method Type": "Baseline", "Model": "N/A", "Model Size": "N/A", "ToxicChat": 0.445, "XSTest-response": 0.916, "JailBreakBench": 0.985, "AUPRC": 0.782, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Wildguard", "Attribution Method Type": "Baseline", "Model": "N/A", "Model Size": "N/A", "ToxicChat": 0.560, "XSTest-response": 0.930, "JailBreakBench": 0.989, "AUPRC": 0.827, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "ShieldGemma-2b", "Attribution Method Type": "Baseline", "Model": "N/A", "Model Size": "N/A", "ToxicChat": 0.170, "XSTest-response": 0.740, "JailBreakBench": 0.664, "AUPRC": 0.525, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "AEGIS-Defensive", "Attribution Method Type": "Baseline", "Model": "N/A", "Model Size": "N/A", "ToxicChat": 0.376, "XSTest-response": 0.274, "JailBreakBench": 0.346, "AUPRC": 0.332, "Paper/Code/Contact Link": ""},
+
+    {"Rank": 0, "Method": "Rep-Sim", "Attribution Method Type": "Similarity", "Model": "Pythia-1b", "Model Size": "1B", "ToxicChat": 0.374, "XSTest-response": 0.657, "JailBreakBench": 0.986, "AUPRC": 0.672, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Dot", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "ToxicChat": 0.084, "XSTest-response": 0.483, "JailBreakBench": 0.999, "AUPRC": 0.522, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Sim", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "ToxicChat": 0.106, "XSTest-response": 0.647, "JailBreakBench": 1.000, "AUPRC": 0.584, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "LESS", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "ToxicChat": 0.388, "XSTest-response": 0.724, "JailBreakBench": 1.000, "AUPRC": 0.704, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "DataInf", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "ToxicChat": 0.204, "XSTest-response": 0.487, "JailBreakBench": 0.999,  "AUPRC": 0.563, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "EKFAC", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "ToxicChat": 0.216, "XSTest-response": 0.497, "JailBreakBench": 1.000,  "AUPRC": 0.571, "Paper/Code/Contact Link": ""},
+
+    {"Rank": 0, "Method": "Rep-Sim", "Attribution Method Type": "Similarity", "Model": "Llama-3.2-1B", "Model Size": "1B", "ToxicChat": 0.632, "XSTest-response": 0.792, "JailBreakBench": 0.854, "AUPRC": 0.759, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Dot", "Attribution Method Type": "Gradient", "Model": "Llama-3.2-1B", "Model Size": "1B", "ToxicChat": 0.212, "XSTest-response": 0.437, "JailBreakBench": 1.000, "AUPRC": 0.550, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Sim", "Attribution Method Type": "Gradient", "Model": "Llama-3.2-1B", "Model Size": "1B", "ToxicChat": 0.259, "XSTest-response": 0.798, "JailBreakBench": 1.000, "AUPRC": 0.686, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "LESS", "Attribution Method Type": "Gradient", "Model": "Llama-3.2-1B", "Model Size": "1B", "ToxicChat": 0.294, "XSTest-response": 0.792, "JailBreakBench": 1.000, "AUPRC": 0.695, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "DataInf", "Attribution Method Type": "Gradient", "Model": "Llama-3.2-1B", "Model Size": "1B", "ToxicChat": 0.215, "XSTest-response": 0.442, "JailBreakBench": 1.000, "AUPRC": 0.552, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "EKFAC", "Attribution Method Type": "Gradient", "Model": "Llama-3.2-1B", "Model Size": "1B", "ToxicChat": 0.264, "XSTest-response": 0.562, "JailBreakBench": 1.000, "AUPRC": 0.609, "Paper/Code/Contact Link": ""},
+
+    {"Rank": 0, "Method": "Rep-Sim", "Attribution Method Type": "Similarity", "Model": "Llama-3.1-8B", "Model Size": "8B", "ToxicChat": 0.989, "XSTest-response": 0.999, "JailBreakBench": 0.980, "AUPRC": 0.989, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Dot", "Attribution Method Type": "Gradient", "Model": "Llama-3.1-8B", "Model Size": "8B", "ToxicChat": 0.470, "XSTest-response": 0.368, "JailBreakBench": 0.274, "AUPRC": 0.371, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Sim", "Attribution Method Type": "Gradient", "Model": "Llama-3.1-8B", "Model Size": "8B", "ToxicChat": 0.280, "XSTest-response": 0.603, "JailBreakBench": 0.820, "AUPRC": 0.567, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "LESS", "Attribution Method Type": "Gradient", "Model": "Llama-3.1-8B", "Model Size": "8B", "ToxicChat": 0.499, "XSTest-response": 0.615, "JailBreakBench": 0.767, "AUPRC": 0.627, "Paper/Code/Contact Link": ""}
+]

data/DataSelection/finetune.json

@@ -0,0 +1,68 @@
+[
+  {
+    "Rank": 0,
+    "Method": "Random Avg",
+    "Attribution Method Type": "Other",
+    "Model": "Llama-3.1-8B",
+    "Model Size": "8B",
+    "MMLU": 60.2,
+    "GSM8K": 59.6,
+    "BBH": 65.6,
+    "Paper/Code/Contact Link": ""
+  },
+  {
+    "Rank": 0,
+    "Method": "BM25",
+    "Attribution Method Type": "Lexical",
+    "Model": "Llama-3.1-8B",
+    "Model Size": "8B",
+    "MMLU": 59.5,
+    "GSM8K": 60.2,
+    "BBH": 62.5,
+    "Paper/Code/Contact Link": ""
+  },
+  {
+    "Rank": 0,
+    "Method": "Rep Sim",
+    "Attribution Method Type": "Similarity",
+    "Model": "Llama-3.1-8B",
+    "Model Size": "8B",
+    "MMLU": 61.2,
+    "GSM8K": 59.2,
+    "BBH": 65.9,
+    "Paper/Code/Contact Link": ""
+  },
+  {
+    "Rank": 0,
+    "Method": "RDS+",
+    "Attribution Method Type": "Similarity",
+    "Model": "Llama-3.1-8B",
+    "Model Size": "8B",
+    "MMLU": 62.4,
+    "GSM8K": 59.6,
+    "BBH": 66.9,
+    "Paper/Code/Contact Link": ""
+  },
+  {
+    "Rank": 0,
+    "Method": "Grad Sim",
+    "Attribution Method Type": "Gradient",
+    "Model": "Llama-3.1-8B",
+    "Model Size": "8B",
+    "MMLU": 58.4,
+    "GSM8K": 57.8,
+    "BBH": 65.5,
+    "Paper/Code/Contact Link": ""
+  },
+  {
+    "Rank": 0,
+    "Method": "LESS (optimizer)",
+    "Attribution Method Type": "Gradient",
+    "Model": "Llama-3.1-8B",
+    "Model Size": "8B",
+    "MMLU": 60,
+    "GSM8K": 59.5,
+    "BBH": 64.2,
+    "Paper/Code/Contact Link": ""
+  }
+]

data/DataSelection/pythia1b-10k-lambada.json

@@ -0,0 +1,8 @@
+[
+    {"Rank": 0, "Method": "Random", "Attribution Method Type": "Other", "Model": "Pythia-1b", "Model Size": "1B", "avg": 45.34, "sciq": 0.689, "arc_easy": 0.431, "arc_challenge": 0.244, "logiqa": 0.275, "boolq": 0.520, "hellaswag": 0.407, "piqa": 0.690, "winogrande": 0.535, "openbookqa": 0.290, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "BM25", "Attribution Method Type": "Lexical", "Model": "Pythia-1b", "Model Size": "1B", "avg": 45.72, "sciq": 0.692, "arc_easy": 0.439, "arc_challenge": 0.239, "logiqa": 0.260, "boolq": 0.556, "hellaswag": 0.406, "piqa": 0.696, "winogrande": 0.531, "openbookqa": 0.296, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Sim", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "avg": 45.98, "sciq": 0.689, "arc_easy": 0.440, "arc_challenge": 0.240, "logiqa": 0.272, "boolq": 0.556, "hellaswag": 0.406, "piqa": 0.690, "winogrande": 0.537, "openbookqa": 0.308, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Rep Sim", "Attribution Method Type": "Similarity", "Model": "Pythia-1b", "Model Size": "1B", "avg": 46.00, "sciq": 0.691, "arc_easy": 0.441, "arc_challenge": 0.237, "logiqa": 0.275, "boolq": 0.561, "hellaswag": 0.409, "piqa": 0.695, "winogrande": 0.537, "openbookqa": 0.294, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Mates", "Attribution Method Type": "Modeling", "Model": "Pythia-1b", "Model Size": "1B", "avg": 45.76, "sciq": 0.685, "arc_easy": 0.441, "arc_challenge": 0.241, "logiqa": 0.269, "boolq": 0.563, "hellaswag": 0.408, "piqa": 0.696, "winogrande": 0.523, "openbookqa": 0.292, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Edu", "Attribution Method Type": "Other", "Model": "Pythia-1b", "Model Size": "1B", "avg": 45.83, "sciq": 0.688, "arc_easy": 0.452, "arc_challenge": 0.240, "logiqa": 0.264, "boolq": 0.571, "hellaswag": 0.409, "piqa": 0.689, "winogrande": 0.520, "openbookqa": 0.292, "Paper/Code/Contact Link": ""}
+]

data/DataSelection/pythia1b-30k-lambada.json

@@ -0,0 +1,7 @@
+[
+    {"Rank": 0, "Method": "Random", "Attribution Method Type": "Other", "Model": "Pythia-1b", "Model Size": "1B", "avg": 47.49, "sciq": 0.695, "arc_easy": 0.456, "arc_challenge": 0.255, "logiqa": 0.276, "boolq": 0.602, "hellaswag": 0.475, "piqa": 0.711, "winogrande": 0.510, "openbookqa": 0.294, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Rep Sim", "Attribution Method Type": "Similarity", "Model": "Pythia-1b", "Model Size": "1B", "avg": 47.83, "sciq": 0.697, "arc_easy": 0.462, "arc_challenge": 0.259, "logiqa": 0.263, "boolq": 0.612, "hellaswag": 0.474, "piqa": 0.712, "winogrande": 0.526, "openbookqa": 0.300, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Grad Sim", "Attribution Method Type": "Gradient", "Model": "Pythia-1b", "Model Size": "1B", "avg": 47.92, "sciq": 0.711, "arc_easy": 0.465, "arc_challenge": 0.254, "logiqa": 0.273, "boolq": 0.606, "hellaswag": 0.475, "piqa": 0.711, "winogrande": 0.522, "openbookqa": 0.296, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Mates", "Attribution Method Type": "Modeling", "Model": "Pythia-1b", "Model Size": "1B", "avg": 47.64, "sciq": 0.702, "arc_easy": 0.464, "arc_challenge": 0.253, "logiqa": 0.260, "boolq": 0.617, "hellaswag": 0.474, "piqa": 0.708, "winogrande": 0.518, "openbookqa": 0.292, "Paper/Code/Contact Link": ""},
+    {"Rank": 0, "Method": "Edu", "Attribution Method Type": "Other", "Model": "Pythia-1b", "Model Size": "1B", "avg": 48.02, "sciq": 0.700, "arc_easy": 0.471, "arc_challenge": 0.262, "logiqa": 0.267, "boolq": 0.616, "hellaswag": 0.474, "piqa": 0.709, "winogrande": 0.511, "openbookqa": 0.312, "Paper/Code/Contact Link": ""}
+]

filePaths.py

@@ -0,0 +1,10 @@
+PRETRAIN_10K = "./data/DataSelection/pythia1b-10k-lambada.json"
+PRETRAIN_30K = "./data/DataSelection/pythia1b-30k-lambada.json"
+FINETUNE = "./data/DataSelection/finetune.json"
+
+TOXICITY_HOMOGENEOUS = "./data/Applications/toxicity-homogeneous.json"
+TOXICITY_HETEROGENEOUS = "./data/Applications/toxicity-heterogeneous.json"
+FACTUAL = "./data/Applications/factual.json"
+
+PATHLIST = [PRETRAIN_10K, PRETRAIN_30K, FINETUNE, TOXICITY_HOMOGENEOUS, TOXICITY_HETEROGENEOUS, FACTUAL]
+

nameMapping.py

@@ -0,0 +1,20 @@
+# Sub-Tab Display
+LEADERBOARD_NAMES = ["Pre-Training (10K)", 
+                     "Pre-Training (30K)",
+                     "Fine-Tuning", 
+                     "Homogeneous", "Heterogeneous", 
+                     "Factual Attribution"]
+
+TRAINING_LEADERBOARDS = {"Pre-Training (10K)", "Pre-Training (30K)", "Fine-Tuning"}
+
+# Submission Drop-Down Display
+DROPDOWN_NAME_MAPPING = {"toxicity": {"Homogeneous", "Heterogeneous"},
+                         "factual": {"Factual Attribution"},
+                         "finetune": {"Fine-Tuning"},
+                         "pretrain": {"Pre-Training (10K)", "Pre-Training (30K)"}}
+
+# Leaderboard Columns
+TOXICITY_COLS = ["Rank", "Method", "Attribution Method Type", "Model", "Model Size", "ToxicChat", "XSTest-response", "JailBreakBench", "AUPRC", "Paper/Code/Contact Link"]
+FACTUAL_COLS = ["Rank", "Method", "Attribution Method Type", "Model", "Model Size", "Recall@50", "MRR", "Paper/Code/Contact Link"]
+FINETUNE_COLS = ["Rank", "Method", "Attribution Method Type", "Model", "Model Size", "MMLU", "GSM8K", "BBH", "Paper/Code/Contact Link"]
+PRETRAIN_COLS = ["Rank", "Method", "Attribution Method Type", "Model", "Model Size", "avg", "sciq", "arc_easy", "arc_challenge", "logiqa", "boolq", "hellaswag", "piqa", "winogrande", "openbookqa", "Paper/Code/Contact Link"]

pr.py

@@ -0,0 +1,91 @@
+from github import Github
+from datetime import datetime, timezone
+import os
+import json
+import nameMapping
+
+###################### Push Up to Github #################################
+
+REPO_NAME = "DataAttributionEval/DATE-LM-Leaderboard"
+GITHUB_TOKEN = os.getenv("GITHUB_TOKEN")
+reviewer = "cathyjiao"
+
+def submit_and_open_PR(selected_leaderboard, *new_entry):
+    # Unpack data
+    (method_name, method_dropdown, model_name, model_size, paper_link, scores,
+     pre_avg, pre_sciq, pre_arc_easy, pre_arc_chall, pre_logiqa, 
+     pre_boolq, pre_hellaswag, pre_piqa, pre_wino, pre_open, 
+     fine_mmlu, fine_gsm, fine_bbh, 
+     tox_toxicChat, tox_xsTest, tox_jbb, tox_auprc, 
+     fac_recall, fac_mrr) = new_entry
+
+    # Save metadata
+    timestamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
+    branch_name = f"{timestamp}-{''.join(method_name.split())}-{method_name}"
+
+    submission_data = {
+        "Metadata Path": f"submissions/{branch_name}/metadata.json",
+        "Leaderboard": selected_leaderboard,
+        "Date of Submission": timestamp,
+        "Method": method_name,
+        "Attribution Method Type": method_dropdown,
+        "Model": model_name,
+        "Model Size": model_size,
+        "Paper/Code/Contact Link": paper_link
+    }
+
+    nameMap = nameMapping.DROPDOWN_NAME_MAPPING
+    if selected_leaderboard in nameMap['pretrain']:
+        fields = ["avg", "sciq", "arc_easy", "arc_challenge", "logiqa", \
+                  "boolq", "hellaswag", "piqa", "winogrande", "openbookqa"]
+        vals = [pre_avg, pre_sciq, pre_arc_easy, pre_arc_chall, pre_logiqa, \
+                pre_boolq, pre_hellaswag, pre_piqa, pre_wino, pre_open]
+        submission_data.update(dict(zip(fields, vals)))
+    elif selected_leaderboard in nameMap['finetune']:
+        submission_data["MMLU"] = fine_mmlu
+        submission_data["GSM8K"] = fine_gsm
+        submission_data["BBH"] = fine_bbh
+    elif selected_leaderboard in nameMap['toxicity']:
+        submission_data["ToxicChat"] = tox_toxicChat
+        submission_data["XSTest-response"] = tox_xsTest
+        submission_data["JailBreakBench"] = tox_jbb
+        submission_data["AUPRC"] = tox_auprc
+    elif selected_leaderboard in nameMap['factual']:
+        submission_data["Recall@50"] = fac_recall
+        submission_data["MRR"] = fac_mrr
+
+    g = Github(GITHUB_TOKEN)
+    repo = g.get_repo(REPO_NAME)
+
+    # Create a unique branch name
+    base = repo.get_branch("main")
+    repo.create_git_ref(ref=f"refs/heads/{branch_name}", sha=base.commit.sha)
+
+    # Upload score files
+    with open(scores.name, "rb") as f:
+        repo.create_file(
+            path=f"submissions/{branch_name}/{os.path.basename(scores.name)}",
+            message=f"Uploaded Scores File",
+            content=f.read(),
+            branch=branch_name
+    )
+
+    # Add PR metadata.json file
+    repo.create_file(
+        path=f"submissions/{branch_name}/metadata.json",
+        message="Submission Form Metadata",
+        content=json.dumps(submission_data, indent=2),
+        branch=branch_name
+    )
+
+    # Create pull request
+    pr = repo.create_pull(
+        title=f"[HF Leaderboard Submission] {method_name} for {selected_leaderboard}",
+        body=f"Auto-Generated Leaderboard Submission PR from HF Space\n{json.dumps(submission_data, indent=4)}",
+        head=branch_name,
+        base="main"
+    )
+    pr.add_to_labels("leaderboard-submission")
+    pr.create_review_request(reviewers=[reviewer])
+
+    return f"✅ PR created: {pr.html_url}"

requirements.txt

@@ -0,0 +1,4 @@
+gradio
+gradio_leaderboard
+pandas
+PyGithub

scripts/merge_data.py

@@ -0,0 +1,52 @@
+import json
+from pathlib import Path
+import sys
+import os
+
+def create_data_entry(submitted_json):
+    entry =  {
+        "Rank": 0,
+    }
+
+    submitted_dict = dict(submitted_json)
+    del submitted_dict["Metadata Path"]
+    del submitted_dict["Leaderboard"]
+    del submitted_dict["Date of Submission"]
+    
+    entry.update(submitted_dict)
+
+    return submitted_json.get("Leaderboard"), entry
+
+def merge_metadata(metadata_path):
+    file_map = {"Pre-Training (10K)": ['data', 'DataSelection', 'pythia1b-10k-lambada.json'],
+            "Pre-Training (30K)": ['data', 'DataSelection', 'pythia1b-30k-lambada.json'], 
+            "Fine-Tuning": ['data', 'DataSelection', 'finetune.json'],
+            "Homogeneous": ['data', 'Applications', 'toxicity-homogeneous.json'],
+            "Heterogeneous": ['data', 'Applications', 'toxicity-heterogeneous.json'],
+            "Factual Attribution": ['data', 'Applications', 'factual.json']
+           }
+
+    with open(metadata_path, "r") as f:
+        metadata_json = json.load(f)
+        leaderboard, entry = create_data_entry(metadata_json)
+        target_file = file_map.get(leaderboard)
+        repo_root = os.environ.get('GITHUB_WORKSPACE', os.getcwd())
+
+        target_path = os.path.join(repo_root, *target_file)
+        
+        if target_path is not None:
+            with open(target_path, "r") as f:
+                existing = json.load(f)
+        else:
+            # existing data should not be empty
+            raise ValueError("Could not retreive file content.")
+
+        existing.append(entry)
+
+        # Save updated list
+        with open(target_path, "w") as f:
+            json.dump(existing, f, indent=2)
+
+if __name__ == "__main__":
+    file_path = sys.argv[1]
+    merge_metadata(file_path)