judge_disagreements.py

import sys
import pandas as pd
from models import judge

if __name__ == "__main__":

    """
    Run the judge on all rows from 'data/disagreements_for_AI.csv' and save results in 'data/AI_judgments_unaligned.csv'.
    """

    # Read the data
    df = pd.read_csv("data/disagreements_for_AI.csv")

    # Add empty columns for AI judgments
    df["noteworthy"] = None
    df["reasoning"] = None

    # We run the unaligned judge unless the script is called with --aligned-fewshot or --aligned--heuristic
    mode = "unaligned"
    outfile = "data/AI_judgments_unaligned.csv"
    # Check if an argument was passed
    if len(sys.argv) > 1:
        # sys.argv[0] is the script name, sys.argv[1] is the first argument
        argument = sys.argv[1]
        if argument == "--aligned-fewshot":
            mode = "aligned-fewshot"
            outfile = "data/AI_judgments_fewshot.csv"
        elif argument == "--aligned-heuristic":
            mode = "aligned-heuristic"
            outfile = "data/AI_judgments_heuristic.csv"
        else:
            raise ValueError(f"Unknown argument: {argument}")

    print(f"Saving judgments to {outfile}")

    for index, row in df.iterrows():
        # Change this if needed (to restart after errors)
        if index < 0:
            next
        else:
            # Print the title to see progress
            print(row["title"])
            # Run judge
            try:
                output = judge(
                    df.iloc[index]["old_revision"],
                    df.iloc[index]["new_revision"],
                    df.iloc[index]["heuristic_rationale"],
                    df.iloc[index]["few-shot_rationale"],
                    mode=mode,
                )
            except:
                output = {"noteworthy": None, "reasoning": None}
            print(output)
            # Update data frame
            df.at[index, "noteworthy"] = output["noteworthy"]
            df.at[index, "reasoning"] = output["reasoning"]
            # Write CSV in every loop to avoid data loss if errors occur
            df.to_csv(outfile, index=False, encoding="utf-8")