Initial commit

README.md

@@ -1,6 +1,6 @@
 ---
 title: Noteworthy Differences
-emoji: 🏃
+emoji: 🔔
 colorFrom: red
 colorTo: pink
 sdk: gradio

app.py

@@ -0,0 +1,478 @@
+import gradio as gr
+from wiki_data_fetcher import (
+    get_previous_revisions,
+    get_revision_from_age,
+    get_wikipedia_introduction,
+    extract_revision_info,
+    get_revisions_behind,
+    get_random_wikipedia_title,
+)
+from models import classifier, judge
+import logfire
+from dotenv import load_dotenv
+
+# Load API keys
+load_dotenv()
+# Setup logging with Logfire
+logfire.configure()
+
+# If running a standalone Gradio app via `demo.launch()` within a script,
+# Logfire's auto-instrumentation for FastAPI is often automatically handled
+# if installed. If mounting within a separate FastAPI app, use:
+# logfire.instrument_fastapi(app)
+
+
+@logfire.instrument("Step 1: Fetch current revision")
+def fetch_current_revision(title: str):
+    """
+    Fetch current revision of a Wikipedia article and return its introduction.
+
+    Args:
+        title: Wikipedia article title
+
+    Returns:
+        Tuple of (introduction, timestamp)
+    """
+    if not title or not title.strip():
+        error_msg = "Please enter a Wikipedia page title."
+        raise gr.Error(error_msg, print_exception=False)
+        return None, None
+
+    try:
+        # Get current revision (revision 0)
+        json_data = get_previous_revisions(title, revisions=0)
+        revision_info = extract_revision_info(json_data, revision=0)
+
+        if not revision_info.get("revid"):
+            error_msg = f"Error: Could not find Wikipedia page '{title}'. Please check the title."
+            raise gr.Error(error_msg, print_exception=False)
+            return None, None
+
+        revid = revision_info["revid"]
+        timestamp = revision_info["timestamp"]
+
+        # Get introduction
+        introduction = get_wikipedia_introduction(revid)
+
+        if introduction is None:
+            introduction = f"Error: Could not retrieve introduction for current revision (revid: {revid})"
+
+        # Format timestamp for display
+        timestamp = f"**Timestamp:** {timestamp}" if timestamp else ""
+
+        # Return introduction text and timestamp
+        return introduction, timestamp
+
+    except Exception as e:
+        error_msg = f"Error occurred: {str(e)}"
+        raise gr.Error(error_msg, print_exception=False)
+        return None, None
+
+
+@logfire.instrument("Step 2: Fetch previous revision")
+def fetch_previous_revision(title: str, unit: str, number: int, new_revision: str):
+    """
+    Fetch previous revision of a Wikipedia article and return its introduction.
+
+    Args:
+        title: Wikipedia article title
+        unit: "days" or "revisions"
+        number: Number of days or revisions behind
+
+    Returns:
+        Tuple of (introduction, timestamp)
+    """
+
+    # If we get here with an empty new revision, then an error should have been raised
+    # in fetch_current_revision, so just return empty values without raising another error
+    if not new_revision:
+        return None, None
+
+    try:
+        # Get previous revision based on unit
+        if unit == "revisions":
+            json_data = get_previous_revisions(title, revisions=number)
+            revision_info = extract_revision_info(json_data, revision=number)
+        else:  # unit == "days"
+            revision_info = get_revision_from_age(title, age_days=number)
+
+        if not revision_info.get("revid"):
+            error_msg = f"Error: Could not find revision {number} {'revisions' if unit == 'revisions' else 'days'} behind for '{title}'."
+            raise gr.Error(error_msg, print_exception=False)
+            return None, None
+
+        revid = revision_info["revid"]
+        timestamp = revision_info["timestamp"]
+
+        # Get introduction
+        introduction = get_wikipedia_introduction(revid)
+
+        if introduction is None:
+            introduction = f"Error: Could not retrieve introduction for previous revision (revid: {revid})"
+
+        # Get revisions_behind
+        if unit == "revisions":
+            revisions_behind = number
+        else:
+            revisions_behind = get_revisions_behind(title, revid)
+            # For a negative number, replace the negative sign with ">"
+            if revisions_behind < 0:
+                revisions_behind = str(revisions_behind).replace("-", ">")
+
+        # Format timestamp for display
+        timestamp = (
+            f"**Timestamp:** {timestamp}, {revisions_behind} revisions behind"
+            if timestamp
+            else ""
+        )
+
+        # Return introduction text and timestamp
+        return introduction, timestamp
+
+    except Exception as e:
+        error_msg = f"Error occurred: {str(e)}"
+        raise gr.Error(error_msg, print_exception=False)
+        return None, None
+
+
+def run_classifier(old_revision: str, new_revision: str, prompt_style: str):
+    """
+    Run a classification model on the revisions.
+
+    Args:
+        old_revision: Old revision text
+        new_revision: New revision text
+        prompt_style: heuristic or few-shot
+
+    Returns:
+        Tuple of (noteworthy, rationale) (bool, str)
+    """
+
+    # Values to return if there is an error
+    noteworthy, rationale = None, None
+    if not old_revision or not new_revision:
+        return noteworthy, rationale
+
+    try:
+        # Run classifier model
+        result = classifier(old_revision, new_revision, prompt_style=prompt_style)
+        if result:
+            noteworthy = result.get("noteworthy", None)
+            rationale = result.get("rationale", "")
+        else:
+            error_msg = f"Error: Could not get {prompt_style} model result"
+            raise gr.Error(error_msg, print_exception=False)
+
+    except Exception as e:
+        error_msg = f"Error running model: {str(e)}"
+        raise gr.Error(error_msg, print_exception=False)
+
+    return noteworthy, rationale
+
+
+@logfire.instrument("Step 3a: Run heuristic classifier")
+def run_heuristic_classifier(old_revision: str, new_revision: str):
+    return run_classifier(old_revision, new_revision, prompt_style="heuristic")
+
+
+@logfire.instrument("Step 3b: Run few-shot classifier")
+def run_fewshot_classifier(old_revision: str, new_revision: str):
+    return run_classifier(old_revision, new_revision, prompt_style="few-shot")
+
+
+@logfire.instrument("Step 4: Run judge")
+def run_judge(
+    old_revision: str,
+    new_revision: str,
+    heuristic_rationale,
+    fewshot_rationale,
+    judge_mode: str,
+):
+    """
+    Run classification models and judge on the revisions.
+
+    Args:
+        old_revision: Old revision text
+        new_revision: New revision text
+        heuristic_rationale: Heuristic model's rationale
+        fewshot_rationale: Few-shot model's rationale
+        judge_mode: Mode for judge function ("unaligned", "aligned-fewshot", "aligned-heuristic")
+
+    Returns:
+        Tuple of (noteworthy, reasoning) (bool, str)
+    """
+
+    # Values to return if there is an error
+    noteworthy, reasoning = None, None
+    if (
+        not old_revision
+        or not new_revision
+        or not heuristic_rationale
+        or not fewshot_rationale
+    ):
+        return noteworthy, reasoning
+
+    try:
+        # Run judge
+        result = judge(
+            old_revision,
+            new_revision,
+            heuristic_rationale,
+            fewshot_rationale,
+            mode=judge_mode,
+        )
+        if result:
+            noteworthy = result.get("noteworthy", "")
+            reasoning = result.get("reasoning", "")
+        else:
+            error_msg = f"Error: Could not get judge's result"
+            raise gr.Error(error_msg, print_exception=False)
+
+    except Exception as e:
+        error_msg = f"Error running judge: {str(e)}"
+        raise gr.Error(error_msg, print_exception=False)
+
+    return noteworthy, reasoning
+
+
+def format_noteworthy(noteworthy, reasoning):
+    """
+    Format judge's noteworthy label as text.
+    """
+    if not reasoning:
+        # If the reasoning is empty, return nothing
+        return None
+    else:
+        # Format noteworthy boolean as text
+        return str(noteworthy)
+
+
+def compute_confidence(
+    heuristic_noteworthy,
+    fewshot_noteworthy,
+    judge_noteworthy,
+    heuristic_rationale,
+    fewshot_rationale,
+    judge_reasoning,
+):
+    """
+    Compute a confidence label using the noteworthy booleans.
+    """
+    # Return None if any of the rationales or reasoning is missing.
+    if not heuristic_rationale or not fewshot_rationale or not judge_reasoning:
+        return None
+    if heuristic_noteworthy == fewshot_noteworthy == judge_noteworthy:
+        # Classifiers and judge all agree
+        return "High"
+    elif heuristic_noteworthy != fewshot_noteworthy:
+        # Classifiers disagree, judge decides
+        return "Moderate"
+    else:
+        # Classifiers agree, judge vetoes
+        return "Questionable"
+
+
+# Setup theme without background image
+theme = gr.Theme.from_hub("NoCrypt/miku")
+theme.set(body_background_fill="#FFFFFF", body_background_fill_dark="#000000")
+
+# Create Gradio interface
+with gr.Blocks(theme=theme, title="Noteworthy Differences") as demo:
+    with gr.Row():
+        with gr.Column(scale=2):
+            gr.Markdown(
+                """
+            # Noteworthy Differences
+            Compare the current revision of a Wikipedia article (introduction only) with an old revision (number of days or revisions behind).<br>
+            Two classifier models, with relatively short heuristic and few-shot prompts, and a judge predict the noteworthiness of the differences.<br>
+            The judge has a longer prompt for AI alignment, also in heuristic or few-shot styles, produced as described in the
+            [GitHub repository](https://github.com/jedick/noteworthy-differences).
+            """
+            )
+        with gr.Column(scale=1):
+            gr.Markdown(
+                """#### Confidence Key
+                - **High:** heuristic = few-shot = judge
+                - **Moderate:** heuristic ≠ few-shot, judge decides
+                - **Questionable:** heuristic = few-shot, judge vetoes
+                """
+            )
+
+    with gr.Row():
+        title_input = gr.Textbox(
+            label="Wikipedia Page Title", placeholder="e.g., Albert Einstein", value=""
+        )
+        number_input = gr.Number(label="Number", value=100, minimum=0, precision=0)
+        unit_dropdown = gr.Dropdown(
+            choices=["days", "revisions"], value="days", label="Unit"
+        )
+        judge_mode_dropdown = gr.Dropdown(
+            choices=["unaligned", "aligned-fewshot", "aligned-heuristic"],
+            value="aligned-heuristic",
+            label="Judge Mode",
+        )
+        with gr.Column():
+            random_btn = gr.Button("Get Random Page Title")
+            submit_btn = gr.Button("Fetch Revisions and Run Model", variant="primary")
+
+    with gr.Row():
+        with gr.Column():
+            gr.Markdown("### Old Revision")
+            old_timestamp = gr.Markdown("")
+            old_revision = gr.Textbox(label="", lines=15, max_lines=30, container=False)
+            gr.Markdown(
+                """
+            - Page title is case-sensitive; use underscores or spaces.
+            - Specify any number of days or up to 499 revisions behind.
+            """
+            )
+
+        with gr.Column():
+            gr.Markdown("### Current Revision")
+            new_timestamp = gr.Markdown("")
+            new_revision = gr.Textbox(label="", lines=15, max_lines=30, container=False)
+
+        with gr.Column():
+            gr.Markdown("### Model Output")
+            heuristic_rationale = gr.Textbox(
+                label="Heuristic Model's Rationale",
+                lines=2,
+                max_lines=7,
+            )
+            fewshot_rationale = gr.Textbox(
+                label="Few-shot Model's Rationale",
+                lines=2,
+                max_lines=7,
+            )
+            judge_reasoning = gr.Textbox(
+                label="Judge's Reasoning",
+                lines=2,
+                max_lines=7,
+            )
+            with gr.Row(variant="default"):
+                noteworthy_text = gr.Textbox(
+                    label="Noteworthy Differences",
+                    lines=1,
+                    interactive=False,
+                )
+                confidence = gr.Textbox(
+                    label="Confidence",
+                    lines=1,
+                    interactive=False,
+                )
+            rerun_btn = gr.Button("Rerun Model", variant="primary")
+
+    # Hidden checkboxes to store boolean values
+    heuristic_noteworthy = gr.Checkbox(visible=False)
+    fewshot_noteworthy = gr.Checkbox(visible=False)
+    judge_noteworthy = gr.Checkbox(visible=False)
+
+    random_btn.click(
+        fn=get_random_wikipedia_title,
+        inputs=None,
+        outputs=[title_input],
+    )
+
+    gr.on(
+        # Press Enter in textbox or use button to submit
+        triggers=[title_input.submit, submit_btn.click],
+        # Clear the new_revision and new_timestamp values before proceeding.
+        # The empty values will propagate to the other components (through function return values) if there is an error.
+        fn=lambda: (gr.update(value=""), gr.update(value="")),
+        inputs=None,
+        outputs=[new_revision, new_timestamp],
+        api_name=False,
+    ).then(
+        fn=fetch_current_revision,
+        inputs=[title_input],
+        outputs=[new_revision, new_timestamp],
+        api_name=False,
+    ).then(
+        fn=fetch_previous_revision,
+        inputs=[title_input, unit_dropdown, number_input, new_revision],
+        outputs=[old_revision, old_timestamp],
+        api_name=False,
+    ).then(
+        fn=run_heuristic_classifier,
+        inputs=[old_revision, new_revision],
+        outputs=[heuristic_noteworthy, heuristic_rationale],
+        api_name=False,
+    ).then(
+        fn=run_fewshot_classifier,
+        inputs=[old_revision, new_revision],
+        outputs=[fewshot_noteworthy, fewshot_rationale],
+        api_name=False,
+    ).then(
+        fn=run_judge,
+        inputs=[
+            old_revision,
+            new_revision,
+            heuristic_rationale,
+            fewshot_rationale,
+            judge_mode_dropdown,
+        ],
+        outputs=[judge_noteworthy, judge_reasoning],
+        api_name=False,
+    ).then(
+        fn=format_noteworthy,
+        inputs=[judge_noteworthy, judge_reasoning],
+        outputs=[noteworthy_text],
+        api_name=False,
+    ).then(
+        fn=compute_confidence,
+        inputs=[
+            heuristic_noteworthy,
+            fewshot_noteworthy,
+            judge_noteworthy,
+            heuristic_rationale,
+            fewshot_rationale,
+            judge_reasoning,
+        ],
+        outputs=[confidence],
+        api_name=False,
+    )
+
+    # Rerun model when rerun button is clicked
+    gr.on(
+        triggers=[rerun_btn.click],
+        fn=run_heuristic_classifier,
+        inputs=[old_revision, new_revision],
+        outputs=[heuristic_noteworthy, heuristic_rationale],
+        api_name=False,
+    ).then(
+        fn=run_fewshot_classifier,
+        inputs=[old_revision, new_revision],
+        outputs=[fewshot_noteworthy, fewshot_rationale],
+        api_name=False,
+    ).then(
+        fn=run_judge,
+        inputs=[
+            old_revision,
+            new_revision,
+            heuristic_rationale,
+            fewshot_rationale,
+            judge_mode_dropdown,
+        ],
+        outputs=[judge_noteworthy, judge_reasoning],
+        api_name=False,
+    ).then(
+        fn=format_noteworthy,
+        inputs=[judge_noteworthy, judge_reasoning],
+        outputs=[noteworthy_text],
+        api_name=False,
+    ).then(
+        fn=compute_confidence,
+        inputs=[
+            heuristic_noteworthy,
+            fewshot_noteworthy,
+            judge_noteworthy,
+            heuristic_rationale,
+            fewshot_rationale,
+            judge_reasoning,
+        ],
+        outputs=[confidence],
+        api_name=False,
+    )
+
+if __name__ == "__main__":
+    demo.launch()

collect_data.py

@@ -0,0 +1,68 @@
+import time
+import csv
+from wiki_data_fetcher import (
+    get_previous_revisions,
+    extract_revision_info,
+    get_wikipedia_introduction,
+)
+
+title = []
+revid_0, revid_10, revid_100 = [], [], []
+ts_0, ts_10, ts_100 = [], [], []
+intro_0, intro_10, intro_100 = [], [], []
+
+
+if __name__ == "__main__":
+
+    # Open the file in read mode
+    with open("data/wikipedia_titles.txt", "r") as file:
+        # Iterate through each line in the file
+        for line in file:
+            # Get title from each line without trailing newline characters
+            this_title = line.strip()
+            print(this_title)
+            # Append title
+            title.append(this_title)
+            # Get info for most recent 100 revisions
+            json_data = get_previous_revisions(this_title, revisions=100)
+            # Append data for current revision
+            info_0 = extract_revision_info(json_data, 0)
+            revid_0.append(info_0["revid"])
+            ts_0.append(info_0["timestamp"])
+            intro_0.append(get_wikipedia_introduction(info_0["revid"]))
+            # Append data for 10th revision before current
+            info_10 = extract_revision_info(json_data, 10)
+            revid_10.append(info_10["revid"])
+            ts_10.append(info_10["timestamp"])
+            intro_10.append(get_wikipedia_introduction(info_10["revid"]))
+            # Append data for 100th revision before current
+            info_100 = extract_revision_info(json_data, 100)
+            revid_100.append(info_100["revid"])
+            ts_100.append(info_100["timestamp"])
+            intro_100.append(get_wikipedia_introduction(info_100["revid"]))
+
+            # Write the CSV in each loop in case we need to restart after an error
+            # Combine the lists
+            # fmt: off
+            export_data = zip(
+                title, revid_0, revid_10, revid_100,
+                ts_0, ts_10, ts_100, intro_0, intro_10, intro_100,
+            )
+            column_names = [
+                "title", "revid_0", "revid_10", "revid_100",
+                "ts_0", "ts_10", "ts_100",
+                "intro_0", "intro_10", "intro_100",
+            ]
+            # fmt: on
+
+            with open(
+                "data/wikipedia_introductions.csv", "w", newline="", encoding="utf-8"
+            ) as myfile:
+                wr = csv.writer(myfile)
+                # Write a header row
+                wr.writerow(column_names)
+                # Write the combined data rows
+                wr.writerows(export_data)
+
+            # Rate limit our API calls
+            time.sleep(5)

create_examples.py

@@ -0,0 +1,69 @@
+import pandas as pd
+from models import classifier
+
+
+def run_classifier(row):
+    """
+    Run the model on one row of data from 'data/wikipedia_introductions.csv'.
+    The model is run up to four times: two prompt styles (heuristic and few-shot)
+    and two revision intervals (from 10th and 100th previous revisions to current).
+
+    Usage:
+
+    df = pd.read_csv("data/wikipedia_introductions.csv")
+    row = df.iloc[38]
+    run_classifier(row)
+    """
+
+    # Initialize output dict
+    output = {}
+
+    output["heuristic_10"] = classifier(row["intro_10"], row["intro_0"], "heuristic")
+    output["few-shot_10"] = classifier(row["intro_10"], row["intro_0"], "few-shot")
+    output["heuristic_100"] = classifier(row["intro_100"], row["intro_0"], "heuristic")
+    output["few-shot_100"] = classifier(row["intro_100"], row["intro_0"], "few-shot")
+
+    return output
+
+
+if __name__ == "__main__":
+
+    """
+    Run the classifier on all rows from 'data/wikipedia_introductions.csv' and save results in 'data/examples.csv'.
+    """
+
+    # Read the data
+    df = pd.read_csv("data/wikipedia_introductions.csv")
+
+    # For reference: Find row indices with at least one missing value
+    # missing_rows = df.index[df.isnull().any(axis=1)].tolist()
+    # print("\nRow indices with missing values:", missing_rows)
+
+    # Initialize output data frame
+    df_out = None
+
+    for index, row in df.iterrows():
+        # Print the title to see progress
+        print(row["title"])
+        # Run classifier
+        output = run_classifier(row)
+        print(output)
+        # Create column names and row for data frame
+        column_names = [
+            outer_k + "_" + inner_k
+            for outer_k in output.keys()
+            for inner_k in output[outer_k].keys()
+        ]
+        row_values = [
+            inner_v for outer_k in output.keys() for inner_v in output[outer_k].values()
+        ]
+        # Add title to output
+        column_names = ["title"] + column_names
+        row_values = [row["title"]] + row_values
+        df_row = pd.DataFrame([row_values], columns=column_names)
+        if df_out is None:
+            df_out = df_row
+        else:
+            df_out = pd.concat([df_out, df_row])
+        # Write CSV in every loop to avoid data loss if errors occur
+        df_out.to_csv("data/examples.csv", index=False, encoding="utf-8")

data/alignment_fewshot.txt

@@ -0,0 +1,26 @@
+Model 1: The differences are minor grammatical adjustments and the addition of a more specific locality (Çağırkan) within the already stated Kaman District, which provides deeper detail but does not alter the fundamental understanding of the museum's location. Model 2: The new revision provides a more precise location for the museum by adding "Çağırkan," which enhances the factual accuracy and completeness of the article. Human: The addition of a city name adds detail but doesn’t change the main content. Noteworthy: False.
+Model 1: The differences are not noteworthy because the change only involves the removal of a single word ("lucky"), which is a minor word choice correction and does not alter the facts, people, places, dates, or analysis of the event. Model 2: The new revision removes the inappropriate word "lucky" when referring to the people killed, correcting the tone and factual accuracy of the sentence, which is a noteworthy change. Human: The word “lucky” is vandalism that radically alters the tone and destroys the neutrality of the article. Noteworthy: True.
+Model 1: The new revision adds "juvenile detention" and changes "sexual affairs" to "sexual awakening" as key life events for the protagonist István, which constitutes a change to major events in his life story. Model 2: The new revision adds minor details about the protagonist's journey and makes a few stylistic changes, but these do not substantially alter the overall summary or critical reception of the novel. Human: The change from “sexual affairs” to “sexual awakening” is a nuance in interpretation that doesn’t warrant a noteworthy label. Noteworthy: False.
+Model 1: The differences are not noteworthy as they only involve an update to the population data with a more recent projection, which constitutes a deeper analysis of a topic without changing the overall conclusion or major facts about the department. Model 2: The new revision updates the population data for Cundinamarca by adding 2023 projections, which is a significant improvement in accuracy and currency for an informational article. Human: The population projections for 2023 make the article five years more up-to-date, which is a significant change. Noteworthy: True.
+Model 1: The differences are not noteworthy because they primarily consist of deeper analysis and minor clarifications, such as adding pronunciation, specifying the French origin, and detailing seating configurations, rather than changing the fundamental definition or conclusion about what a coupe is. Model 2: The new revision significantly expands the definition of a 'coupe' by adding pronunciation, clarifying the typical door count, detailing seating configurations, and distinguishing it from a 'sports coupe', which provides a more comprehensive understanding of the term. Human: The changes are noteworthy for extending the definition of a coupe to include four seats and addition of different pronunciations in the US and elsewhere.   Noteworthy: True.
+Model 1: The differences are not noteworthy as they involve minor rephrasing, a slight broadening of scope for roof variations, and the removal of "cargo space" as a listed drawback, none of which substantially alters the overall conclusion or understanding of convertibles. Model 2: The new revision removes "cargo space" as a stated potential drawback of convertibles and rephrases the description of reduced structural rigidity, altering the completeness of the information regarding convertible disadvantages. Human: There are small changes like “models” vs “eras and manufacturers” and not mentioning cargo space in the new revision, but these don’t alter crucial information. Noteworthy: False.
+Model 1: The differences are noteworthy because they include changes to biographical details such as the addition of a middle name, the removal of a birthplace, and a change in nationality from English to British. Model 2: The new revision makes minor changes to the subject's full name, nationality, and birth details, which do not substantially affect the biographical overview. Human: The change from the birth city (Sheffield, South Yorkshire) to “British” is a matter of reporting depth rather than an actual change to the facts and therefore is not notable for most readers. Noteworthy: False.
+Model 1: The differences between the revisions are not noteworthy as they primarily consist of minor grammatical and wording changes, the removal of a specific historical detail about aluminium's use in the World Wars, and the addition of a sentence providing deeper analysis about its density, none of which alter the main conclusions or understanding of the topic. Model 2: The new revision is noteworthy because it adds a scientific detail explaining that aluminium's low standard atomic weight contributes to its low density, which is responsible for many of its uses, thus enhancing the reader's understanding of its properties and applications. Human: The new revision adds that “aluminium” is the preferred IUPAC name and also gives a reason for the relatively low density compared to other metals, both of which contribute to a more complete overview. Noteworthy: True.
+Model 1: The differences between the revisions are not noteworthy as they primarily involve minor rephrasing, changes in structure, and deeper analysis of the engine assembly process (such as the  Model 2: The new revision provides more specific and detailed information regarding the assembly process of the different engine variants, clarifying the 'one man, one engine' principle for AMG engines versus machine assembly for mainstream Mercedes-Benz models. Human: The new revision omits the detail about two blocks sharing a single crankshaft, but this is implied by the M133 ancestry, and also has less detail about where specific variants are manufactured, none of which are highly notable changes. Noteworthy: False.
+Model 1: The differences are noteworthy because the new revision adds Bayden Barber's birth year, which is a change to a date/personal detail. Model 2: The new revision adds a birth year and corrects a spelling of a Māori term, which are minor factual updates that do not substantially change the biographical overview. Human: Birth year is important information for a biographical entry. Noteworthy: True.
+Model 1: The differences are not noteworthy as they primarily consist of minor word choice changes, rephrasing, removal of descriptive details, and a clarification regarding Yorkshire's current administrative status and the specific dynasty of the House of York, none of which alter the overall meaning or conclusion about Yorkshire's history and identity. Model 2: The new revision clarifies that Yorkshire is no longer used for administration, adds important historical context about the "Plantagenet" royal House of York, and refines some geographical descriptions, leading to a more precise and updated overview of the region. Human: The mention of “no longer being used for administration” is a notable change in the new revision; text clean-up (less repetition of compass directions), and a name in the emblem’s origin (Plantagenet) are changes that would not be counted as noteworthy. Noteworthy: True.
+Model 1: The differences are not noteworthy because they consist of minor statistical updates (death toll), small factual additions (Lincoln as Republican, Lincoln giving Grant command), and a more explicit but not substantially different analysis of the Confederacy's motivation for secession related to slavery, which does not change the overall conclusion that slavery was the central cause of the war. Model 2: The new revision explicitly states that the Confederacy was formed to preserve African American slavery and clarifies that Lincoln gave Grant command of Union armies, providing a more precise and complete account of the war's origins and leadership. Human: These revisions introduce the article with quite different settings of the conflict (“expand in the western territories” vs “the election of Abraham Lincoln and the abolitionist movement in the North”), resulting in a noteworthy difference in historical outlook. Noteworthy: True.
+Model 1: The new revision adds a specific, technical detail about the duration of 1908, which provides a deeper analysis of the topic without changing the overall conclusion or introducing new people, places, dates, or major events. Model 2: The new revision adds a significant factual detail, stating that 1908 was the longest year in either the Julian or Gregorian calendars, measured in Terrestrial Time. Human: The fact that 1908 was the longest year in either calendar would be a piece of trivia in other contexts, but on the Wikipedia page for the year itself is a notable mention of a record-breaking characteristic. Noteworthy: True.
+Model 1: The new revision only adds "mass population displacement" as an additional contributing factor to the population decline, which is a minor detail that does not alter the overall analysis or conclusion of the article. Model 2: The new revision adds "mass population displacement" as a contributing factor to the sharp decline in the Congolese population, which provides a more complete understanding of the atrocities' impact. Human: The addition of mass population displacement as a cause of population decline is a noteworthy change in the new revision. Noteworthy: True.
+Model 1: The differences are not noteworthy as they consist of minor word choice changes like "locals" to "Indigenous people," grammatical adjustments, and the addition of the French name for the state, none of which alter the core facts, analysis, or conclusion of the article. Model 2: The new revision improves factual accuracy by adding the French name of the state and uses more precise and respectful terminology by changing "locals" to "Indigenous people" when referring to the inhabitants. Human: There are minor changes in wording (e.g. “went there” vs “visited it” and “local” vs “Indigenous”) that do not substantially alter the information presented.  Noteworthy: False.
+Model 1: The new revision adds 'and photographer' to the description of Howard H. Baker Jr.'s professions, which is a factual detail that offers a slightly deeper understanding of his overall life but does not change the main narrative or conclusions about his political and diplomatic career. Model 2: The new revision adds "and photographer" to Howard Henry Baker Jr.'s listed professions, which is a noteworthy addition to his biographical overview. Human: The new revision adds an occupation (photographer) that is notable for making a more complete biographical entry. Noteworthy: True.
+Model 1: The differences are not noteworthy as they involve minor word choice changes, the addition of a colloquial nickname for the city ("La Nevera"), and the removal of a general descriptive phrase ("one of the largest cities in the world"), none of which substantially alter the main facts, analysis, or conclusion about Bogotá. Model 2: The new revision removes the claim that Bogotá is "one of the largest cities in the world" and adds a new nickname, which significantly alters the description of the city's global standing and cultural information. Human: The new revision adds a nickname ("La Nevera" (The Fridge")) that makes the changes noteworthy; the ommision of minor trivia (“one of the largest cities in the world”) is not a notable change. Noteworthy: True.
+Model 1: The differences are not noteworthy because they involve minor rephrasing and the addition of a geographical detail (located along the North Pacific Ocean) that does not change the overall understanding or conclusion about Kushiro. Model 2: The new revision is noteworthy because it adds specific geographical information about Kushiro, stating that it is "Located along the coast of the North Pacific Ocean", which provides a more complete description of the city's location. Human: The new revision adds coastal location, an important characteristic that is not shared by all cities in Japan and that cannot be inferred from the old revision. Noteworthy: True.
+Model 1: The differences are not noteworthy as they involve minor grammatical corrections, stylistic changes, and the addition of a specific financial detail (cost cap) which provides deeper analysis without changing the overall conclusion about Formula One's financial landscape. Model 2: The new revision adds important information about the current F1 cost cap, providing a more complete financial overview of the sport beyond just the average team running costs, and also corrects a minor typo. Human: There is a small typo in the old revision (okFIA) that is not significant; however, the cost cap added to the new revision is a notable enhancement despite the poor grammar in the edit. Noteworthy: False.
+Model 1: The changes primarily involve adding alternative names for the Concordat of Worms and clarifying terminology (e.g., "temporalities"), while removing some extraneous initial text, none of which alter the core facts or conclusions about the agreement. Model 2: The new revision is noteworthy because it removes irrelevant headings, adds crucial alternative names and translations for the Concordat of Worms, and refines the terminology regarding the lands associated with episcopal sees, thereby significantly enhancing the article's clarity, comprehensiveness, and accuracy. Human: The addition of Latin and German names is a small but useful addition to the new revision. Noteworthy: True.
+Model 1: The differences include a minor clarification by adding "(cubes)" after "voxels" and an additional sentence about Minecraft being a cultural touchstone for Generation Z, which offers deeper analysis of the game's impact but does not alter the overall conclusion of the article. Model 2: The new revision adds a noteworthy point about Minecraft being a cultural touchstone for Generation Z, which enhances the understanding of the game's societal impact and popularization. Human: The reference to Generation Z in the new revision places the article in a firmer context. Noteworthy: True.
+Model 1: The differences are not noteworthy as they primarily consist of minor grammatical restructuring, slight changes in terminology (e.g., "Native Americans" to "Indigenous Americans"), and the addition of specific Cherokee script for a name, none of which alter the core factual information or analysis of the article's subject. Model 2: The new revision is noteworthy because it provides the actual Cherokee script and romanized version of Wilma Mankiller's name, replacing a generic template, and updates the terminology from "Native Americans" to "Indigenous Americans" in one instance, making the biographical overview more complete and current. Human: The addition of Cherokee and romanized names gives greater recognition to the cultural background of the subject of this article. Noteworthy: True.
+Model 1: The differences are not noteworthy because the new revision offers deeper analysis by clarifying specific human factors like the destruction of natural topsoil and adding context about an existing agricultural recession, but these details do not change the fundamental conclusions about the Dust Bowl's causes or effects. Model 2: The new revision provides a more detailed explanation of the human-made causes of the Dust Bowl, highlights its exacerbation of an existing agricultural recession, and includes more specific details about related cultural works, offering a more comprehensive account. Human: The new revision goes into greater depth on causes (destruction of natural topsoil by settlers) and consequences (exacerbated an already existing drought condition) but does not substantially change the main message. Noteworthy: False.
+Model 1: The differences are not noteworthy because the new revision primarily adds a parenthetical explanation for the cool wet season's role in soil moisture and removes a sentence about arid-zone agriculture development, without fundamentally changing the definition or analysis of dryland farming. Model 2: The new revision adds an important clarification about the role of the cool wet season in charging the soil with moisture for dryland farming and removes a sentence about arid-zone agriculture being developed, which significantly enhances the explanation of the agricultural technique. Human: The phrase about “which charges the soil with virtually all the moisture that the crops will receive prior to harvest” in the new revision is a clarification rather than a significant change. Noteworthy: False.
+Model 1: The differences are not noteworthy because they consist of minor additions, such as the descriptor 'landlocked' and the nickname 'Gateway to the West,' and a minor word change from 'makes up' to 'forms' for the eastern border, none of which alter the core information or analysis of the article. Model 2: The new revision adds the significant nickname "Gateway to the West" to the state's description, providing a more complete overview of how Missouri is historically and culturally recognized. Human: The new revision adds landlocked, but this can be inferred by the lack of an aquatic border, and changes the order of nicknames, which is a minor change. Noteworthy: False.
+Model 1: The differences are not noteworthy because the new revision only adds a more recent population estimate for 2024, which is a deeper analysis of the topic without changing the conclusion about Tracy's status. Model 2: The new revision includes an updated population estimate for 2024, providing more current information about the city's demographics. Human: Addition of population estimate in the new revision indicates the growth of the city in recent years. Noteworthy: True.

data/alignment_heuristic.txt

@@ -0,0 +1,55 @@
+## Noteworthy Changes (True)
+
+**Factual Additions/Corrections:**
+- Adding significant biographical details (birth year, occupations, cultural names)
+- Updating population data to be significantly more current (5+ years)
+- Adding record-breaking characteristics or superlatives relevant to the article subject
+- Inserting important geographical features (coastal location, landlocked status when not obvious)
+- Including cultural/historical context that wasn't inferable (nicknames with meaning, generational impact)
+
+**Tone & Neutrality:**
+- Removing vandalism or inappropriate language that destroys article neutrality
+- Correcting words that radically alter tone (e.g., "lucky" when referring to deaths)
+
+**Substantive Content Changes:**
+- Altering the framing of historical events (different causes or contexts presented)
+- Adding causes or factors to complex events (mass displacement, additional contributing factors)
+- Expanding definitions to include significant variations (four-door coupes, multiple pronunciations)
+- Adding technical/scientific explanations for key properties or phenomena
+
+**Cultural Recognition:**
+- Including indigenous language names/scripts that honor cultural background
+- Adding alternative official names (Latin, other languages) for historical agreements or places
+
+## Not Noteworthy Changes (False)
+
+**Minor Details:**
+- Grammatical adjustments, rephrasing, or structural reorganization
+- Adding locality specificity within already-stated regions (sub-district within district)
+- Removing minor trivia that doesn't affect core understanding
+- Adding nicknames without significant cultural/historical weight
+- Word choice refinements that don't change meaning ("locals" → "Indigenous people" as terminology update only)
+
+**Inferable Information:**
+- Details implied by other information (e.g., "landlocked" when no coastal borders mentioned)
+- Ancestry relationships that explain omitted technical details
+
+**Depth vs. Substance:**
+- Deeper analysis that doesn't change fundamental conclusions
+- Additional technical details that provide nuance but not new understanding
+- Clarifications of already-conveyed information
+- Minor statistical updates that don't represent significant temporal gaps
+
+**Administrative/Format Changes:**
+- Order changes in lists
+- Removal of extraneous text or typos (unless they substantially affected meaning)
+- Template replacements with equivalent content
+
+## Key Principles
+
+1. **Inferability Test**: If the new information could be reasonably inferred from the old revision, it's likely not noteworthy
+2. **Completeness vs. Depth**: Adding information that makes an entry more complete (new occupation, birth year) is noteworthy; adding depth to existing information usually isn't
+3. **Context Matters**: The same change can be noteworthy or not depending on article context (longest year trivia is noteworthy on a year's own page)
+4. **Temporal Significance**: Updates spanning 5+ years are noteworthy; minor year-to-year updates typically aren't
+5. **Framing Changes**: Alterations to how events/subjects are presented or understood are noteworthy; rewordings that preserve meaning aren't
+6. **Cultural Respect**: Additions recognizing cultural identity, indigenous languages, or heritage are noteworthy

judge_disagreements.py

@@ -0,0 +1,59 @@
+import sys
+import pandas as pd
+from models import judge
+
+if __name__ == "__main__":
+
+    """
+    Run the judge on all rows from 'data/disagreements_for_AI.csv' and save results in 'data/AI_judgments_unaligned.csv'.
+    """
+
+    # Read the data
+    df = pd.read_csv("data/disagreements_for_AI.csv")
+
+    # Add empty columns for AI judgments
+    df["noteworthy"] = None
+    df["reasoning"] = None
+
+    # We run the unaligned judge unless the script is called with --aligned-fewshot or --aligned--heuristic
+    mode = "unaligned"
+    outfile = "data/AI_judgments_unaligned.csv"
+    # Check if an argument was passed
+    if len(sys.argv) > 1:
+        # sys.argv[0] is the script name, sys.argv[1] is the first argument
+        argument = sys.argv[1]
+        if argument == "--aligned-fewshot":
+            mode = "aligned-fewshot"
+            outfile = "data/AI_judgments_fewshot.csv"
+        elif argument == "--aligned-heuristic":
+            mode = "aligned-heuristic"
+            outfile = "data/AI_judgments_heuristic.csv"
+        else:
+            raise ValueError(f"Unknown argument: {argument}")
+
+    print(f"Saving judgments to {outfile}")
+
+    for index, row in df.iterrows():
+        # Change this if needed (to restart after errors)
+        if index < 0:
+            next
+        else:
+            # Print the title to see progress
+            print(row["title"])
+            # Run judge
+            try:
+                output = judge(
+                    df.iloc[index]["old_revision"],
+                    df.iloc[index]["new_revision"],
+                    df.iloc[index]["heuristic_rationale"],
+                    df.iloc[index]["few-shot_rationale"],
+                    mode=mode,
+                )
+            except:
+                output = {"noteworthy": None, "reasoning": None}
+            print(output)
+            # Update data frame
+            df.at[index, "noteworthy"] = output["noteworthy"]
+            df.at[index, "reasoning"] = output["reasoning"]
+            # Write CSV in every loop to avoid data loss if errors occur
+            df.to_csv(outfile, index=False, encoding="utf-8")

models.py

@@ -0,0 +1,132 @@
+# Noteworthy Differences:
+# Classification of noteworthy differences between revisions of Wikipedia articles: an AI alignment project
+# 20251114 jmd version 1
+
+from google import genai
+from google.genai import types
+from pydantic import BaseModel
+from dotenv import load_dotenv
+import json
+import os
+import pandas as pd
+from prompts import analyzer_prompts, judge_prompt
+from retry_with_backoff import retry_with_backoff
+import logfire
+
+# Load API keys
+load_dotenv()
+
+# Setup Logfire
+logfire.configure()
+
+# This wraps Google Gen AI client calls
+# to capture prompts, responses, and metadata
+logfire.instrument_google_genai()
+
+# Initialize the Gemini LLM
+client = genai.Client()
+
+
+@retry_with_backoff()
+def classifier(old_revision, new_revision, prompt_style):
+    """
+    Classify noteworthy differences between revisions of a Wikipedia article
+
+    Args:
+        old_revision: Old revision of article
+        new_revision: New revision of article
+
+    Returns:
+        noteworthy: True if the differences are noteworthy; False if not
+        rationale: One-sentence rational for the classification
+    """
+
+    # Return None for missing revisions
+    if not pd.notna(old_revision) or not pd.notna(new_revision):
+        return {"noteworthy": None, "rationale": None}
+
+    # Get prompt template for given style
+    prompt_template = analyzer_prompts[prompt_style]
+
+    # Add article revisions to prompt
+    prompt = prompt_template.replace("{{old_revision}}", old_revision).replace(
+        "{{new_revision}}", new_revision
+    )
+
+    # Define response schema
+    class Response(BaseModel):
+        noteworthy: bool
+        rationale: str
+
+    # Generate response
+    response = client.models.generate_content(
+        model="gemini-2.5-flash",
+        contents=prompt,
+        config=types.GenerateContentConfig(
+            response_mime_type="application/json",
+            response_schema=Response.model_json_schema(),
+        ),
+    )
+
+    return json.loads(response.text)
+
+
+@retry_with_backoff()
+def judge(old_revision, new_revision, rationale_1, rationale_2, mode="unaligned"):
+    """
+    AI judge to settle disagreements between classification models
+
+    Args:
+        old_revision: Old revision of article
+        new_revision: New revision of article
+        rationale_1: Rationale provided by model 1 (i.e., heuristic prompt)
+        rationale_2: Rationale provided by model 2 (i.e., few-shot prompt)
+        mode: Prompt mode: unaligned, aligned-fewshot, or aligned-heuristic
+
+    Returns:
+        noteworthy: True if the differences are noteworthy; False if not
+        reasoning: One-sentence reason for the judgment
+    """
+
+    prompt = judge_prompt
+    # Add article revisions to prompt
+    prompt = prompt.replace("{{old_revision}}", old_revision).replace(
+        "{{new_revision}}", new_revision
+    )
+    # Add rationales to prompt
+    prompt = prompt.replace("{{model_1_rationale}}", rationale_1).replace(
+        "{{model_2_rationale}}", rationale_2
+    )
+
+    # Optionally add alignment text to prompt
+    if mode == "unaligned":
+        alignment_text = ""
+    elif mode == "aligned-fewshot":
+        with open("data/alignment_fewshot.txt", "r") as file:
+            lines = file.readlines()
+            alignment_text = "".join(lines)
+    elif mode == "aligned-heuristic":
+        with open("data/alignment_heuristic.txt", "r") as file:
+            lines = file.readlines()
+            alignment_text = "".join(lines)
+    else:
+        raise ValueError(f"Unknown mode: {mode}")
+
+    prompt = prompt.replace("{{alignment_text}}", alignment_text)
+
+    # Define response schema
+    class Response(BaseModel):
+        noteworthy: bool
+        reasoning: str
+
+    # Generate response
+    response = client.models.generate_content(
+        model="gemini-2.5-flash",
+        contents=prompt,
+        config=types.GenerateContentConfig(
+            response_mime_type="application/json",
+            response_schema=Response.model_json_schema(),
+        ),
+    )
+
+    return json.loads(response.text)

prompts.py

@@ -0,0 +1,91 @@
+skeleton = """
+You are a reading assistant tasked with finding noteworthy differences between revisions of a Wikipedia article.
+Decide if the differences between the old and new revisions are noteworthy.
+
+{{instructions}}
+
+Return a JSON-formatted response with keys for:
+    - 'noteworthy' (True if differences between revisions are noteworthy or False if they are not)
+    - 'rationale' (one sentence explaining why the differences are or are not noteworthy, including a summary of the differences)
+
+<old_revision>
+{{old_revision}}
+</old_revision>
+
+<new_revision>
+{{new_revision}}
+</new_revision>
+"""
+
+analyzer_prompts = {
+    "heuristic": skeleton.replace(
+        "{{instructions}}",
+        """
+Noteworthy differences are characterized by:
+    - Different people or places mentioned
+    - Changes to dates or major events
+    - Different analysis of a topic leading to a substantially different conclusion
+
+These are differences that are not noteworthy:
+    - Changes to grammar or minor word choice
+    - Different structure but same meaning
+    - Deeper analysis of a topic that does not change the conclusion
+""",
+    ),
+    "few-shot": skeleton.replace(
+        "{{instructions}}",
+        """
+Example of noteworthy differences:
+
+Old revision: David Szalay (/ˈsɒlɔɪ/; born 1974 in Montreal, Canada) is a Canadian born-Hungarian-British writer. His sixth novel, Flesh, won the 2025 Booker Prize.[1] 
+
+New revision: David Szalay (/ˈsɒlɔɪ/ SOL-oy; born January 1974) is a Canadian-born Hungarian-British writer. His novels All That Man Is[1] and Turbulence[2] are noted for their unique narrative structure, being collections of intertwined short stories. All That Man Is was shortlisted for the 2016 Man Booker Prize and won the 2016 Gordon Burn Prize. His sixth novel, Flesh,[3] won the 2025 Booker Prize.[4][5][6] 
+
+Rationale: The new revision provides more information about the author's work leading to a more complete biographical overview.
+
+Example of differences that are not noteworthy:
+
+Old revision: David Szalay (/ˈsɒlɔɪ/ SOL-oy; born January 1974) is a Canadian-born Hungarian-British writer. His novels All That Man Is and Turbulence are noted for their unique narrative structure, being collections of intertwined short stories. All That Man Is was shortlisted for the 2016 Man Booker Prize and won the 2016 Gordon Burn Prize. His sixth novel, Flesh, featured a more traditional narrative but was noted for its ability for readers to connect with its protagonist in spite of its sparse prose and dialogue. Flesh won the 2025 Booker Prize. 
+
+New revision: David Szalay (/ˈsɒlɔɪ/ SOL-oy; born January 1974) is a Canadian-born Hungarian-British writer. His novels All That Man Is[1] and Turbulence[2] are noted for their unique narrative structure, being collections of intertwined short stories. All That Man Is was shortlisted for the 2016 Man Booker Prize and won the 2016 Gordon Burn Prize. His sixth novel, Flesh,[3] won the 2025 Booker Prize.[4][5][6] 
+
+Rationale: The old revision analyzes a book in more depth but does not substantially affect the biographical overview.
+""",
+    ),
+}
+
+judge_prompt = """
+You are a judge tasked with using the output of two classification models together with human preferences to make a final decision.
+The models were asked to provide rationales about whether noteworthy differences exist between old and new revisions of a Wikipedia article.
+
+If the models disagree:
+    Use the rationales and article revisions to make an informed judgment about which model is correct.
+
+If the models agree:
+    You may veto the models and change the label only if there would be strong human preference to do so.
+
+In both cases, align your response to human preferences (if available) and state how this affects your reasoning.
+Use the examples (if available) to infer patterns of human preference that can be generalized to the topics and situations in any article.
+
+{{alignment_text}}
+
+Return a JSON-formatted response with keys for:
+    - 'noteworthy' (True if differences between revisions are noteworthy or False if they are not)
+    - 'reasoning' (one sentence explaining how you made the judgment)
+
+<old_revision>
+{{old_revision}}
+</old_revision>
+
+<new_revision>
+{{new_revision}}
+</new_revision>
+
+<model_1_rationale>
+{{model_1_rationale}}
+</model_1_rationale>
+
+<model_2_rationale>
+{{model_2_rationale}}
+</model_2_rationale>
+"""

requirements.txt

@@ -0,0 +1,8 @@
+google-genai
+pydantic
+pandas
+dotenv
+gradio
+requests
+logfire
+opentelemetry-instrumentation-google-genai

retry_with_backoff.py

@@ -0,0 +1,42 @@
+import time
+import functools
+import random
+
+
+def retry_with_backoff(
+    max_retries=5, base_delay=2, backoff_factor=2, exceptions=(Exception,)
+):
+    """
+    Decorator to retry a function with exponential backoff.
+
+    Args:
+        max_retries (int): Maximum number of retries before giving up.
+        base_delay (float): Initial delay in seconds before retrying.
+        backoff_factor (float): Multiplier for delay after each failure.
+        exceptions (tuple): Exception types to catch and retry on.
+    """
+
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            delay = base_delay
+            attempt = 0
+            while attempt < max_retries:
+                try:
+                    # Pass args and kwargs
+                    return func(*args, **kwargs)
+                except exceptions as e:
+                    attempt += 1
+                    if attempt >= max_retries:
+                        # Raise the last exception if max retries reached
+                        raise
+                    print(
+                        f"[Retry {attempt}/{max_retries}] Error: {e}. Retrying in {delay:.2f}s..."
+                    )
+                    time.sleep(delay)
+                    # Exponential backoff with jitter
+                    delay *= backoff_factor + random.uniform(0, 1)
+
+        return wrapper
+
+    return decorator

test_models.py

@@ -0,0 +1,131 @@
+from models import classifier, judge
+from dotenv import load_dotenv
+import logfire
+
+
+# Load API keys
+load_dotenv()
+# Setup Logfire
+# We need send_to_logfire=True to capture traces under Pytest
+# https://logfire.pydantic.dev/docs/reference/advanced/testing/
+logfire.configure(send_to_logfire=True)
+
+
+def classifier_logic(i):
+    """
+    Return scenario flags for heuristic/few-shot classifier outputs.
+
+    Args:
+        i: Current iteration (for logging)
+
+    """
+
+    old_revision = """Henry Purcell (/ˈpɜːrsəl/, rare: /pərˈsɛl/;[n 1] c. 10 September 1659[n 2] – 21 November 1695) was an English composer of Baroque music.  He composed more than 100 songs, a tragic opera Dido and Aeneas, and wrote incidental music to a version of Shakespeare's A Midsummer Night's Dream called The Fairy Queen."""
+
+    new_revision = """Henry Purcell (/ˈpɜːrsəl/, rare: /pərˈsɛl/;[n 1] c. 10 September 1659[n 2] – 21 November 1695) was an English composer and organist of the middle Baroque era.  He composed more than 100 songs, a tragic opera Dido and Aeneas, and wrote incidental music to a version of Shakespeare's A Midsummer Night's Dream called The Fairy Queen."""
+
+    with logfire.span("classifier_logic {i}", i=i):
+        # Run classifier models
+        heuristic = classifier(old_revision, new_revision, "heuristic")
+        few_shot = classifier(old_revision, new_revision, "few-shot")
+        heuristic_true = heuristic["noteworthy"] is True
+        few_shot_true = few_shot["noteworthy"] is True
+
+        only_heuristic_true = heuristic_true and not few_shot_true
+        only_few_shot_true = few_shot_true and not heuristic_true
+        both_true = heuristic_true and few_shot_true
+        both_false = (heuristic_true is False) and (few_shot_true is False)
+
+    return (
+        only_heuristic_true,
+        only_few_shot_true,
+        both_true,
+        both_false,
+    )
+
+
+def judge_logic(i):
+    """
+    Return scenario flags for judge outputs.
+
+    Args:
+        i: Current iteration (for logging)
+
+    """
+
+    old_revision = """Kaman-Kalehöyük Archaeological Museum (Turkish: Kaman-Kalehöyük Arkeoloji Müzesi) is an archaeological museum in Kaman District of Kırşehir Province in Turkey. It exhibits artifacts of seven civilizations excavated in the nearby multi-period mound Kaman-Kalehöyük. It was opened in 2010. A Japanese garden is next to the museum building.[1][2]"""
+
+    new_revision = """The Kaman-Kalehöyük Archaeological Museum (Turkish: Kaman-Kalehöyük Arkeoloji Müzesi) is an archaeological museum in Çağırkan, Kaman District, Kırşehir Province, Turkey. It exhibits artifacts of seven civilizations excavated in the nearby multi-period mound Kaman-Kalehöyük. It opened in 2010. A Japanese garden is next to the museum building.[1][2]"""
+
+    with logfire.span("judge_logic {i}", i=i):
+        heuristic = classifier(old_revision, new_revision, "heuristic")
+        few_shot = classifier(old_revision, new_revision, "few-shot")
+        judge_few_shot = judge(
+            old_revision,
+            new_revision,
+            heuristic["rationale"],
+            few_shot["rationale"],
+            mode="aligned-fewshot",
+        )
+        judge_heuristic = judge(
+            old_revision,
+            new_revision,
+            heuristic["rationale"],
+            few_shot["rationale"],
+            mode="aligned-heuristic",
+        )
+
+    # Test condition is True if aligned judges both give False
+    judge_condition = (
+        judge_few_shot["noteworthy"] == False and judge_heuristic["noteworthy"] == False
+    )
+
+    return judge_condition
+
+
+# pytest -vv test_models.py::test_classifier
+def test_classifier():
+    """Run classifier logic 5 times and compare outcomes."""
+    tries = 5
+    with logfire.span("test_classifier"):
+        outcomes = [classifier_logic(i) for i in range(tries)]
+
+    only_heuristic_true = sum(result[0] for result in outcomes)
+    only_few_shot_true = sum(result[1] for result in outcomes)
+    both_true = sum(result[2] for result in outcomes)
+    both_false = sum(result[3] for result in outcomes)
+
+    heuristic_true_count = only_heuristic_true + both_true
+    few_shot_true_count = only_few_shot_true + both_true
+    disagree_count = only_heuristic_true + only_few_shot_true
+    agree_count = both_true + both_false
+
+    few_shot_more_often = few_shot_true_count > heuristic_true_count
+    disagree_more_than_agree = disagree_count > agree_count
+
+    if not few_shot_more_often:
+        print(
+            "Few-shot classifier did not return True more often than the heuristic classifier."
+        )
+    if not disagree_more_than_agree:
+        print("Classifiers did not disagree more often than they agreed.")
+
+    assert few_shot_more_often and disagree_more_than_agree
+
+
+# pytest -vv test_models.py::test_judge
+def test_judge():
+    """Run judge logic up to 5 times"""
+    current_try = 0
+    max_trys = 5
+    with logfire.span("test_judge"):
+        while current_try < max_trys:
+            result = judge_logic(current_try)
+            current_try += 1
+            if result is True:
+                print(f"Try {current_try} succeeded")
+                break
+            else:
+                print(f"Try {current_try} failed")
+    # The assert for pytest
+    assert result is True

test_workflows.py

@@ -0,0 +1,30 @@
+from workflows import llm_workflow
+
+
+def llm_workflow_logic():
+
+    old_revision = """Kaman-Kalehöyük Archaeological Museum (Turkish: Kaman-Kalehöyük Arkeoloji Müzesi) is an archaeological museum in Kaman District of Kırşehir Province in Turkey. It exhibits artifacts of seven civilizations excavated in the nearby multi-period mound Kaman-Kalehöyük. It was opened in 2010. A Japanese garden is next to the museum building.[1][2]"""
+
+    new_revision = """The Kaman-Kalehöyük Archaeological Museum (Turkish: Kaman-Kalehöyük Arkeoloji Müzesi) is an archaeological museum in Çağırkan, Kaman District, Kırşehir Province, Turkey. It exhibits artifacts of seven civilizations excavated in the nearby multi-period mound Kaman-Kalehöyük. It opened in 2010. A Japanese garden is next to the museum building.[1][2]"""
+
+    response = llm_workflow(old_revision, new_revision, "aligned-fewshot")
+
+    # The judge should responsd with noteworthy: False regardless of the classifier models' responses
+    return response["judge"]["noteworthy"] is False
+
+
+# pytest -vv test_workflows.py::test_llm_workflow
+def test_llm_workflow():
+    """Run LLM workflow logic up to 5 times"""
+    current_try = 0
+    max_trys = 5
+    while current_try < max_trys:
+        current_try += 1
+        result = llm_workflow_logic()
+        if result is True:
+            print(f"Try {current_try} succeeded")
+            break
+        else:
+            print(f"Try {current_try} failed")
+    # The actual test for pytest
+    assert result is True

wiki_data_fetcher.py

@@ -0,0 +1,339 @@
+import requests
+from datetime import datetime, timedelta
+from typing import Dict, Optional
+import re
+
+
+def run_get_request(params: dict):
+    """
+    Utility function to run GET request against Wikipedia API
+    """
+    base_url = "https://en.wikipedia.org/w/api.php"
+
+    # We need to supply headers for the request to work
+    headers = {
+        "User-Agent": f"NoteworthyDifferences/1.0 (j3ffdick@gmail.com) requests/{requests.__version__}"
+    }
+
+    response = requests.get(base_url, params=params, headers=headers)
+    # Handle HTTP errors
+    response.raise_for_status()
+
+    try:
+        json_data = response.json()
+    except Exception:
+        raise ValueError(f"Unable to parse response: {response}")
+
+    return json_data
+
+
+def extract_revision_info(json_data, revision=0):
+    """
+    Utility function to extract page revision info from JSON data returned from API call
+
+    Args:
+        revision: revision before current
+
+    Examples:
+        title = 'David_Szalay'
+        json_data = get_previous_revisions(title, revisions = 100)
+        extract_revision_info(json_data)       # Current revision
+        extract_revision_info(json_data, 10)   # 10th revision before current
+        extract_revision_info(json_data, 100)  # 10th revision before current
+    """
+    # Extract page and revision info
+    pages = json_data["query"]["pages"]
+    page_id = list(pages.keys())[0]
+
+    if page_id == "-1":
+        # Page not found, return empty dict
+        return {"revid": None, "timestamp": None}
+
+    try:
+        # Get the specified revision
+        revision = pages[page_id]["revisions"][revision]
+        revid = revision["revid"]
+        timestamp = revision["timestamp"]
+    except:
+        # Revision not found, return empty dict
+        return {"revid": None, "timestamp": None}
+
+    # NOTUSED: Create permanent URL
+    # permanent_url = f"https://en.wikipedia.org/w/index.php?title={title}&oldid={revid}"
+
+    # Remove the parentid key because we don't use it
+    _ = revision.pop("parentid", None)
+    return revision
+
+
+def get_revision_from_age(title: str, age_days: int = 0) -> Dict[str, str]:
+    """
+    Get the revision info of a Wikipedia article closest to the age in days.
+
+    Args:
+        title: Wikipedia article title (e.g., 'David_Szalay')
+        age_days: Age of the article revision in days (0 for current)
+
+    Returns:
+        Dictionary containing:
+        - 'revid': Revision id of the article revision
+        - 'timestamp': Timestamp of the article revision
+    """
+
+    # Get the target date
+    target_date = datetime.utcnow() - timedelta(days=age_days)
+
+    # Get the revision closest to the target date
+    params = {
+        "action": "query",
+        "titles": title,
+        "prop": "revisions",
+        "rvlimit": 1,
+        "rvdir": "older",
+        "rvstart": target_date.isoformat() + "Z",
+        "rvprop": "ids|timestamp",
+        "format": "json",
+    }
+
+    # Run GET request
+    json_data = run_get_request(params)
+
+    # Return revision info
+    return extract_revision_info(json_data)
+
+
+def get_previous_revisions(title: str, revisions: int = 0) -> Dict[str, str]:
+    """
+    Get the revision info of a Wikipedia article a certain number of revisions before the current one.
+
+    Args:
+        title: Wikipedia article title (e.g., 'David_Szalay')
+        revision: What revision before current (0 for current, must be between 0 and 499)
+
+    Returns:
+        Dictionary containing:
+        - 'revid': Revision id of the article revision
+        - 'timestamp': Timestamp of the article revision
+
+    Note:
+        In the Wikipedia API, rvlimit is how many revisions will be returned and must be between 1 and 500
+        rvlimit = 1 returns a single revision: the current one
+        rvlimit = 101 returns the 100 most recent revisions and the current one
+        This is why we use rvlimit = revision + 1
+    """
+
+    # Get the revision closest to the target date
+    params = {
+        "action": "query",
+        "prop": "revisions",
+        "titles": title,
+        "rvlimit": revisions + 1,
+        "rvdir": "older",
+        "rvprop": "ids|timestamp",
+        "format": "json",
+    }
+
+    # Run GET request
+    json_data = run_get_request(params)
+
+    # Return info for all revisions
+    return json_data
+
+
+def get_wikipedia_introduction(revid: int) -> Dict[str, str]:
+    """
+    Retrieve the introduction of a Wikipedia article.
+
+    Args:
+        revid: Revision id of the article
+
+    Returns:
+        Text of the introduction
+
+    Example:
+        # Get intro from current article revision
+        revision_info = get_revision_from_age("David_Szalay")
+        get_wikipedia_introduction(revision_info["revid"])
+    """
+
+    # Return None for missing revid
+    if not revid:
+        return None
+
+    # Get the content of this specific revision
+    params = {"action": "parse", "oldid": revid, "prop": "text", "format": "json"}
+
+    json_data = run_get_request(params)
+
+    # Sometimes a revision is deleted and can't be viewed
+    # E.g. revid = '1276494621' for Turin
+    try:
+        html_content = json_data["parse"]["text"]["*"]
+    except:
+        return None
+
+    # Extract introduction (text before first section heading)
+    # Remove everything from the first <h2> tag onwards
+    intro_html = re.split(r"<h2", html_content, maxsplit=1)[0]
+
+    # Extract text from paragraphs, excluding certain elements
+    from html.parser import HTMLParser
+
+    class IntroParser(HTMLParser):
+        def __init__(self):
+            super().__init__()
+            self.text = []
+            self.in_p = False
+            self.skip = False
+
+        def handle_starttag(self, tag, attrs):
+            if tag == "p":
+                self.in_p = True
+            # Skip certain elements
+            if tag in ["style", "script", "table", "div"]:
+                attrs_dict = dict(attrs)
+                # Skip infoboxes, navboxes, etc.
+                if "class" in attrs_dict:
+                    if any(
+                        x in attrs_dict["class"]
+                        for x in ["infobox", "navbox", "metadata", "toc"]
+                    ):
+                        self.skip = True
+                if tag in ["style", "script"]:
+                    self.skip = True
+
+        def handle_endtag(self, tag):
+            if tag == "p":
+                if self.in_p and self.text and not self.text[-1].endswith("\n\n"):
+                    self.text.append("\n\n")
+                self.in_p = False
+            if tag in ["style", "script", "table", "div"]:
+                self.skip = False
+
+        def handle_data(self, data):
+            if self.in_p and not self.skip:
+                # *Don't* clean up whitespace here - it makes run-on words
+                # text = " ".join(data.split())
+                text = data
+                if text:
+                    self.text.append(text)
+
+    parser = IntroParser()
+    parser.feed(intro_html)
+
+    # Join and clean up the text
+    introduction = "".join(parser.text).strip()
+
+    # Remove multiple newlines
+    introduction = re.sub(r"\n{3,}", "\n\n", introduction)
+
+    # Remove empty paragraphs
+    paragraphs = [p.strip() for p in introduction.split("\n\n") if p.strip()]
+    introduction = "\n\n".join(paragraphs)
+
+    return introduction
+
+
+def get_revisions_behind(title: str, revid: int) -> int:
+    """
+    Get the number of revisions a given revid is behind the current revision of the page.
+
+    Args:
+        revid: Revision ID of the page
+
+    Returns:
+        Integer representing the number of revisions back (0 if it's the current revision)
+
+    Example:
+        # Get how many revisions behind a specific revid is
+        revisions_behind = get_revisions_behind(123456789)
+    """
+
+    ## First, get the page title from the revid
+    # params = {"action": "parse", "oldid": revid, "prop": "title", "format": "json"}
+    # try:
+    #    json_data = run_get_request(params)
+    #    title = json_data["parse"]["title"]
+    # except Exception:
+    #    # If we can't get the title, the revid might be invalid
+    #    raise ValueError(f"Could not retrieve page title for revid {revid}. The revid may be invalid or deleted.")
+
+    # Search through revisions going back from current
+    # We'll paginate through results if needed
+    revision_count = 0
+    continue_token = None
+
+    # Run the loop twice to get up to 1000 revisions behind
+    for i in range(2):
+        params = {
+            "action": "query",
+            "titles": title,
+            "prop": "revisions",
+            "rvlimit": 500,  # API limit per request
+            "rvdir": "older",
+            "rvprop": "ids",
+            "format": "json",
+        }
+
+        if continue_token:
+            params["rvcontinue"] = continue_token
+
+        try:
+            json_data = run_get_request(params)
+            pages = json_data["query"]["pages"]
+            page_id = list(pages.keys())[0]
+
+            if page_id == "-1":
+                raise ValueError(f"Page not found for revid {revid}")
+
+            revisions = pages[page_id]["revisions"]
+
+            # Find the index of the given revid in the current batch of revisions
+            for i, revision in enumerate(revisions):
+                if revision["revid"] == revid:
+                    return revision_count + i
+
+            # Update the count of revisions we've checked
+            revision_count += len(revisions)
+
+            # Check if there are more revisions to search
+            continue_token = json_data.get("continue", {}).get("rvcontinue")
+
+            if not continue_token:
+                # Reached the end of revisions but didn't find the revid
+                raise ValueError(
+                    f"Revid {revid} not found in the revision history of the page. "
+                    f"It may be from a different page or may have been deleted."
+                )
+
+        except ValueError:
+            # Re-raise ValueError exceptions
+            raise
+        except Exception as e:
+            raise ValueError(f"Error searching for revid {revid}: {e}")
+
+    # If we looped without returning the revision count, return it as a negative number
+    negative_revision_count = -revision_count
+    return negative_revision_count
+
+
+def get_random_wikipedia_title():
+    url = "https://en.wikipedia.org/w/api.php"
+    params = {
+        "action": "query",
+        "list": "random",
+        "rnnamespace": 0,
+        "rnlimit": 1,
+        "format": "json",
+    }
+
+    try:
+        json_data = run_get_request(params)
+
+        # Extract the title
+        title = json_data["query"]["random"][0]["title"]
+        return title
+
+    except requests.RequestException as e:
+        print(f"Error fetching random Wikipedia title: {e}")
+        return None

workflows.py

@@ -0,0 +1,23 @@
+from models import classifier, judge
+
+
+def llm_workflow(old_revision, new_revision, mode="aligned-fewshot"):
+    """
+    Run LLM workflow (input to response)
+
+    Args:
+        mode: "aligned-fewshot" for few-shot alignment or "aligned-heuristic" for heuristic alignment
+    """
+
+    # Run classifier and judge models
+    heuristic = classifier(old_revision, new_revision, "heuristic")
+    few_shot = classifier(old_revision, new_revision, "few-shot")
+    judge_response = judge(
+        old_revision,
+        new_revision,
+        heuristic["rationale"],
+        few_shot["rationale"],
+        mode=mode,
+    )
+
+    return {"heuristic": heuristic, "few-shot": few_shot, "judge": judge_response}