Add evaluation scripts and result artifacts

evaluation/custom_argument_eval_metrics.json

@@ -0,0 +1,79 @@
+{
+  "n_examples": 48,
+  "accuracy": 0.6666666666666666,
+  "macro_f1": 0.6626780626780626,
+  "classification_report": {
+    "claim": {
+      "precision": 1.0,
+      "recall": 0.5,
+      "f1-score": 0.6666666666666666,
+      "support": 12.0
+    },
+    "counter_claim": {
+      "precision": 0.7333333333333333,
+      "recall": 0.9166666666666666,
+      "f1-score": 0.8148148148148148,
+      "support": 12.0
+    },
+    "premise": {
+      "precision": 0.7142857142857143,
+      "recall": 0.8333333333333334,
+      "f1-score": 0.7692307692307693,
+      "support": 12.0
+    },
+    "unknown": {
+      "precision": 0.38461538461538464,
+      "recall": 0.4166666666666667,
+      "f1-score": 0.4,
+      "support": 12.0
+    },
+    "accuracy": 0.6666666666666666,
+    "macro avg": {
+      "precision": 0.7080586080586081,
+      "recall": 0.6666666666666666,
+      "f1-score": 0.6626780626780626,
+      "support": 48.0
+    },
+    "weighted avg": {
+      "precision": 0.7080586080586079,
+      "recall": 0.6666666666666666,
+      "f1-score": 0.6626780626780627,
+      "support": 48.0
+    }
+  },
+  "confusion_matrix": {
+    "labels": [
+      "claim",
+      "counter_claim",
+      "premise",
+      "unknown"
+    ],
+    "matrix": [
+      [
+        6,
+        0,
+        0,
+        6
+      ],
+      [
+        0,
+        11,
+        0,
+        1
+      ],
+      [
+        0,
+        1,
+        10,
+        1
+      ],
+      [
+        0,
+        3,
+        4,
+        5
+      ]
+    ]
+  },
+  "predictions_csv": "evaluation\\custom_argument_eval_predictions.csv"
+}

evaluation/custom_argument_eval_predictions.csv

@@ -0,0 +1,49 @@
+example_id,topic,parent_text,current_text,gold_label,rationale,probe_type,pred_label,correct,source_url
+eval_001,AI in education,,Universities should allow students to use generative AI tools in coursework.,claim,Root-level debatable position.,root_claim,unknown,False,
+eval_002,AI in education,Universities should allow students to use generative AI tools in coursework.,AI tools should be banned from graded assignments because they make authorship impossible to verify.,counter_claim,Directly opposes the parent position.,direct_disagreement,counter_claim,True,
+eval_003,AI in education,AI tools should be banned from graded assignments because they make authorship impossible to verify.,Detection systems have produced false positives against students who did not use AI.,premise,Gives a supporting reason against relying on bans or detection.,evidence_reason,premise,True,
+eval_004,AI in education,Detection systems have produced false positives against students who did not use AI.,Do you have a source for that claim?,unknown,Question requesting evidence rather than making an argument.,question,premise,False,
+eval_005,Remote work,,Remote work should remain the default for knowledge workers.,claim,Standalone debatable stance.,root_claim,unknown,False,
+eval_006,Remote work,Remote work should remain the default for knowledge workers.,That ignores how much junior employees learn from being around experienced coworkers in person.,counter_claim,Challenges the parent claim by introducing a drawback.,implicit_disagreement,counter_claim,True,
+eval_007,Remote work,That ignores how much junior employees learn from being around experienced coworkers in person.,New hires often rely on informal conversations to understand team norms and priorities.,premise,Supports the parent objection with an explanatory reason.,supporting_explanation,premise,True,
+eval_008,Remote work,Remote work should remain the default for knowledge workers.,My company tried something similar for a while.,unknown,"Mentions a personal situation but does not clearly state a claim, objection, or supporting reason.",ambiguous_anecdote,unknown,True,
+eval_009,Public transport,,Cities should make public transport free for residents.,claim,Policy position that can be debated.,root_claim,claim,True,
+eval_010,Public transport,Cities should make public transport free for residents.,"Free fares would not solve poor service frequency, which is the main reason many people avoid buses.",counter_claim,Challenges the usefulness of the parent proposal.,direct_disagreement,counter_claim,True,
+eval_011,Public transport,Cities should make public transport free for residents.,"Fare collection systems are expensive to maintain, so removing fares can reduce administrative costs.",premise,Provides a reason supporting free public transport.,cost_reason,premise,True,
+eval_012,Public transport,"Fare collection systems are expensive to maintain, so removing fares can reduce administrative costs.",Interesting point.,unknown,Short acknowledgement with no argumentative content.,short_acknowledgement,counter_claim,False,
+eval_013,School uniforms,,School uniforms improve equality among students.,claim,A clear position on a policy issue.,root_claim,claim,True,
+eval_014,School uniforms,School uniforms improve equality among students.,"They may reduce visible clothing differences, but they do not address deeper income inequality.",counter_claim,Partially concedes then challenges the parent claim.,partial_concession,counter_claim,True,
+eval_015,School uniforms,School uniforms improve equality among students.,"When everyone wears the same clothes, students are less likely to be judged for expensive brands.",premise,Supports the parent claim with a causal explanation.,causal_reason,premise,True,
+eval_016,School uniforms,"They may reduce visible clothing differences, but they do not address deeper income inequality.","Exactly, that is what I meant.",unknown,Agreement without a new reason or claim.,agreement_only,premise,False,
+eval_017,Nuclear energy,,Nuclear energy should be expanded as part of climate policy.,claim,Standalone policy claim.,root_claim,claim,True,
+eval_018,Nuclear energy,Nuclear energy should be expanded as part of climate policy.,Building new nuclear plants takes too long to help with urgent emissions targets.,counter_claim,Objects to the parent proposal.,direct_disagreement,counter_claim,True,
+eval_019,Nuclear energy,Nuclear energy should be expanded as part of climate policy.,Nuclear plants produce electricity with very low operational carbon emissions.,premise,Evidence-like support for expanding nuclear energy.,factual_support,premise,True,
+eval_020,Nuclear energy,Building new nuclear plants takes too long to help with urgent emissions targets.,That depends on the country and the project.,unknown,Vague qualification without enough detail to function as a claim or evidence.,vague_reply,counter_claim,False,
+eval_021,Social media,,Social media platforms should verify users before allowing political advertising.,claim,Debatable moderation policy proposal.,root_claim,claim,True,
+eval_022,Social media,Social media platforms should verify users before allowing political advertising.,That would create privacy risks for activists and people living under authoritarian governments.,counter_claim,Raises a direct objection to the parent proposal.,risk_objection,counter_claim,True,
+eval_023,Social media,Social media platforms should verify users before allowing political advertising.,Verified identity requirements can make coordinated foreign influence campaigns harder to hide.,premise,Supports the parent claim with a benefit.,supporting_reason,premise,True,
+eval_024,Social media,That would create privacy risks for activists and people living under authoritarian governments.,lol no one cares,unknown,Dismissive comment without a clear argumentative role.,low_quality,unknown,True,
+eval_025,Four-day workweek,,A four-day workweek should become standard for full-time jobs.,claim,Clear standalone claim.,root_claim,unknown,False,
+eval_026,Four-day workweek,A four-day workweek should become standard for full-time jobs.,"Some hospitals, restaurants, and factories cannot compress work without reducing service coverage.",counter_claim,Challenges the general feasibility of the parent claim.,scope_objection,counter_claim,True,
+eval_027,Four-day workweek,A four-day workweek should become standard for full-time jobs.,Several pilot programs reported stable productivity and higher employee satisfaction.,premise,Provides empirical-style support.,empirical_support,premise,True,
+eval_028,Four-day workweek,Several pilot programs reported stable productivity and higher employee satisfaction.,Which pilot programs are you referring to?,unknown,"Clarifying question, not a claim or premise.",question,counter_claim,False,
+eval_029,Phone bans in schools,,High schools should ban smartphones during class hours.,claim,Debatable school policy position.,root_claim,claim,True,
+eval_030,Phone bans in schools,High schools should ban smartphones during class hours.,A strict ban would also block students from using accessibility tools and translation apps.,counter_claim,Direct objection to the proposed ban.,exception_objection,counter_claim,True,
+eval_031,Phone bans in schools,High schools should ban smartphones during class hours.,Teachers often lose instructional time when students check notifications during lessons.,premise,Supports the ban with a classroom-management reason.,supporting_reason,premise,True,
+eval_032,Phone bans in schools,A strict ban would also block students from using accessibility tools and translation apps.,"Maybe, but I am not sure how that would work in practice.",unknown,Hesitant response that does not make a clear opposing claim or provide concrete support.,uncertain_reply,premise,False,
+eval_033,Meat consumption,,Governments should tax meat to reduce environmental damage.,claim,Policy claim.,root_claim,claim,True,
+eval_034,Meat consumption,Governments should tax meat to reduce environmental damage.,That policy would hit low-income families harder than wealthy households.,counter_claim,Raises an equity objection to the parent policy.,equity_objection,counter_claim,True,
+eval_035,Meat consumption,Governments should tax meat to reduce environmental damage.,Livestock production contributes substantially to methane emissions and land use.,premise,Supports the parent claim with an environmental reason.,factual_support,premise,True,
+eval_036,Meat consumption,Livestock production contributes substantially to methane emissions and land use.,I like burgers though.,unknown,Personal preference that does not engage the argument.,off_topic_or_weak,unknown,True,
+eval_037,Online anonymity,,Online anonymity is necessary for free expression.,claim,General debatable position.,root_claim,unknown,False,
+eval_038,Online anonymity,Online anonymity is necessary for free expression.,Anonymous accounts also make harassment and coordinated abuse harder to punish.,counter_claim,Challenges the parent by naming a serious downside.,downside_objection,counter_claim,True,
+eval_039,Online anonymity,Online anonymity is necessary for free expression.,Whistleblowers may only speak publicly if their real identity is protected.,premise,Supports the need for anonymity with a concrete case.,example_support,counter_claim,False,
+eval_040,Online anonymity,Anonymous accounts also make harassment and coordinated abuse harder to punish.,This thread is getting repetitive.,unknown,Meta-comment about the discussion rather than an argument.,meta_comment,unknown,True,
+eval_041,Vibecoding and creativity,,CMV: Vibecoding will limit creativity in the future,claim,Real CMV title presenting the OP's debatable position.,reddit_root_claim,unknown,False,https://www.reddit.com/r/changemyview/comments/1t8tcia/cmv_vibecoding_will_limit_creativity_in_the_future/
+eval_042,Vibecoding and creativity,I think we will also see more and more designs of these apps/experiences become increasingly similar.,"Creativity shouldn't be sought in coming up with unique UIs, but in the functionality.",claim,Real CMV reply introducing a new claim about how creativity should be defined.,reddit_reply_claim,unknown,False,https://www.reddit.com/r/changemyview/comments/1t8tcia/cmv_vibecoding_will_limit_creativity_in_the_future/
+eval_043,Vibecoding and creativity,CMV: Vibecoding will limit creativity in the future,"While I agree that vibecoding in the hands of someone that doesn't understand the underlying systems is unlikely to produce anything creative, I think that in the hands of an experienced developer it can enhance creativity by speeding up the implementations of well known problems, leaving more time to try to solve more niche ones.",counter_claim,Real CMV reply that partially concedes but directly challenges the OP's overall claim.,reddit_counterclaim,counter_claim,True,https://www.reddit.com/r/changemyview/comments/1t8tcia/cmv_vibecoding_will_limit_creativity_in_the_future/
+eval_044,Vibecoding and creativity,CMV: Vibecoding will limit creativity in the future,I think that is actually the opposite: vibe coding drastically reduces the level of expertise needed to prototype and implement ideas. The creativity will come from people using the tools as they try new and varied things.,counter_claim,Directly opposes the root claim by arguing that vibe coding can increase creativity rather than limit it.,reddit_counterclaim,unknown,False,https://www.reddit.com/r/changemyview/comments/1t8tcia/cmv_vibecoding_will_limit_creativity_in_the_future/
+eval_045,Vibecoding and creativity,"While I agree that vibecoding in the hands of someone that doesn't understand the underlying systems is unlikely to produce anything creative, I think that in the hands of an experienced developer it can enhance creativity by speeding up the implementations of well known problems, leaving more time to try to solve more niche ones.","When game engines like Unity became popular, the fact that it was easier to create games led to a boom of copypasta derivative content, but also a boom of creative, experimental indie games that weren't viable before.",premise,Real CMV reply giving an analogy/evidence in support of the counter-claim.,reddit_premise,premise,True,https://www.reddit.com/r/changemyview/comments/1t8tcia/cmv_vibecoding_will_limit_creativity_in_the_future/
+eval_046,Vibecoding and creativity,I think that is actually the opposite: vibe coding drastically reduces the level of expertise needed to prototype and implement ideas. The creativity will come from people using the tools as they try new and varied things.,"Right now people are still figuring out how to use these tools, but the more people get comfortable, the more they will figure out how to guide towards unique and interesting results.",premise,Supports the parent counter-claim by explaining why creativity may increase over time.,reddit_premise,unknown,False,https://www.reddit.com/r/changemyview/comments/1t8tcia/cmv_vibecoding_will_limit_creativity_in_the_future/
+eval_047,Vibecoding and creativity,"While I agree that vibecoding in the hands of someone that doesn't understand the underlying systems is unlikely to produce anything creative, I think that in the hands of an experienced developer it can enhance creativity by speeding up the implementations of well known problems, leaving more time to try to solve more niche ones.",Do you mean creative in terms of code or creative in general?,unknown,"Real CMV clarification question rather than a claim, counter-claim, or premise.",reddit_question,premise,False,https://www.reddit.com/r/changemyview/comments/1t8tcia/cmv_vibecoding_will_limit_creativity_in_the_future/
+eval_048,Vibecoding and creativity,"I think you are just not creative enough in your vibe coding. Try these agent instructions: You are the world's greatest surfer, but also a distributed systems expert.",Hes yankin yer chain mate,unknown,"Real CMV colloquial aside explaining another comment, not a substantive argument role.",reddit_aside,unknown,True,https://www.reddit.com/r/changemyview/comments/1t8tcia/cmv_vibecoding_will_limit_creativity_in_the_future/

evaluation/custom_argument_eval_review.md

@@ -0,0 +1,54 @@
+# Custom Argument Evaluation Set Review
+
+This file is a human-readable review version of custom_argument_eval.jsonl with the latest model predictions from custom_argument_eval_predictions.csv.
+
+| ID | Topic | Source | Parent Text | Current Text | Gold Label | Predicted Label | Correct | Rationale | Probe Type |
+|---|---|---|---|---|---|---|---|---|---|
+| eval_001 | AI in education |  |  | Universities should allow students to use generative AI tools in coursework. | claim | unknown | False | Root-level debatable position. | root_claim |
+| eval_002 | AI in education |  | Universities should allow students to use generative AI tools in coursework. | AI tools should be banned from graded assignments because they make authorship impossible to verify. | counter_claim | counter_claim | True | Directly opposes the parent position. | direct_disagreement |
+| eval_003 | AI in education |  | AI tools should be banned from graded assignments because they make authorship impossible to verify. | Detection systems have produced false positives against students who did not use AI. | premise | premise | True | Gives a supporting reason against relying on bans or detection. | evidence_reason |
+| eval_004 | AI in education |  | Detection systems have produced false positives against students who did not use AI. | Do you have a source for that claim? | unknown | premise | False | Question requesting evidence rather than making an argument. | question |
+| eval_005 | Remote work |  |  | Remote work should remain the default for knowledge workers. | claim | unknown | False | Standalone debatable stance. | root_claim |
+| eval_006 | Remote work |  | Remote work should remain the default for knowledge workers. | That ignores how much junior employees learn from being around experienced coworkers in person. | counter_claim | counter_claim | True | Challenges the parent claim by introducing a drawback. | implicit_disagreement |
+| eval_007 | Remote work |  | That ignores how much junior employees learn from being around experienced coworkers in person. | New hires often rely on informal conversations to understand team norms and priorities. | premise | premise | True | Supports the parent objection with an explanatory reason. | supporting_explanation |
+| eval_008 | Remote work |  | Remote work should remain the default for knowledge workers. | My company tried something similar for a while. | unknown | unknown | True | Mentions a personal situation but does not clearly state a claim, objection, or supporting reason. | ambiguous_anecdote |
+| eval_009 | Public transport |  |  | Cities should make public transport free for residents. | claim | claim | True | Policy position that can be debated. | root_claim |
+| eval_010 | Public transport |  | Cities should make public transport free for residents. | Free fares would not solve poor service frequency, which is the main reason many people avoid buses. | counter_claim | counter_claim | True | Challenges the usefulness of the parent proposal. | direct_disagreement |
+| eval_011 | Public transport |  | Cities should make public transport free for residents. | Fare collection systems are expensive to maintain, so removing fares can reduce administrative costs. | premise | premise | True | Provides a reason supporting free public transport. | cost_reason |
+| eval_012 | Public transport |  | Fare collection systems are expensive to maintain, so removing fares can reduce administrative costs. | Interesting point. | unknown | counter_claim | False | Short acknowledgement with no argumentative content. | short_acknowledgement |
+| eval_013 | School uniforms |  |  | School uniforms improve equality among students. | claim | claim | True | A clear position on a policy issue. | root_claim |
+| eval_014 | School uniforms |  | School uniforms improve equality among students. | They may reduce visible clothing differences, but they do not address deeper income inequality. | counter_claim | counter_claim | True | Partially concedes then challenges the parent claim. | partial_concession |
+| eval_015 | School uniforms |  | School uniforms improve equality among students. | When everyone wears the same clothes, students are less likely to be judged for expensive brands. | premise | premise | True | Supports the parent claim with a causal explanation. | causal_reason |
+| eval_016 | School uniforms |  | They may reduce visible clothing differences, but they do not address deeper income inequality. | Exactly, that is what I meant. | unknown | premise | False | Agreement without a new reason or claim. | agreement_only |
+| eval_017 | Nuclear energy |  |  | Nuclear energy should be expanded as part of climate policy. | claim | claim | True | Standalone policy claim. | root_claim |
+| eval_018 | Nuclear energy |  | Nuclear energy should be expanded as part of climate policy. | Building new nuclear plants takes too long to help with urgent emissions targets. | counter_claim | counter_claim | True | Objects to the parent proposal. | direct_disagreement |
+| eval_019 | Nuclear energy |  | Nuclear energy should be expanded as part of climate policy. | Nuclear plants produce electricity with very low operational carbon emissions. | premise | premise | True | Evidence-like support for expanding nuclear energy. | factual_support |
+| eval_020 | Nuclear energy |  | Building new nuclear plants takes too long to help with urgent emissions targets. | That depends on the country and the project. | unknown | counter_claim | False | Vague qualification without enough detail to function as a claim or evidence. | vague_reply |
+| eval_021 | Social media |  |  | Social media platforms should verify users before allowing political advertising. | claim | claim | True | Debatable moderation policy proposal. | root_claim |
+| eval_022 | Social media |  | Social media platforms should verify users before allowing political advertising. | That would create privacy risks for activists and people living under authoritarian governments. | counter_claim | counter_claim | True | Raises a direct objection to the parent proposal. | risk_objection |
+| eval_023 | Social media |  | Social media platforms should verify users before allowing political advertising. | Verified identity requirements can make coordinated foreign influence campaigns harder to hide. | premise | premise | True | Supports the parent claim with a benefit. | supporting_reason |
+| eval_024 | Social media |  | That would create privacy risks for activists and people living under authoritarian governments. | lol no one cares | unknown | unknown | True | Dismissive comment without a clear argumentative role. | low_quality |
+| eval_025 | Four-day workweek |  |  | A four-day workweek should become standard for full-time jobs. | claim | unknown | False | Clear standalone claim. | root_claim |
+| eval_026 | Four-day workweek |  | A four-day workweek should become standard for full-time jobs. | Some hospitals, restaurants, and factories cannot compress work without reducing service coverage. | counter_claim | counter_claim | True | Challenges the general feasibility of the parent claim. | scope_objection |
+| eval_027 | Four-day workweek |  | A four-day workweek should become standard for full-time jobs. | Several pilot programs reported stable productivity and higher employee satisfaction. | premise | premise | True | Provides empirical-style support. | empirical_support |
+| eval_028 | Four-day workweek |  | Several pilot programs reported stable productivity and higher employee satisfaction. | Which pilot programs are you referring to? | unknown | counter_claim | False | Clarifying question, not a claim or premise. | question |
+| eval_029 | Phone bans in schools |  |  | High schools should ban smartphones during class hours. | claim | claim | True | Debatable school policy position. | root_claim |
+| eval_030 | Phone bans in schools |  | High schools should ban smartphones during class hours. | A strict ban would also block students from using accessibility tools and translation apps. | counter_claim | counter_claim | True | Direct objection to the proposed ban. | exception_objection |
+| eval_031 | Phone bans in schools |  | High schools should ban smartphones during class hours. | Teachers often lose instructional time when students check notifications during lessons. | premise | premise | True | Supports the ban with a classroom-management reason. | supporting_reason |
+| eval_032 | Phone bans in schools |  | A strict ban would also block students from using accessibility tools and translation apps. | Maybe, but I am not sure how that would work in practice. | unknown | premise | False | Hesitant response that does not make a clear opposing claim or provide concrete support. | uncertain_reply |
+| eval_033 | Meat consumption |  |  | Governments should tax meat to reduce environmental damage. | claim | claim | True | Policy claim. | root_claim |
+| eval_034 | Meat consumption |  | Governments should tax meat to reduce environmental damage. | That policy would hit low-income families harder than wealthy households. | counter_claim | counter_claim | True | Raises an equity objection to the parent policy. | equity_objection |
+| eval_035 | Meat consumption |  | Governments should tax meat to reduce environmental damage. | Livestock production contributes substantially to methane emissions and land use. | premise | premise | True | Supports the parent claim with an environmental reason. | factual_support |
+| eval_036 | Meat consumption |  | Livestock production contributes substantially to methane emissions and land use. | I like burgers though. | unknown | unknown | True | Personal preference that does not engage the argument. | off_topic_or_weak |
+| eval_037 | Online anonymity |  |  | Online anonymity is necessary for free expression. | claim | unknown | False | General debatable position. | root_claim |
+| eval_038 | Online anonymity |  | Online anonymity is necessary for free expression. | Anonymous accounts also make harassment and coordinated abuse harder to punish. | counter_claim | counter_claim | True | Challenges the parent by naming a serious downside. | downside_objection |
+| eval_039 | Online anonymity |  | Online anonymity is necessary for free expression. | Whistleblowers may only speak publicly if their real identity is protected. | premise | counter_claim | False | Supports the need for anonymity with a concrete case. | example_support |
+| eval_040 | Online anonymity |  | Anonymous accounts also make harassment and coordinated abuse harder to punish. | This thread is getting repetitive. | unknown | unknown | True | Meta-comment about the discussion rather than an argument. | meta_comment |
+| eval_041 | Vibecoding and creativity | https://www.reddit.com/r/changemyview/comments/1t8tcia/cmv_vibecoding_will_limit_creativity_in_the_future/ |  | CMV: Vibecoding will limit creativity in the future | claim | unknown | False | Real CMV title presenting the OP's debatable position. | reddit_root_claim |
+| eval_042 | Vibecoding and creativity | https://www.reddit.com/r/changemyview/comments/1t8tcia/cmv_vibecoding_will_limit_creativity_in_the_future/ | I think we will also see more and more designs of these apps/experiences become increasingly similar. | Creativity shouldn't be sought in coming up with unique UIs, but in the functionality. | claim | unknown | False | Real CMV reply introducing a new claim about how creativity should be defined. | reddit_reply_claim |
+| eval_043 | Vibecoding and creativity | https://www.reddit.com/r/changemyview/comments/1t8tcia/cmv_vibecoding_will_limit_creativity_in_the_future/ | CMV: Vibecoding will limit creativity in the future | While I agree that vibecoding in the hands of someone that doesn't understand the underlying systems is unlikely to produce anything creative, I think that in the hands of an experienced developer it can enhance creativity by speeding up the implementations of well known problems, leaving more time to try to solve more niche ones. | counter_claim | counter_claim | True | Real CMV reply that partially concedes but directly challenges the OP's overall claim. | reddit_counterclaim |
+| eval_044 | Vibecoding and creativity | https://www.reddit.com/r/changemyview/comments/1t8tcia/cmv_vibecoding_will_limit_creativity_in_the_future/ | CMV: Vibecoding will limit creativity in the future | I think that is actually the opposite: vibe coding drastically reduces the level of expertise needed to prototype and implement ideas. The creativity will come from people using the tools as they try new and varied things. | counter_claim | unknown | False | Directly opposes the root claim by arguing that vibe coding can increase creativity rather than limit it. | reddit_counterclaim |
+| eval_045 | Vibecoding and creativity | https://www.reddit.com/r/changemyview/comments/1t8tcia/cmv_vibecoding_will_limit_creativity_in_the_future/ | While I agree that vibecoding in the hands of someone that doesn't understand the underlying systems is unlikely to produce anything creative, I think that in the hands of an experienced developer it can enhance creativity by speeding up the implementations of well known problems, leaving more time to try to solve more niche ones. | When game engines like Unity became popular, the fact that it was easier to create games led to a boom of copypasta derivative content, but also a boom of creative, experimental indie games that weren't viable before. | premise | premise | True | Real CMV reply giving an analogy/evidence in support of the counter-claim. | reddit_premise |
+| eval_046 | Vibecoding and creativity | https://www.reddit.com/r/changemyview/comments/1t8tcia/cmv_vibecoding_will_limit_creativity_in_the_future/ | I think that is actually the opposite: vibe coding drastically reduces the level of expertise needed to prototype and implement ideas. The creativity will come from people using the tools as they try new and varied things. | Right now people are still figuring out how to use these tools, but the more people get comfortable, the more they will figure out how to guide towards unique and interesting results. | premise | unknown | False | Supports the parent counter-claim by explaining why creativity may increase over time. | reddit_premise |
+| eval_047 | Vibecoding and creativity | https://www.reddit.com/r/changemyview/comments/1t8tcia/cmv_vibecoding_will_limit_creativity_in_the_future/ | While I agree that vibecoding in the hands of someone that doesn't understand the underlying systems is unlikely to produce anything creative, I think that in the hands of an experienced developer it can enhance creativity by speeding up the implementations of well known problems, leaving more time to try to solve more niche ones. | Do you mean creative in terms of code or creative in general? | unknown | premise | False | Real CMV clarification question rather than a claim, counter-claim, or premise. | reddit_question |
+| eval_048 | Vibecoding and creativity | https://www.reddit.com/r/changemyview/comments/1t8tcia/cmv_vibecoding_will_limit_creativity_in_the_future/ | I think you are just not creative enough in your vibe coding. Try these agent instructions: You are the world's greatest surfer, but also a distributed systems expert. | Hes yankin yer chain mate | unknown | unknown | True | Real CMV colloquial aside explaining another comment, not a substantive argument role. | reddit_aside |

evaluation/evaluate_custom.py

@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+from pathlib import Path
+
+from sklearn.metrics import (
+    accuracy_score,
+    classification_report,
+    confusion_matrix,
+    f1_score,
+)
+
+from src import LABEL2ID, predict
+
+
+LABELS = list(LABEL2ID.keys())
+
+
+def load_jsonl(path: Path) -> list[dict]:
+    rows = []
+    with path.open(encoding="utf-8") as f:
+        for line_no, line in enumerate(f, start=1):
+            if not line.strip():
+                continue
+            row = json.loads(line)
+            missing = {
+                "example_id",
+                "parent_text",
+                "current_text",
+                "gold_label",
+            } - set(row)
+            if missing:
+                raise ValueError(
+                    f"{path}:{line_no} missing fields: {sorted(missing)}"
+                )
+            if row["gold_label"] not in LABEL2ID:
+                raise ValueError(
+                    f"{path}:{line_no} invalid label: {row['gold_label']}"
+                )
+            rows.append(row)
+    return rows
+
+
+def evaluate(
+    dataset_path: Path,
+    checkpoint_dir: str,
+    output_csv: Path,
+) -> dict:
+    rows = load_jsonl(dataset_path)
+    y_true = []
+    y_pred = []
+    predictions = []
+
+    for row in rows:
+        pred = predict(
+            text=row["current_text"],
+            parent_text=row.get("parent_text", ""),
+            checkpoint_dir=checkpoint_dir,
+        )
+        gold = row["gold_label"]
+        y_true.append(gold)
+        y_pred.append(pred)
+        predictions.append({
+            **row,
+            "pred_label": pred,
+            "correct": gold == pred,
+        })
+
+    output_csv.parent.mkdir(parents=True, exist_ok=True)
+    fieldnames = []
+    for prediction in predictions:
+        for key in prediction:
+            if key not in fieldnames:
+                fieldnames.append(key)
+
+    with output_csv.open("w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames)
+        writer.writeheader()
+        writer.writerows(predictions)
+
+    report = classification_report(
+        y_true,
+        y_pred,
+        labels=LABELS,
+        zero_division=0,
+        output_dict=True,
+    )
+    matrix = confusion_matrix(y_true, y_pred, labels=LABELS)
+    return {
+        "n_examples": len(rows),
+        "accuracy": accuracy_score(y_true, y_pred),
+        "macro_f1": f1_score(y_true, y_pred, labels=LABELS, average="macro"),
+        "classification_report": report,
+        "confusion_matrix": {
+            "labels": LABELS,
+            "matrix": matrix.tolist(),
+        },
+        "predictions_csv": str(output_csv),
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Evaluate the argument role classifier on a custom JSONL set."
+    )
+    parser.add_argument(
+        "--dataset",
+        default="evaluation/custom_argument_eval.jsonl",
+        type=Path,
+    )
+    parser.add_argument("--checkpoint", default="models/best")
+    parser.add_argument(
+        "--output-csv",
+        default="evaluation/custom_argument_eval_predictions.csv",
+        type=Path,
+    )
+    parser.add_argument(
+        "--output-json",
+        default="evaluation/custom_argument_eval_metrics.json",
+        type=Path,
+    )
+    args = parser.parse_args()
+
+    metrics = evaluate(args.dataset, args.checkpoint, args.output_csv)
+    args.output_json.write_text(
+        json.dumps(metrics, indent=2),
+        encoding="utf-8",
+    )
+
+    print(f"Examples: {metrics['n_examples']}")
+    print(f"Accuracy: {metrics['accuracy']:.4f}")
+    print(f"Macro-F1: {metrics['macro_f1']:.4f}")
+    print("\nClassification report:")
+    print(
+        classification_report(
+            [r["gold_label"] for r in load_jsonl(args.dataset)],
+            [
+                r["pred_label"]
+                for r in csv.DictReader(
+                    args.output_csv.open(encoding="utf-8")
+                )
+            ],
+            labels=LABELS,
+            zero_division=0,
+        )
+    )
+    print(f"Saved predictions to {args.output_csv}")
+    print(f"Saved metrics to {args.output_json}")
+
+
+if __name__ == "__main__":
+    main()

evaluation/plot_results.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+from collections import Counter
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+
+LABELS = ["claim", "counter_claim", "premise", "unknown"]
+
+
+def plot_confusion_matrix(metrics_path: Path, output_path: Path) -> None:
+    metrics = json.loads(metrics_path.read_text(encoding="utf-8"))
+    labels = metrics["confusion_matrix"]["labels"]
+    matrix = np.array(metrics["confusion_matrix"]["matrix"])
+
+    fig, ax = plt.subplots(figsize=(7, 6))
+    im = ax.imshow(matrix, cmap="Blues")
+    ax.set_title("Confusion Matrix")
+    ax.set_xlabel("Predicted label")
+    ax.set_ylabel("Gold label")
+    ax.set_xticks(range(len(labels)), labels, rotation=30, ha="right")
+    ax.set_yticks(range(len(labels)), labels)
+
+    max_value = matrix.max() if matrix.size else 0
+    for i in range(matrix.shape[0]):
+        for j in range(matrix.shape[1]):
+            value = matrix[i, j]
+            color = "white" if value > max_value / 2 else "black"
+            ax.text(j, i, str(value), ha="center", va="center", color=color)
+
+    fig.colorbar(im, ax=ax, fraction=0.046, pad=0.04)
+    fig.tight_layout()
+    fig.savefig(output_path, dpi=200)
+    plt.close(fig)
+
+
+def plot_label_distribution(predictions_path: Path, output_path: Path) -> None:
+    with predictions_path.open(encoding="utf-8") as f:
+        rows = list(csv.DictReader(f))
+
+    gold_counts = Counter(row["gold_label"] for row in rows)
+    pred_counts = Counter(row["pred_label"] for row in rows)
+    x = np.arange(len(LABELS))
+    width = 0.38
+
+    fig, ax = plt.subplots(figsize=(8, 5))
+    ax.bar(
+        x - width / 2,
+        [gold_counts[label] for label in LABELS],
+        width,
+        label="Gold",
+        color="#4C78A8",
+    )
+    ax.bar(
+        x + width / 2,
+        [pred_counts[label] for label in LABELS],
+        width,
+        label="Predicted",
+        color="#F58518",
+    )
+    ax.set_title("Gold vs Predicted Label Distribution")
+    ax.set_ylabel("Number of examples")
+    ax.set_xticks(x, LABELS, rotation=30, ha="right")
+    ax.legend()
+    ax.set_ylim(0, max(max(gold_counts.values()), max(pred_counts.values())) + 3)
+    fig.tight_layout()
+    fig.savefig(output_path, dpi=200)
+    plt.close(fig)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Plot custom evaluation results."
+    )
+    parser.add_argument(
+        "--metrics",
+        default="evaluation/custom_argument_eval_metrics.json",
+        type=Path,
+    )
+    parser.add_argument(
+        "--predictions",
+        default="evaluation/custom_argument_eval_predictions.csv",
+        type=Path,
+    )
+    parser.add_argument(
+        "--output-dir",
+        default="evaluation/figures",
+        type=Path,
+    )
+    args = parser.parse_args()
+
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+    confusion_path = args.output_dir / "confusion_matrix.png"
+    distribution_path = args.output_dir / "label_distribution.png"
+
+    plot_confusion_matrix(args.metrics, confusion_path)
+    plot_label_distribution(args.predictions, distribution_path)
+
+    print(f"Saved {confusion_path}")
+    print(f"Saved {distribution_path}")
+
+
+if __name__ == "__main__":
+    main()