Add data loading pipeline (Person 1)

Unified schema (Argument, Debate dataclasses), loaders for CMV offline
dataset, IBM Debater via HuggingFace, and a live Reddit scraper via PRAW.
Preprocessing module strips Reddit noise and filters short/deleted text.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

.env.example

@@ -0,0 +1,3 @@
+REDDIT_CLIENT_ID=your_client_id_here
+REDDIT_CLIENT_SECRET=your_client_secret_here
+REDDIT_USER_AGENT=nlp-project/1.0 by your_reddit_username

data/README.md

@@ -0,0 +1,59 @@
+# Data module
+
+Owns all data loading, cleaning, and the unified schema. Everything downstream imports from here.
+
+## Unified schema
+
+```python
+from data import Argument, Debate
+
+debate = Debate(id="abc", title="CMV: cats > dogs", source="cmv")
+arg   = Argument(id="1", text="Cats are independent.", arg_type="premise")
+```
+
+`arg_type` must be one of: `claim`, `counter_claim`, `premise`, `unknown`.
+
+## Sources
+
+### CMV (offline)
+
+Download the Tan et al. 2016 dataset:
+```bash
+mkdir -p data/raw/cmv
+curl -O https://chenhaot.com/data/cmv/cmv.tar.bz2
+tar -xjf cmv.tar.bz2 -C data/raw/cmv/
+```
+
+Then:
+```python
+from data import load_cmv, clean_debates
+debates = clean_debates(load_cmv("train"))
+```
+
+### IBM Debater (auto-download)
+
+```python
+from data import load_ibm
+debates = load_ibm("train")  # downloads via HuggingFace datasets on first run
+```
+
+### Live Reddit scraper
+
+Add credentials to `.env` (copy from `.env.example`):
+```
+REDDIT_CLIENT_ID=your_id
+REDDIT_CLIENT_SECRET=your_secret
+REDDIT_USER_AGENT=nlp-project/1.0 by your_username
+```
+
+Then:
+```python
+from data import scrape_cmv, clean_debates
+debates = clean_debates(scrape_cmv(limit=100, sort="top", time_filter="month"))
+```
+
+## Preprocessing
+
+`clean_debates()` applies Reddit-specific noise removal (quoted text, URLs,
+edit notes, user mentions) and drops arguments shorter than 5 tokens. Call it
+on any list of `Debate` objects before passing them downstream.

data/__init__.py

@@ -0,0 +1,12 @@
+from data.schema import Argument, Debate
+from data.loaders import load_cmv, load_ibm, scrape_cmv
+from data.preprocessing import clean_debates
+
+__all__ = [
+    "Argument",
+    "Debate",
+    "load_cmv",
+    "load_ibm",
+    "scrape_cmv",
+    "clean_debates",
+]

data/loaders/__init__.py

@@ -0,0 +1,5 @@
+from data.loaders.cmv import load_cmv
+from data.loaders.ibm import load_ibm
+from data.loaders.reddit import scrape_cmv
+
+__all__ = ["load_cmv", "load_ibm", "scrape_cmv"]

data/loaders/cmv.py

@@ -0,0 +1,94 @@
+"""
+Loader for the Change My View (CMV) dataset.
+
+Source: Tan et al. 2016 "Winning Arguments" — r/changemyview threads
+where the OP awards a delta (∆) to a reply that changed their view.
+
+Expected raw file: data/raw/cmv/train.jsonl (and test.jsonl)
+Each line is a JSON object:
+  {
+    "id": "<reddit_post_id>",
+    "title": "CMV: ...",
+    "selftext": "<OP body>",
+    "op_author": "<username>",
+    "comments": [
+      {
+        "id": "<comment_id>",
+        "body": "<text>",
+        "author": "<username>",
+        "score": <int>,
+        "parent_id": "<post_id or comment_id>",
+        "delta": <bool>   # True if OP awarded a delta to this comment
+      }, ...
+    ]
+  }
+
+Download: https://chenhaot.com/data/cmv/cmv.tar.bz2
+"""
+
+import json
+from pathlib import Path
+from typing import List
+
+from data.schema import Argument, Debate
+
+_CMV_RAW = Path("data/raw/cmv")
+
+
+def _comment_to_arg_type(comment: dict) -> str:
+    """
+    Heuristic arg_type assignment:
+    - delta-awarded comments are counter_claims (they changed the OP's mind)
+    - top-level comments (parent == post) are claims
+    - nested replies are premises
+    """
+    if comment.get("delta"):
+        return "counter_claim"
+    if str(comment.get("parent_id", "")).startswith("t3_"):  # t3_ = link/post prefix
+        return "claim"
+    return "premise"
+
+
+def load_cmv(split: str = "train") -> List[Debate]:
+    path = _CMV_RAW / f"{split}.jsonl"
+    if not path.exists():
+        raise FileNotFoundError(
+            f"CMV raw file not found at {path}.\n"
+            "Download from https://chenhaot.com/data/cmv/cmv.tar.bz2 "
+            "and extract into data/raw/cmv/"
+        )
+
+    debates = []
+    with open(path) as f:
+        for line in f:
+            post = json.loads(line)
+            args = [
+                Argument(
+                    id=post["id"],
+                    text=post["selftext"],
+                    arg_type="claim",
+                    author=post.get("op_author"),
+                    metadata={"title": post["title"]},
+                )
+            ]
+            for c in post.get("comments", []):
+                args.append(
+                    Argument(
+                        id=c["id"],
+                        text=c["body"],
+                        arg_type=_comment_to_arg_type(c),
+                        parent_id=c.get("parent_id"),
+                        author=c.get("author"),
+                        score=c.get("score"),
+                        metadata={"delta": c.get("delta", False)},
+                    )
+                )
+            debates.append(
+                Debate(
+                    id=post["id"],
+                    title=post["title"],
+                    source="cmv",
+                    arguments=args,
+                )
+            )
+    return debates

data/loaders/ibm.py

@@ -0,0 +1,60 @@
+"""
+Loader for IBM Debater — Argument Quality Ranking dataset.
+
+HuggingFace: ibm/argument_quality_ranking_30k
+~30k argument–topic pairs with human quality scores.
+
+We map this to the unified schema:
+  - topic → Debate with a single claim (the motion)
+  - argument → Argument of type 'premise' attached to that claim
+  - WA (weighted average quality) stored in metadata
+"""
+
+from typing import List, Dict
+
+from data.schema import Argument, Debate
+
+
+def load_ibm(split: str = "train") -> List[Debate]:
+    try:
+        from datasets import load_dataset
+    except ImportError:
+        raise ImportError("Install 'datasets': pip install datasets")
+
+    ds = load_dataset("ibm/argument_quality_ranking_30k", split=split, trust_remote_code=True)
+
+    # Group arguments by topic so each topic becomes one Debate
+    topics: Dict[str, List[dict]] = {}
+    for row in ds:
+        topics.setdefault(row["topic"], []).append(row)
+
+    debates = []
+    for topic, rows in topics.items():
+        topic_id = topic.replace(" ", "_")[:64]
+        claim = Argument(
+            id=f"{topic_id}__claim",
+            text=topic,
+            arg_type="claim",
+        )
+        premises = [
+            Argument(
+                id=f"{topic_id}__{i}",
+                text=row["argument"],
+                arg_type="premise",
+                parent_id=claim.id,
+                metadata={
+                    "quality_score": row.get("WA"),
+                    "stance": row.get("stance"),
+                },
+            )
+            for i, row in enumerate(rows)
+        ]
+        debates.append(
+            Debate(
+                id=topic_id,
+                title=topic,
+                source="ibm",
+                arguments=[claim] + premises,
+            )
+        )
+    return debates

data/loaders/reddit.py

@@ -0,0 +1,142 @@
+"""
+Live Reddit scraper for r/changemyview threads via PRAW.
+
+Requires a .env file with:
+  REDDIT_CLIENT_ID=...
+  REDDIT_CLIENT_SECRET=...
+  REDDIT_USER_AGENT=nlp-project/1.0 by <your_username>
+
+Register a script app at https://www.reddit.com/prefs/apps
+"""
+
+import os
+from typing import List, Optional
+
+from dotenv import load_dotenv
+
+from data.schema import Argument, Debate
+
+load_dotenv()
+
+
+def _get_reddit():
+    try:
+        import praw
+    except ImportError:
+        raise ImportError("Install 'praw': pip install praw")
+
+    client_id = os.getenv("REDDIT_CLIENT_ID")
+    client_secret = os.getenv("REDDIT_CLIENT_SECRET")
+    user_agent = os.getenv("REDDIT_USER_AGENT", "nlp-project/1.0")
+
+    if not client_id or not client_secret:
+        raise EnvironmentError(
+            "Missing Reddit credentials. Set REDDIT_CLIENT_ID and "
+            "REDDIT_CLIENT_SECRET in your .env file."
+        )
+
+    return praw.Reddit(
+        client_id=client_id,
+        client_secret=client_secret,
+        user_agent=user_agent,
+    )
+
+
+def _flatten_comments(comment, depth: int = 0) -> List[dict]:
+    """Recursively flatten a comment tree into a list of dicts."""
+    results = []
+    try:
+        body = comment.body
+    except AttributeError:
+        return results  # MoreComments object — skip
+
+    has_delta = "∆" in body or "!delta" in body.lower()
+    results.append({
+        "id": comment.id,
+        "body": body,
+        "author": str(comment.author) if comment.author else "[deleted]",
+        "score": comment.score,
+        "parent_id": comment.parent_id,
+        "delta": has_delta,
+        "depth": depth,
+    })
+    for reply in comment.replies:
+        results.extend(_flatten_comments(reply, depth + 1))
+    return results
+
+
+def scrape_cmv(
+    limit: int = 50,
+    sort: str = "top",
+    time_filter: str = "month",
+    min_comments: int = 5,
+) -> List[Debate]:
+    """
+    Scrape r/changemyview threads and return them as Debate objects.
+
+    Args:
+        limit: number of posts to fetch (Reddit caps at 1000)
+        sort: 'top', 'hot', 'new', or 'controversial'
+        time_filter: 'day', 'week', 'month', 'year', 'all' (only for 'top')
+        min_comments: skip threads with fewer than this many comments
+    """
+    reddit = _get_reddit()
+    subreddit = reddit.subreddit("changemyview")
+
+    if sort == "top":
+        posts = subreddit.top(time_filter=time_filter, limit=limit)
+    elif sort == "hot":
+        posts = subreddit.hot(limit=limit)
+    elif sort == "new":
+        posts = subreddit.new(limit=limit)
+    elif sort == "controversial":
+        posts = subreddit.controversial(time_filter=time_filter, limit=limit)
+    else:
+        raise ValueError(f"Unknown sort '{sort}'")
+
+    debates = []
+    for post in posts:
+        if post.num_comments < min_comments:
+            continue
+
+        post.comments.replace_more(limit=0)  # skip MoreComments placeholders
+        flat_comments = []
+        for c in post.comments:
+            flat_comments.extend(_flatten_comments(c))
+
+        args = [
+            Argument(
+                id=post.id,
+                text=post.selftext,
+                arg_type="claim",
+                author=str(post.author) if post.author else "[deleted]",
+                score=post.score,
+                metadata={"title": post.title, "url": post.url},
+            )
+        ]
+        for c in flat_comments:
+            arg_type = "counter_claim" if c["delta"] else (
+                "claim" if c["depth"] == 0 else "premise"
+            )
+            args.append(
+                Argument(
+                    id=c["id"],
+                    text=c["body"],
+                    arg_type=arg_type,
+                    parent_id=c["parent_id"],
+                    author=c["author"],
+                    score=c["score"],
+                    metadata={"delta": c["delta"], "depth": c["depth"]},
+                )
+            )
+
+        debates.append(
+            Debate(
+                id=post.id,
+                title=post.title,
+                source="reddit",
+                arguments=args,
+                metadata={"subreddit": "changemyview", "sort": sort},
+            )
+        )
+    return debates

data/preprocessing/__init__.py

@@ -0,0 +1,3 @@
+from data.preprocessing.clean import clean_text, clean_debate, clean_debates, is_valid
+
+__all__ = ["clean_text", "clean_debate", "clean_debates", "is_valid"]

data/preprocessing/clean.py

@@ -0,0 +1,71 @@
+"""
+Text cleaning utilities shared across all loaders.
+
+Applied before tokenization — keeps text usable for both
+classical NLP pipelines and transformer tokenizers.
+"""
+
+import re
+from typing import List
+
+from data.schema import Argument, Debate
+
+# Reddit-specific noise
+_REDDIT_QUOTE = re.compile(r"^>.*$", re.MULTILINE)
+_URL = re.compile(r"https?://\S+|www\.\S+")
+_SUBREDDIT_MENTION = re.compile(r"r/\w+")
+_USER_MENTION = re.compile(r"u/\w+")
+_EDIT_NOTE = re.compile(r"\*?edit\*?:.*", re.IGNORECASE | re.DOTALL)
+_WHITESPACE = re.compile(r"\s+")
+
+# Deleted/removed placeholder strings
+_DELETED = {"[deleted]", "[removed]", ""}
+
+
+def clean_text(text: str) -> str:
+    text = _REDDIT_QUOTE.sub("", text)
+    text = _URL.sub("", text)
+    text = _SUBREDDIT_MENTION.sub("", text)
+    text = _USER_MENTION.sub("", text)
+    text = _EDIT_NOTE.sub("", text)
+    text = _WHITESPACE.sub(" ", text)
+    return text.strip()
+
+
+def is_valid(text: str, min_tokens: int = 5) -> bool:
+    """Return False for deleted posts or suspiciously short text."""
+    if text in _DELETED:
+        return False
+    return len(text.split()) >= min_tokens
+
+
+def clean_debate(debate: Debate, min_tokens: int = 5) -> Debate:
+    """Return a new Debate with cleaned argument texts, dropping invalid ones."""
+    cleaned_args = []
+    for arg in debate.arguments:
+        text = clean_text(arg.text)
+        if is_valid(text, min_tokens):
+            cleaned_args.append(
+                Argument(
+                    id=arg.id,
+                    text=text,
+                    arg_type=arg.arg_type,
+                    parent_id=arg.parent_id,
+                    author=arg.author,
+                    score=arg.score,
+                    metadata=arg.metadata,
+                )
+            )
+    return Debate(
+        id=debate.id,
+        title=debate.title,
+        source=debate.source,
+        arguments=cleaned_args,
+        metadata=debate.metadata,
+    )
+
+
+def clean_debates(debates: List[Debate], min_tokens: int = 5) -> List[Debate]:
+    cleaned = [clean_debate(d, min_tokens) for d in debates]
+    # Drop debates that lost their root claim during cleaning
+    return [d for d in cleaned if d.root() is not None]

data/schema.py

@@ -0,0 +1,48 @@
+from dataclasses import dataclass, field
+from typing import List, Optional, Dict
+
+ARG_TYPES = {"claim", "counter_claim", "premise", "unknown"}
+
+
+@dataclass
+class Argument:
+    id: str
+    text: str
+    arg_type: str  # one of ARG_TYPES
+    parent_id: Optional[str] = None
+    author: Optional[str] = None
+    score: Optional[int] = None  # Reddit upvotes/downvotes
+    metadata: Dict = field(default_factory=dict)
+
+    def __post_init__(self):
+        if self.arg_type not in ARG_TYPES:
+            raise ValueError(f"arg_type must be one of {ARG_TYPES}, got '{self.arg_type}'")
+
+
+@dataclass
+class Debate:
+    id: str
+    title: str
+    source: str  # 'cmv', 'ibm', 'reddit'
+    arguments: List[Argument] = field(default_factory=list)
+    metadata: Dict = field(default_factory=dict)
+
+    def root(self) -> Optional[Argument]:
+        """The top-level claim (no parent)."""
+        roots = [a for a in self.arguments if a.parent_id is None]
+        return roots[0] if roots else None
+
+    def claims(self) -> List[Argument]:
+        return [a for a in self.arguments if a.arg_type == "claim"]
+
+    def counter_claims(self) -> List[Argument]:
+        return [a for a in self.arguments if a.arg_type == "counter_claim"]
+
+    def premises(self) -> List[Argument]:
+        return [a for a in self.arguments if a.arg_type == "premise"]
+
+    def replies_to(self, argument_id: str) -> List[Argument]:
+        return [a for a in self.arguments if a.parent_id == argument_id]
+
+    def __len__(self) -> int:
+        return len(self.arguments)