Initial commit: DataCleanEnv core environment

- models.py: Action, Observation, State extending OpenEnv base types
- server/tasks.py: 3 hardcoded tasks (easy/medium/hard) with issue registries
- server/graders.py: Validation-based grading functions
- server/action_parser.py: Robust string command parser
- server/environment.py: Core environment logic (reset/step/state)
- server/app.py: FastAPI server wiring
- client.py: EnvClient subclass

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

.gitignore

@@ -0,0 +1,16 @@
+__pycache__/
+*.py[cod]
+*$py.class
+*.egg-info/
+dist/
+build/
+.eggs/
+*.egg
+.env
+.venv
+venv/
+ENV/
+.claude/
+*.lock
+.DS_Store
+Thumbs.db

data_clean_env/__init__.py

@@ -0,0 +1 @@
+from .models import DataCleanAction, DataCleanObservation, DataCleanState

data_clean_env/client.py

@@ -0,0 +1,34 @@
+"""DataClean Environment Client."""
+
+from openenv.core.env_client import EnvClient
+from openenv.core.client_types import StepResult
+
+from .models import DataCleanAction, DataCleanObservation, DataCleanState
+
+
+class DataCleanEnv(EnvClient[DataCleanAction, DataCleanObservation, DataCleanState]):
+    """Client for interacting with a DataClean environment server.
+
+    Example:
+        >>> with DataCleanEnv(base_url="http://localhost:8000").sync() as env:
+        ...     result = env.reset(task_id="customer_contacts")
+        ...     print(result.observation.data_preview)
+        ...     result = env.step(DataCleanAction(command='inspect("email")'))
+        ...     result = env.step(DataCleanAction(command='fix(3, "email", "test@example.com")'))
+        ...     result = env.step(DataCleanAction(command='submit()'))
+        ...     print(f"Score: {result.observation.current_score}")
+    """
+
+    def _step_payload(self, action: DataCleanAction) -> dict:
+        return action.model_dump()
+
+    def _parse_result(self, payload: dict) -> StepResult[DataCleanObservation]:
+        obs = DataCleanObservation(**payload.get("observation", payload))
+        return StepResult(
+            observation=obs,
+            reward=obs.reward,
+            done=obs.done,
+        )
+
+    def _parse_state(self, payload: dict) -> DataCleanState:
+        return DataCleanState(**payload)

data_clean_env/models.py

@@ -0,0 +1,49 @@
+from typing import Any, Dict, List, Optional
+
+from pydantic import Field
+
+from openenv.core.env_server.types import Action, Observation, State
+
+
+class DataCleanAction(Action):
+    """A string command for data cleaning operations.
+
+    Supported commands:
+        inspect("column_name") - View column statistics and issue hints
+        fix(row, "column", "new_value") - Correct a cell value
+        delete(row) - Remove a duplicate/invalid row
+        submit() - Finalize and get scored
+    """
+
+    command: str = Field(
+        ..., min_length=1, description="Command string to execute"
+    )
+
+
+class DataCleanObservation(Observation):
+    """Observation returned after each step."""
+
+    task_id: str = Field(default="")
+    task_description: str = Field(default="")
+    difficulty: str = Field(default="easy")
+    data_preview: str = Field(default="", description="Formatted text table of current data")
+    column_info: str = Field(default="", description="Column names, types, and stats")
+    feedback: str = Field(default="", description="Result of last action")
+    actions_remaining: int = Field(default=0)
+    issues_found: int = Field(default=0)
+    issues_fixed: int = Field(default=0)
+    total_issues: int = Field(default=0)
+    current_score: float = Field(default=0.0)
+    action_history: List[str] = Field(default_factory=list)
+
+
+class DataCleanState(State):
+    """Full environment state."""
+
+    task_id: str = Field(default="")
+    difficulty: str = Field(default="easy")
+    total_issues: int = Field(default=0)
+    fixed_issues: int = Field(default=0)
+    damaged_cells: int = Field(default=0)
+    max_steps: int = Field(default=15)
+    score: float = Field(default=0.0)

data_clean_env/server/action_parser.py

@@ -0,0 +1,102 @@
+"""Robust parser for data cleaning commands."""
+
+import re
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass
+class ParsedAction:
+    command_type: str  # "inspect", "fix", "delete", "submit", "error"
+    args: dict
+    error_message: Optional[str] = None
+
+
+# Strip markdown code fences and leading "action:" prefixes
+_PREFIX_RE = re.compile(
+    r"^(?:```\w*\s*\n?|action\s*[:\-]\s*|next\s*action\s*[:\-]\s*)",
+    re.IGNORECASE,
+)
+_SUFFIX_RE = re.compile(r"\s*```\s*$")
+
+
+def _strip_quotes(s: str) -> str:
+    s = s.strip()
+    if len(s) >= 2 and s[0] == s[-1] and s[0] in ("'", '"'):
+        return s[1:-1]
+    return s
+
+
+def parse_action(raw: str) -> ParsedAction:
+    """Parse a raw command string into a structured ParsedAction."""
+    if not raw or not raw.strip():
+        return ParsedAction("error", {}, "Empty command. Use inspect/fix/delete/submit.")
+
+    text = raw.strip()
+    text = _PREFIX_RE.sub("", text)
+    text = _SUFFIX_RE.sub("", text)
+    text = text.strip()
+
+    # Try each command pattern
+    for parser in [_parse_submit, _parse_inspect, _parse_delete, _parse_fix]:
+        result = parser(text)
+        if result is not None:
+            return result
+
+    return ParsedAction(
+        "error",
+        {},
+        f"Could not parse: '{raw.strip()[:80]}'. "
+        "Expected: inspect(\"col\"), fix(row, \"col\", \"val\"), delete(row), or submit()",
+    )
+
+
+def _parse_submit(text: str) -> Optional[ParsedAction]:
+    if re.match(r"^submit\s*(\(\s*\))?\s*$", text, re.IGNORECASE):
+        return ParsedAction("submit", {})
+    return None
+
+
+def _parse_inspect(text: str) -> Optional[ParsedAction]:
+    m = re.match(
+        r'^inspect\s*\(\s*(["\']?)(\w+)\1\s*\)$', text, re.IGNORECASE
+    )
+    if m:
+        return ParsedAction("inspect", {"column": m.group(2)})
+    return None
+
+
+def _parse_delete(text: str) -> Optional[ParsedAction]:
+    m = re.match(r"^delete\s*\(\s*(\d+)\s*\)$", text, re.IGNORECASE)
+    if m:
+        return ParsedAction("delete", {"row": int(m.group(1))})
+    return None
+
+
+def _parse_fix(text: str) -> Optional[ParsedAction]:
+    # fix(row, "column", "value") — value may contain commas, quotes, parens
+    # Strategy: match the row and column greedily, then take everything else as value
+    m = re.match(
+        r'^fix\s*\(\s*(\d+)\s*,\s*(["\']?)(\w+)\2\s*,\s*(.+)\)$',
+        text,
+        re.IGNORECASE | re.DOTALL,
+    )
+    if m:
+        row = int(m.group(1))
+        column = m.group(3)
+        value = _strip_quotes(m.group(4).strip())
+        return ParsedAction("fix", {"row": row, "column": column, "value": value})
+
+    # Fallback: more permissive pattern for LLMs that format differently
+    m = re.match(
+        r'^fix\s*\(\s*row\s*=\s*(\d+)\s*,\s*(?:column|col)\s*=\s*(["\']?)(\w+)\2\s*,\s*(?:value|val)\s*=\s*(.+)\)$',
+        text,
+        re.IGNORECASE | re.DOTALL,
+    )
+    if m:
+        row = int(m.group(1))
+        column = m.group(3)
+        value = _strip_quotes(m.group(4).strip())
+        return ParsedAction("fix", {"row": row, "column": column, "value": value})
+
+    return None

data_clean_env/server/app.py

@@ -0,0 +1,28 @@
+"""FastAPI application for the DataClean Environment."""
+
+from openenv.core.env_server.http_server import create_app
+
+try:
+    from .environment import DataCleanEnvironment
+    from ..models import DataCleanAction, DataCleanObservation
+except ImportError:
+    import sys, os
+    sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+    from server.environment import DataCleanEnvironment
+    from models import DataCleanAction, DataCleanObservation
+
+app = create_app(
+    DataCleanEnvironment,
+    DataCleanAction,
+    DataCleanObservation,
+    env_name="data_clean_env",
+)
+
+
+def main():
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8000)
+
+
+if __name__ == "__main__":
+    main()

data_clean_env/server/environment.py

@@ -0,0 +1,488 @@
+"""DataClean Environment — core logic."""
+
+import copy
+from typing import Any, Dict, List, Optional, Set
+from uuid import uuid4
+
+from openenv.core.env_server.types import Action, Observation, State
+
+try:
+    from openenv.core.env_server.interfaces import Environment
+except ImportError:
+    from openenv.core.env_server import Environment
+
+try:
+    from ..models import DataCleanAction, DataCleanObservation, DataCleanState
+    from .action_parser import ParsedAction, parse_action
+    from .graders import (
+        VALIDATORS,
+        validate_date_format,
+        validate_row_deleted,
+        validate_temporal_order,
+    )
+    from .tasks import Issue, TaskDefinition, get_task
+except ImportError:
+    from models import DataCleanAction, DataCleanObservation, DataCleanState
+    from server.action_parser import ParsedAction, parse_action
+    from server.graders import (
+        VALIDATORS,
+        validate_date_format,
+        validate_row_deleted,
+        validate_temporal_order,
+    )
+    from server.tasks import Issue, TaskDefinition, get_task
+
+
+def _format_table(data: List[Dict[str, Any]], columns: List[str], max_rows: int = 50) -> str:
+    """Format data as a readable text table."""
+    if not data:
+        return "(empty dataset)"
+
+    # Calculate column widths
+    widths: Dict[str, int] = {}
+    for col in columns:
+        widths[col] = max(
+            len(col),
+            *(len(str(row.get(col, ""))) for row in data[:max_rows]),
+        )
+        widths[col] = min(widths[col], 30)  # cap width
+
+    # Header
+    hdr = "| row | " + " | ".join(col.ljust(widths[col]) for col in columns) + " |"
+    sep = "|-----|" + "|".join("-" * (widths[col] + 2) for col in columns) + "|"
+    lines = [hdr, sep]
+
+    for i, row in enumerate(data[:max_rows]):
+        vals = []
+        for col in columns:
+            v = str(row.get(col, ""))
+            if len(v) > 30:
+                v = v[:27] + "..."
+            vals.append(v.ljust(widths[col]))
+        lines.append(f"| {str(i).rjust(3)} | " + " | ".join(vals) + " |")
+
+    if len(data) > max_rows:
+        lines.append(f"... ({len(data) - max_rows} more rows)")
+
+    return "\n".join(lines)
+
+
+def _column_stats(data: List[Dict[str, Any]], column: str) -> str:
+    """Generate stats for a single column."""
+    values = [row.get(column, "") for row in data]
+    total = len(values)
+    non_null = sum(1 for v in values if v is not None and str(v).strip() != "")
+    unique = len(set(str(v) for v in values))
+
+    lines = [
+        f"Column: {column}",
+        f"  Total rows: {total}",
+        f"  Non-empty: {non_null}",
+        f"  Unique values: {unique}",
+    ]
+
+    # Try numeric stats
+    nums = []
+    for v in values:
+        try:
+            nums.append(float(v))
+        except (ValueError, TypeError):
+            pass
+
+    if nums and len(nums) > total * 0.5:
+        lines.append(f"  Min: {min(nums)}")
+        lines.append(f"  Max: {max(nums)}")
+        lines.append(f"  Mean: {sum(nums) / len(nums):.2f}")
+    else:
+        # Show sample values for string columns
+        samples = sorted(set(str(v) for v in values if v is not None and str(v).strip()))[:8]
+        lines.append(f"  Sample values: {samples}")
+
+    # Flag suspicious values
+    suspicious = []
+    for i, v in enumerate(values):
+        sv = str(v).strip()
+        if sv == "" or sv.lower() in ("none", "null", "nan"):
+            suspicious.append(f"Row {i}: empty/null")
+        elif "  " in sv:
+            suspicious.append(f"Row {i}: excess whitespace")
+    if suspicious:
+        lines.append(f"  Suspicious: {suspicious[:5]}")
+
+    return "\n".join(lines)
+
+
+class DataCleanEnvironment(Environment):
+    """Data quality analysis and cleaning environment."""
+
+    SUPPORTS_CONCURRENT_SESSIONS = True
+
+    def __init__(self):
+        super().__init__()
+        self._state = DataCleanState(episode_id=str(uuid4()))
+        self._task: Optional[TaskDefinition] = None
+        self._current_data: List[Dict[str, Any]] = []
+        self._issue_status: Dict[str, bool] = {}  # issue_id → resolved
+        self._damaged_cells: int = 0
+        self._actions_taken: List[str] = []
+        self._inspected_columns: Set[str] = set()
+        self._score: float = 0.0
+        self._done: bool = False
+        # Track which cells are known-bad (in issue registry) to detect damage
+        self._bad_cells: Set[tuple] = set()  # (row, column)
+        # For row deletion tracking — map original rows by index
+        self._row_index_map: List[int] = []  # current_idx → original_idx
+
+    def reset(
+        self,
+        seed: Optional[int] = None,
+        episode_id: Optional[str] = None,
+        **kwargs: Any,
+    ) -> DataCleanObservation:
+        task_id = kwargs.get("task_id", "customer_contacts")
+        self._task = get_task(task_id)
+
+        self._current_data = copy.deepcopy(self._task.data)
+        self._issue_status = {issue.issue_id: False for issue in self._task.issues}
+        self._damaged_cells = 0
+        self._actions_taken = []
+        self._inspected_columns = set()
+        self._score = 0.0
+        self._done = False
+        self._row_index_map = list(range(len(self._current_data)))
+
+        # Track which cells are in the issue registry
+        self._bad_cells = set()
+        for issue in self._task.issues:
+            if issue.issue_type != "duplicate_row" and issue.column:
+                self._bad_cells.add((issue.row, issue.column))
+
+        self._state = DataCleanState(
+            episode_id=episode_id or str(uuid4()),
+            step_count=0,
+            task_id=self._task.task_id,
+            difficulty=self._task.difficulty,
+            total_issues=len(self._task.issues),
+            fixed_issues=0,
+            damaged_cells=0,
+            max_steps=self._task.max_steps,
+            score=0.0,
+        )
+
+        return self._build_observation(
+            feedback=f"Task '{self._task.task_id}' loaded ({self._task.difficulty}). "
+            f"Dataset has {len(self._current_data)} rows, {len(self._task.columns)} columns, "
+            f"and {len(self._task.issues)} known issues to fix. "
+            f"You have {self._task.max_steps} steps. Use inspect() to examine columns, "
+            f"fix() to correct values, delete() for duplicates, submit() when done."
+        )
+
+    def step(
+        self,
+        action: DataCleanAction,
+        timeout_s: Optional[float] = None,
+        **kwargs: Any,
+    ) -> DataCleanObservation:
+        if self._task is None:
+            return self._build_observation(
+                feedback="Error: No task loaded. Call reset(task_id=...) first.",
+                done=True,
+            )
+
+        if self._done:
+            return self._build_observation(
+                feedback="Episode already finished. Call reset() to start a new one.",
+                done=True,
+            )
+
+        self._state.step_count += 1
+        self._actions_taken.append(action.command)
+
+        parsed = parse_action(action.command)
+
+        if parsed.command_type == "error":
+            feedback = f"Parse error: {parsed.error_message}"
+        elif parsed.command_type == "submit":
+            feedback = self._handle_submit()
+        elif parsed.command_type == "inspect":
+            feedback = self._handle_inspect(parsed)
+        elif parsed.command_type == "fix":
+            feedback = self._handle_fix(parsed)
+        elif parsed.command_type == "delete":
+            feedback = self._handle_delete(parsed)
+        else:
+            feedback = f"Unknown command type: {parsed.command_type}"
+
+        # Check step limit
+        remaining = self._task.max_steps - self._state.step_count
+        if remaining <= 0 and not self._done:
+            self._done = True
+            self._compute_final_score()
+            feedback += f"\n\nMax steps reached. Final score: {self._score:.4f}"
+
+        self._state.fixed_issues = sum(1 for v in self._issue_status.values() if v)
+        self._state.damaged_cells = self._damaged_cells
+        self._state.score = self._score
+
+        return self._build_observation(feedback=feedback)
+
+    @property
+    def state(self) -> DataCleanState:
+        return self._state
+
+    # ------------------------------------------------------------------
+    # Command handlers
+    # ------------------------------------------------------------------
+
+    def _handle_submit(self) -> str:
+        self._done = True
+        self._compute_final_score()
+        fixed = sum(1 for v in self._issue_status.values() if v)
+        total = len(self._task.issues)
+        return (
+            f"Submitted! Fixed {fixed}/{total} issues. "
+            f"Damaged cells: {self._damaged_cells}. "
+            f"Final score: {self._score:.4f}"
+        )
+
+    def _handle_inspect(self, parsed: ParsedAction) -> str:
+        col = parsed.args["column"]
+        if col not in self._task.columns:
+            return (
+                f"Column '{col}' not found. "
+                f"Available: {self._task.columns}"
+            )
+
+        self._inspected_columns.add(col)
+        stats = _column_stats(self._current_data, col)
+        desc = self._task.column_descriptions.get(col, "")
+
+        # Count issues in this column
+        col_issues = [
+            i for i in self._task.issues
+            if i.column == col and not self._issue_status.get(i.issue_id, False)
+        ]
+        hint = f"\n  Issues remaining in this column: {len(col_issues)}"
+
+        return f"Inspecting '{col}' ({desc}):\n{stats}{hint}"
+
+    def _handle_fix(self, parsed: ParsedAction) -> str:
+        row_idx = parsed.args["row"]
+        col = parsed.args["column"]
+        new_value = parsed.args["value"]
+
+        if row_idx < 0 or row_idx >= len(self._current_data):
+            return f"Row {row_idx} out of range (0-{len(self._current_data) - 1})"
+
+        if col not in self._task.columns:
+            return f"Column '{col}' not found. Available: {self._task.columns}"
+
+        # Get the original row index (to match issue registry)
+        orig_idx = self._row_index_map[row_idx]
+        old_value = self._current_data[row_idx].get(col, "")
+
+        # Check if this cell is in the issue registry
+        cell_in_issue = (orig_idx, col) in self._bad_cells
+
+        if not cell_in_issue:
+            # Agent is modifying a cell that had no known issue — damage
+            self._current_data[row_idx][col] = new_value
+            self._damaged_cells += 1
+            self._score = max(0.0, self._score - 0.05)
+            return (
+                f"Warning: Row {row_idx}, column '{col}' had no known issue. "
+                f"Original value '{old_value}' was overwritten. Penalty applied (-0.05)."
+            )
+
+        # Apply the fix
+        # Try to convert to appropriate type
+        converted = self._convert_value(new_value, old_value)
+        self._current_data[row_idx][col] = converted
+
+        # Check all issues involving this cell
+        reward_delta = 0.0
+        fixed_any = False
+        for issue in self._task.issues:
+            if issue.row != orig_idx or issue.column != col:
+                continue
+            if self._issue_status.get(issue.issue_id, False):
+                continue  # already resolved
+
+            if self._check_issue_resolved(issue):
+                self._issue_status[issue.issue_id] = True
+                reward_delta += 1.0 / len(self._task.issues)
+                fixed_any = True
+
+        self._score = min(1.0, max(0.0, self._score + reward_delta))
+
+        if fixed_any:
+            return (
+                f"Fixed: Row {row_idx}, '{col}' changed from '{old_value}' to '{converted}'. "
+                f"Score: {self._score:.4f}"
+            )
+        else:
+            return (
+                f"Changed: Row {row_idx}, '{col}' from '{old_value}' to '{converted}', "
+                f"but the issue is not yet resolved. Check the expected format. "
+                f"Score: {self._score:.4f}"
+            )
+
+    def _handle_delete(self, parsed: ParsedAction) -> str:
+        row_idx = parsed.args["row"]
+
+        if row_idx < 0 or row_idx >= len(self._current_data):
+            return f"Row {row_idx} out of range (0-{len(self._current_data) - 1})"
+
+        orig_idx = self._row_index_map[row_idx]
+        deleted_row = self._current_data[row_idx]
+
+        # Check if this row is a known duplicate
+        reward_delta = 0.0
+        is_duplicate = False
+        for issue in self._task.issues:
+            if issue.issue_type != "duplicate_row":
+                continue
+            if issue.row != orig_idx:
+                continue
+            if self._issue_status.get(issue.issue_id, False):
+                continue
+
+            # The agent is deleting the row that matches the duplicate issue
+            self._issue_status[issue.issue_id] = True
+            reward_delta += 1.0 / len(self._task.issues)
+            is_duplicate = True
+
+        if not is_duplicate:
+            # Deleting a non-duplicate row — penalty
+            self._damaged_cells += 1
+            reward_delta = -0.05
+
+        # Actually remove the row
+        self._current_data.pop(row_idx)
+        self._row_index_map.pop(row_idx)
+
+        self._score = min(1.0, max(0.0, self._score + reward_delta))
+
+        if is_duplicate:
+            return (
+                f"Deleted duplicate row {row_idx}. Score: {self._score:.4f}"
+            )
+        else:
+            return (
+                f"Warning: Row {row_idx} was not a known duplicate. "
+                f"Penalty applied (-0.05). Score: {self._score:.4f}"
+            )
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    def _check_issue_resolved(self, issue: Issue) -> bool:
+        """Check if an issue is resolved in the current data."""
+        if issue.issue_type == "duplicate_row":
+            return validate_row_deleted(self._current_data, issue.original_row_data)
+
+        if issue.issue_type == "temporal_inconsistency":
+            # Find the row by original index
+            current_row_idx = self._find_current_row(issue.row)
+            if current_row_idx is None:
+                return False
+            row = self._current_data[current_row_idx]
+            hire = str(row.get("hire_date", ""))
+            term = str(row.get("termination_date", ""))
+            return validate_temporal_order(hire, term)
+
+        # Standard validation
+        validator = VALIDATORS.get(issue.issue_type)
+        if validator is None:
+            return False
+
+        current_row_idx = self._find_current_row(issue.row)
+        if current_row_idx is None:
+            return False
+
+        val = self._current_data[current_row_idx].get(issue.column, "")
+        return validator(val, **issue.validation_params)
+
+    def _find_current_row(self, original_idx: int) -> Optional[int]:
+        """Find current row index from original index."""
+        try:
+            return self._row_index_map.index(original_idx)
+        except ValueError:
+            return None  # row was deleted
+
+    def _convert_value(self, new_value: str, old_value: Any) -> Any:
+        """Try to convert new_value to match the type of old_value."""
+        if isinstance(old_value, int):
+            try:
+                return int(float(new_value))
+            except (ValueError, TypeError):
+                return new_value
+        if isinstance(old_value, float):
+            try:
+                return float(new_value)
+            except (ValueError, TypeError):
+                return new_value
+        return new_value
+
+    def _compute_final_score(self) -> None:
+        """Recompute score based on all current issue states."""
+        total = len(self._task.issues)
+        if total == 0:
+            self._score = 1.0
+            return
+
+        resolved = 0
+        for issue in self._task.issues:
+            if self._issue_status.get(issue.issue_id, False):
+                resolved += 1
+                continue
+            # One last check in case fixes propagated
+            if self._check_issue_resolved(issue):
+                self._issue_status[issue.issue_id] = True
+                resolved += 1
+
+        self._score = resolved / total
+        # Apply damage penalty
+        self._score = max(0.0, self._score - self._damaged_cells * 0.05)
+        self._score = min(1.0, self._score)
+
+    def _build_observation(
+        self, feedback: str = "", done: Optional[bool] = None,
+    ) -> DataCleanObservation:
+        if done is None:
+            done = self._done
+
+        task = self._task
+        if task is None:
+            return DataCleanObservation(
+                done=done,
+                reward=self._score,
+                feedback=feedback,
+            )
+
+        fixed = sum(1 for v in self._issue_status.values() if v)
+        remaining = max(0, task.max_steps - self._state.step_count)
+
+        col_info_parts = []
+        for col in task.columns:
+            desc = task.column_descriptions.get(col, "")
+            col_info_parts.append(f"  {col}: {desc}")
+        col_info = "\n".join(col_info_parts)
+
+        return DataCleanObservation(
+            done=done,
+            reward=self._score if done else None,
+            task_id=task.task_id,
+            task_description=task.description,
+            difficulty=task.difficulty,
+            data_preview=_format_table(self._current_data, task.columns),
+            column_info=col_info,
+            feedback=feedback,
+            actions_remaining=remaining,
+            issues_found=len(self._inspected_columns),
+            issues_fixed=fixed,
+            total_issues=len(task.issues),
+            current_score=round(self._score, 4),
+            action_history=list(self._actions_taken[-10:]),
+        )

data_clean_env/server/graders.py

@@ -0,0 +1,123 @@
+"""Validation-based grading functions for data quality issues."""
+
+import re
+from datetime import datetime
+from typing import Any, Dict, List, Set
+
+
+def validate_email(value: Any) -> bool:
+    """Value must be a valid email: user@domain.tld"""
+    s = str(value).strip()
+    return bool(re.match(r"^[\w.+\-]+@[\w\-]+\.\w{2,}$", s))
+
+
+def validate_phone(value: Any) -> bool:
+    """Value must contain only digits, dashes, spaces, parens, plus. At least 10 digits."""
+    s = str(value).strip()
+    if not re.match(r"^[\d\-\(\)\s\+\.]+$", s):
+        return False
+    digits = re.sub(r"\D", "", s)
+    return len(digits) >= 10
+
+
+def validate_date_format(value: Any) -> bool:
+    """Value must be a valid YYYY-MM-DD date."""
+    s = str(value).strip()
+    try:
+        datetime.strptime(s, "%Y-%m-%d")
+        return True
+    except ValueError:
+        return False
+
+
+def validate_non_empty(value: Any) -> bool:
+    """Value must not be empty or whitespace-only."""
+    if value is None:
+        return False
+    return str(value).strip() != ""
+
+
+def validate_positive_number(value: Any) -> bool:
+    """Value must be a positive number."""
+    try:
+        return float(value) > 0
+    except (ValueError, TypeError):
+        return False
+
+
+def validate_in_range(value: Any, low: float, high: float) -> bool:
+    """Value must be a number within [low, high]."""
+    try:
+        v = float(value)
+        return low <= v <= high
+    except (ValueError, TypeError):
+        return False
+
+
+def validate_canonical(value: Any, canonical_set: Set[str]) -> bool:
+    """Value must exactly match one of the canonical values."""
+    return str(value).strip() in canonical_set
+
+
+def validate_no_excess_whitespace(value: Any) -> bool:
+    """Value must be trimmed with no double spaces."""
+    s = str(value)
+    return s == s.strip() and "  " not in s
+
+
+def validate_referential_integrity(
+    value: Any, valid_ids: Set[str]
+) -> bool:
+    """Value must be an ID that exists in the given set."""
+    return str(value).strip() in valid_ids
+
+
+def validate_temporal_order(
+    hire_date: str, termination_date: str
+) -> bool:
+    """Termination date must be after hire date. Empty termination is valid."""
+    t = str(termination_date).strip()
+    if not t or t.lower() in ("", "none", "null", "nat"):
+        return True
+    try:
+        h = datetime.strptime(str(hire_date).strip(), "%Y-%m-%d")
+        td = datetime.strptime(t, "%Y-%m-%d")
+        return td > h
+    except ValueError:
+        return False
+
+
+def validate_row_deleted(
+    current_data: List[Dict[str, Any]], original_row: Dict[str, Any]
+) -> bool:
+    """Check that a specific original row no longer exists in the dataset."""
+    for row in current_data:
+        if all(
+            str(row.get(k, "")) == str(v) for k, v in original_row.items()
+        ):
+            return False
+    return True
+
+
+# Issue type → validation function mapping
+VALIDATORS = {
+    "invalid_email": lambda val, **_: validate_email(val),
+    "invalid_phone": lambda val, **_: validate_phone(val),
+    "wrong_date_format": lambda val, **_: validate_date_format(val),
+    "invalid_date": lambda val, **_: validate_date_format(val),
+    "missing_value": lambda val, **_: validate_non_empty(val),
+    "negative_number": lambda val, **_: validate_positive_number(val),
+    "outlier": lambda val, low=0, high=0, **_: validate_in_range(val, low, high),
+    "inconsistent_format": lambda val, canonical_set=frozenset(), **_: validate_canonical(
+        val, canonical_set
+    ),
+    "excess_whitespace": lambda val, **_: validate_no_excess_whitespace(val),
+    "referential_integrity": lambda val, valid_ids=frozenset(), **_: validate_referential_integrity(
+        val, valid_ids
+    ),
+    "duplicate_row": None,  # Handled separately via validate_row_deleted
+    "temporal_inconsistency": None,  # Handled separately with two columns
+    "score_out_of_range": lambda val, low=0, high=10, **_: validate_in_range(
+        val, low, high
+    ),
+}

data_clean_env/server/tasks.py

@@ -0,0 +1,315 @@
+"""Hardcoded task definitions with datasets and issue registries."""
+
+import copy
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Set
+
+
+@dataclass
+class Issue:
+    issue_id: str
+    row: int
+    column: str
+    issue_type: str
+    description: str
+    # Extra params for validation (e.g. canonical_set, low/high range)
+    validation_params: Dict[str, Any] = field(default_factory=dict)
+    # For duplicate_row issues, store the original row data
+    original_row_data: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class TaskDefinition:
+    task_id: str
+    difficulty: str
+    description: str
+    columns: List[str]
+    data: List[Dict[str, Any]]
+    issues: List[Issue]
+    max_steps: int
+    column_descriptions: Dict[str, str]
+
+
+# ---------------------------------------------------------------------------
+# EASY: Customer Contacts
+# ---------------------------------------------------------------------------
+_EASY_DATA = [
+    {"name": "John Smith", "email": "john.smith@gmail.com", "phone": "555-012-3401", "city": "New York", "signup_date": "2024-01-15"},
+    {"name": "Jane Doe", "email": "jane.doe@outlook.com", "phone": "555-012-3402", "city": "Los Angeles", "signup_date": "2024-02-20"},
+    {"name": "Bob Johnson", "email": "bob.j@company.com", "phone": "555-012-3403", "city": "Chicago", "signup_date": "2024-03-10"},
+    {"name": "Alice Brown", "email": "alice.brown[at]mail.com", "phone": "555-012-3404", "city": "Houston", "signup_date": "2024-04-05"},  # E1: invalid email
+    {"name": "Charlie Wilson", "email": "charlie@example.com", "phone": "555-012-3405", "city": "Phoenix", "signup_date": "2024-05-12"},
+    {"name": "Diana Davis", "email": "diana@example.com", "phone": "555-012-3406", "city": "", "signup_date": "2024-06-18"},  # E3: empty city
+    {"name": "Eve Martinez", "email": "eve@example.com", "phone": "555-012-3407", "city": "Philadelphia", "signup_date": "2024-07-22"},
+    {"name": "Frank Taylor", "email": "frank@example.com", "phone": "55A-0B2-34C8", "city": "San Antonio", "signup_date": "2024-08-30"},  # E2: phone has letters
+    {"name": "Grace Lee", "email": "grace@example.com", "phone": "555-012-3409", "city": "San Diego", "signup_date": "2024-09-14"},
+    {"name": "Hank Moore", "email": "hank@@domain.com", "phone": "555-012-3410", "city": "Dallas", "signup_date": "2024-10-01"},  # E5: double @@
+    {"name": "Ivy Clark", "email": "ivy@example.com", "phone": "555-012-3411", "city": "San Jose", "signup_date": "2024-11-18"},
+    {"name": "Jack White", "email": "jack@example.com", "phone": "555-012-3412", "city": "Austin", "signup_date": "03/25/2024"},  # E4: wrong date format
+    {"name": "Karen Lewis", "email": "karen@example.com", "phone": "555-012-3413", "city": "Jacksonville", "signup_date": "2025-01-03"},
+    {"name": "Leo Walker", "email": "leo@example.com", "phone": "555-012-3414", "city": "Columbus", "signup_date": "2025-02-14"},
+    {"name": "John Smith", "email": "john.smith@gmail.com", "phone": "555-012-3401", "city": "New York", "signup_date": "2024-01-15"},  # E6: exact duplicate of row 0
+]
+
+_EASY_ISSUES = [
+    Issue("E1", 3, "email", "invalid_email", "Email uses '[at]' instead of '@'"),
+    Issue("E2", 7, "phone", "invalid_phone", "Phone number contains letters"),
+    Issue("E3", 5, "city", "missing_value", "City is empty"),
+    Issue("E4", 11, "signup_date", "wrong_date_format", "Date is MM/DD/YYYY instead of YYYY-MM-DD"),
+    Issue("E5", 9, "email", "invalid_email", "Email has double @@ symbol"),
+    Issue(
+        "E6", 14, "", "duplicate_row", "Exact duplicate of row 0",
+        original_row_data={"name": "John Smith", "email": "john.smith@gmail.com", "phone": "555-012-3401", "city": "New York", "signup_date": "2024-01-15"},
+    ),
+]
+
+EASY_TASK = TaskDefinition(
+    task_id="customer_contacts",
+    difficulty="easy",
+    description=(
+        "You are cleaning a customer contact list for a CRM import. "
+        "The data should have valid emails (user@domain.tld), phone numbers "
+        "(digits and dashes only, 10+ digits), non-empty cities, dates in "
+        "YYYY-MM-DD format, and no duplicate rows. Find and fix all data "
+        "quality issues."
+    ),
+    columns=["name", "email", "phone", "city", "signup_date"],
+    data=_EASY_DATA,
+    issues=_EASY_ISSUES,
+    max_steps=15,
+    column_descriptions={
+        "name": "Customer full name",
+        "email": "Email address (must be user@domain.tld)",
+        "phone": "Phone number (digits and dashes, 10+ digits)",
+        "city": "City of residence (must not be empty)",
+        "signup_date": "Signup date (must be YYYY-MM-DD format)",
+    },
+)
+
+
+# ---------------------------------------------------------------------------
+# MEDIUM: Sales Records
+# ---------------------------------------------------------------------------
+_MEDIUM_DATA = [
+    {"order_id": "ORD-1001", "customer_name": "Acme Corp", "product": "Widget A", "quantity": 10, "unit_price": 29.99, "order_date": "2024-01-15", "region": "Northeast"},
+    {"order_id": "ORD-1002", "customer_name": "Beta Inc", "product": "Widget B", "quantity": 5, "unit_price": 49.99, "order_date": "2024-01-22", "region": "Southeast"},
+    {"order_id": "ORD-1003", "customer_name": "Gamma LLC", "product": "Gadget X", "quantity": 20, "unit_price": 15.00, "order_date": "2024-02-03", "region": "Midwest"},
+    {"order_id": "ORD-1004", "customer_name": "Delta Co", "product": "Widget A", "quantity": 8, "unit_price": 29.99, "order_date": "2024-02-10", "region": "north-east"},  # M6: inconsistent region
+    {"order_id": "ORD-1005", "customer_name": "Epsilon Ltd", "product": "Gadget Y", "quantity": 3, "unit_price": 89.50, "order_date": "Jan 15, 2024", "region": "West"},  # M1: wrong date format
+    {"order_id": "ORD-1006", "customer_name": "Zeta Group", "product": "Widget C", "quantity": 12, "unit_price": 35.00, "order_date": "2024-03-01", "region": "Northwest"},
+    {"order_id": "", "customer_name": "Eta Partners", "product": "Gadget X", "quantity": 7, "unit_price": 15.00, "order_date": "2024-03-12", "region": "Southeast"},  # M10: missing order_id
+    {"order_id": "ORD-1008", "customer_name": "Theta Corp", "product": "Widget B", "quantity": 15, "unit_price": 49.99, "order_date": "2024-03-20", "region": "Midwest"},
+    {"order_id": "ORD-1009", "customer_name": "Iota Inc", "product": "Gadget Z", "quantity": -5, "unit_price": 120.00, "order_date": "2024-04-02", "region": "Northeast"},  # M3: negative quantity
+    {"order_id": "ORD-1010", "customer_name": "Kappa LLC", "product": "Widget A", "quantity": 25, "unit_price": 29.99, "order_date": "2024-04-15", "region": "West"},
+    {"order_id": "ORD-1011", "customer_name": "Lambda Co", "product": "Gadget Y", "quantity": 6, "unit_price": 89.50, "order_date": "2024-04-28", "region": "Southeast"},
+    {"order_id": "ORD-1012", "customer_name": "Mu Ltd", "product": "Widget C", "quantity": 30, "unit_price": 35.00, "order_date": "2024-05-05", "region": "Northeast"},
+    {"order_id": "ORD-1013", "customer_name": "Nu Group", "product": "Gadget X", "quantity": 4, "unit_price": 15.00, "order_date": "2024/05/18", "region": "Midwest"},  # M2: slash date format
+    {"order_id": "ORD-1014", "customer_name": "Xi Partners", "product": "Widget B", "quantity": 9, "unit_price": 49.99, "order_date": "2024-06-01", "region": "Northwest"},
+    {"order_id": "ORD-1015", "customer_name": "Omicron Corp", "product": "Gadget Z", "quantity": 2, "unit_price": 29999.99, "order_date": "2024-06-15", "region": "West"},  # M5: price outlier
+    {"order_id": "ORD-1016", "customer_name": "Pi Inc", "product": "Widget A", "quantity": 18, "unit_price": 29.99, "order_date": "2024-06-28", "region": "Northeast"},
+    {"order_id": "ORD-1017", "customer_name": "Rho LLC", "product": "Gadget Y", "quantity": 11, "unit_price": 89.50, "order_date": "2024-07-10", "region": "Southeast"},
+    {"order_id": "ORD-1018", "customer_name": "  Sigma  Co  ", "product": "Widget C", "quantity": 7, "unit_price": 35.00, "order_date": "2024-07-22", "region": "Midwest"},  # M11: excess whitespace
+    {"order_id": "ORD-1019", "customer_name": "Tau Group", "product": "Gadget X", "quantity": 14, "unit_price": 15.00, "order_date": "2024-08-05", "region": "Northwest"},
+    {"order_id": "ORD-1020", "customer_name": "Upsilon Ltd", "product": "Widget B", "quantity": -12, "unit_price": 49.99, "order_date": "2024-08-18", "region": "Northeast"},  # M4: negative quantity
+    {"order_id": "ORD-1021", "customer_name": "Phi Corp", "product": "Gadget Z", "quantity": 8, "unit_price": -15.50, "order_date": "2024-09-01", "region": "West"},  # M12: negative price
+    {"order_id": "ORD-1022", "customer_name": "Chi Inc", "product": "Widget A", "quantity": 22, "unit_price": 29.99, "order_date": "2024-09-15", "region": "Southeast"},
+    {"order_id": "ORD-1023", "customer_name": "Psi LLC", "product": "Gadget Y", "quantity": 3, "unit_price": 89.50, "order_date": "2024-09-28", "region": "WEST"},  # M7: inconsistent case
+    {"order_id": "ORD-1024", "customer_name": "Omega Co", "product": "Widget C", "quantity": 16, "unit_price": 35.00, "order_date": "2024-10-10", "region": "Midwest"},
+    {"order_id": "ORD-1025", "customer_name": "Alpha2 Group", "product": "Gadget X", "quantity": 9, "unit_price": 15.00, "order_date": "2024-10-22", "region": "Northwest"},
+    {"order_id": "ORD-1011", "customer_name": "Lambda Co", "product": "Gadget Y", "quantity": 6, "unit_price": 89.50, "order_date": "2024-04-28", "region": "Southeast"},  # M9: duplicate of row 10
+    {"order_id": "ORD-1027", "customer_name": "Beta2 Inc", "product": "Widget B", "quantity": 13, "unit_price": 49.99, "order_date": "2024-11-15", "region": "Northeast"},
+    {"order_id": "ORD-1028", "customer_name": "Gamma2 LLC", "product": "Gadget Z", "quantity": 5, "unit_price": 120.00, "order_date": "2024-11-28", "region": "South East"},  # M8: inconsistent region
+    {"order_id": "ORD-1029", "customer_name": "Delta2 Co", "product": "Widget A", "quantity": 20, "unit_price": 29.99, "order_date": "2024-12-10", "region": "West"},
+    {"order_id": "ORD-1030", "customer_name": "Epsilon2 Ltd", "product": "Gadget Y", "quantity": 7, "unit_price": 89.50, "order_date": "2024-12-22", "region": "Midwest"},
+]
+
+_VALID_REGIONS: Set[str] = {"Northeast", "Southeast", "Midwest", "West", "Northwest"}
+
+_MEDIUM_ISSUES = [
+    Issue("M1", 4, "order_date", "wrong_date_format", "Date is 'Jan 15, 2024' instead of YYYY-MM-DD"),
+    Issue("M2", 12, "order_date", "wrong_date_format", "Date uses slashes '2024/05/18' instead of YYYY-MM-DD"),
+    Issue("M3", 8, "quantity", "negative_number", "Quantity is negative (-5)"),
+    Issue("M4", 19, "quantity", "negative_number", "Quantity is negative (-12)"),
+    Issue("M5", 14, "unit_price", "outlier", "Price 29999.99 is ~1000x normal range",
+          validation_params={"low": 1.0, "high": 500.0}),
+    Issue("M6", 3, "region", "inconsistent_format", "Region 'north-east' should match canonical form",
+          validation_params={"canonical_set": _VALID_REGIONS}),
+    Issue("M7", 22, "region", "inconsistent_format", "Region 'WEST' should match canonical form",
+          validation_params={"canonical_set": _VALID_REGIONS}),
+    Issue("M8", 27, "region", "inconsistent_format", "Region 'South East' should match canonical form",
+          validation_params={"canonical_set": _VALID_REGIONS}),
+    Issue(
+        "M9", 25, "", "duplicate_row", "Exact duplicate of row 10 (same order_id, customer, product)",
+        original_row_data={"order_id": "ORD-1011", "customer_name": "Lambda Co", "product": "Gadget Y", "quantity": 6, "unit_price": 89.50, "order_date": "2024-04-28", "region": "Southeast"},
+    ),
+    Issue("M10", 6, "order_id", "missing_value", "Order ID is empty"),
+    Issue("M11", 17, "customer_name", "excess_whitespace", "Customer name has excess whitespace"),
+    Issue("M12", 20, "unit_price", "negative_number", "Price is negative (-15.50)"),
+]
+
+MEDIUM_TASK = TaskDefinition(
+    task_id="sales_records",
+    difficulty="medium",
+    description=(
+        "You are cleaning sales transaction records for a quarterly report. "
+        "Requirements: order_id must be non-empty (format ORD-XXXX), dates must "
+        "be YYYY-MM-DD, quantities and prices must be positive numbers, prices "
+        "should be in a reasonable range ($1-$500), regions must be one of: "
+        "Northeast, Southeast, Midwest, West, Northwest. Names should have no "
+        "excess whitespace. No duplicate orders."
+    ),
+    columns=["order_id", "customer_name", "product", "quantity", "unit_price", "order_date", "region"],
+    data=_MEDIUM_DATA,
+    issues=_MEDIUM_ISSUES,
+    max_steps=25,
+    column_descriptions={
+        "order_id": "Unique order identifier (format: ORD-XXXX, must not be empty)",
+        "customer_name": "Customer/company name (no excess whitespace)",
+        "product": "Product name",
+        "quantity": "Order quantity (must be positive)",
+        "unit_price": "Price per unit in USD (must be positive, reasonable range $1-$500)",
+        "order_date": "Order date (must be YYYY-MM-DD)",
+        "region": "Sales region (must be: Northeast, Southeast, Midwest, West, or Northwest)",
+    },
+)
+
+
+# ---------------------------------------------------------------------------
+# HARD: Employee Records
+# ---------------------------------------------------------------------------
+_HARD_DATA = [
+    {"emp_id": "EMP-001", "name": "Sarah Chen", "email": "sarah.chen@company.com", "department": "Engineering", "hire_date": "2020-03-15", "termination_date": "", "salary": 95000, "manager_id": "EMP-010", "performance_score": 8.5},
+    {"emp_id": "EMP-002", "name": "James Wilson", "email": "james.w@company.com", "department": "Marketing", "hire_date": "2019-07-01", "termination_date": "", "salary": 78000, "manager_id": "EMP-010", "performance_score": 7.2},
+    {"emp_id": "EMP-003", "name": "Maria Garcia", "email": "maria.g@company.com", "department": "Engineering", "hire_date": "2021-01-10", "termination_date": "", "salary": 15000000, "manager_id": "EMP-001", "performance_score": 9.1},  # H5: salary outlier (15M)
+    {"emp_id": "EMP-004", "name": "David Kim", "email": "david.kim@company.com", "department": "Sales", "hire_date": "2020-09-20", "termination_date": "", "salary": 72000, "manager_id": "EMP-010", "performance_score": 6.8},
+    {"emp_id": "EMP-005", "name": "Emily Patel", "email": "emily.p@company.com", "department": "HR", "hire_date": "2018-04-05", "termination_date": "", "salary": 82000, "manager_id": "EMP-010", "performance_score": 8.0},
+    {"emp_id": "EMP-006", "name": "Michael Brown", "email": "michael.b@company.com", "department": "Engineering", "hire_date": "2022-06-15", "termination_date": "", "salary": 88000, "manager_id": "EMP-099", "performance_score": 7.5},  # H1: manager doesn't exist
+    {"emp_id": "EMP-007", "name": "  Robert   Williams  ", "email": "robert.w@company.com", "department": "Finance", "hire_date": "2019-11-01", "termination_date": "", "salary": 91000, "manager_id": "EMP-010", "performance_score": 8.3},  # H17: excess whitespace in name
+    {"emp_id": "EMP-008", "name": "Lisa Anderson", "email": "lisa.a@company.com", "department": "Sales", "hire_date": "2024-03-15", "termination_date": "2023-01-10", "salary": 68000, "manager_id": "EMP-004", "performance_score": 5.5},  # H3: termination before hire
+    {"emp_id": "EMP-009", "name": "Kevin Taylor", "email": "kevin.t@company.com", "department": "Engineering", "hire_date": "2021-08-20", "termination_date": "", "salary": 93000, "manager_id": "EMP-001", "performance_score": 11.5},  # H9: score > 10
+    {"emp_id": "EMP-010", "name": "Jennifer Martinez", "email": "jennifer.m@company.com", "department": "Operations", "hire_date": "2017-01-15", "termination_date": "", "salary": 120000, "manager_id": "", "performance_score": 9.0},
+    {"emp_id": "EMP-011", "name": "Alice Jones", "email": "alice.jones@", "department": "Marketing", "hire_date": "2022-02-01", "termination_date": "", "salary": 75000, "manager_id": "EMP-002", "performance_score": 7.8},  # H7: incomplete email
+    {"emp_id": "EMP-012", "name": "Thomas Lee", "email": "thomas.l@company.com", "department": "Engineering", "hire_date": "2020-10-10", "termination_date": "", "salary": 97000, "manager_id": "EMP-001", "performance_score": 8.7},
+    {"emp_id": "EMP-013", "name": "Rachel Green", "email": "rachel.g@company.com", "department": "Engg", "hire_date": "2023-04-15", "termination_date": "", "salary": 85000, "manager_id": "EMP-001", "performance_score": 7.0},  # H11: department abbreviation
+    {"emp_id": "EMP-014", "name": "Daniel White", "email": "daniel.w@company.com", "department": "Finance", "hire_date": "2021-05-20", "termination_date": "", "salary": 89000, "manager_id": "EMP-007", "performance_score": 8.1},
+    {"emp_id": "EMP-015", "name": "Sophie Clark", "email": "sophie.c@company.com", "department": "HR", "hire_date": "2023-06-01", "termination_date": "2023-05-15", "salary": 71000, "manager_id": "EMP-005", "performance_score": 6.0},  # H4: termination before hire
+    {"emp_id": "EMP-016", "name": "Chris Johnson", "email": "chris.j@company.com", "department": "Sales", "hire_date": "2020-12-01", "termination_date": "", "salary": 76000, "manager_id": "EMP-004", "performance_score": 7.3},
+    {"emp_id": "EMP-003", "name": "Maria R. Garcia", "email": "maria.g@company.com", "department": "Engineering", "hire_date": "2021-01-10", "termination_date": "", "salary": 15000000, "manager_id": "EMP-001", "performance_score": 9.1},  # H14: semantic dup of row 2 (same emp_id, slightly different name)
+    {"emp_id": "EMP-017", "name": "Amanda Davis", "email": "amanda.d@company.com", "department": "Marketing", "hire_date": "2022-09-15", "termination_date": "", "salary": 73000, "manager_id": "EMP-002", "performance_score": 7.6},
+    {"emp_id": "EMP-018", "name": "Ryan Thomas", "email": "ryan.t@company.com", "department": "Engineering", "hire_date": "2023-01-20", "termination_date": "", "salary": 90000, "manager_id": "EMP-088", "performance_score": 8.0},  # H2: manager doesn't exist
+    {"emp_id": "EMP-019", "name": "Nicole Brown", "email": "nicole.b@company.com", "department": "Finance", "hire_date": "2021-03-01", "termination_date": "", "salary": 87000, "manager_id": "EMP-007", "performance_score": 8.4},
+    {"emp_id": "EMP-020", "name": "Jason Miller", "email": "jason.m@company.com", "department": "Operations", "hire_date": "2024-11-15", "termination_date": "2024-06-30", "salary": 79000, "manager_id": "EMP-010", "performance_score": 6.5},  # H16: termination before hire
+    {"emp_id": "EMP-021", "name": "Laura Wilson", "email": "laura.w@company.com", "department": "Sales", "hire_date": "2022-11-01", "termination_date": "", "salary": 74000, "manager_id": "EMP-004", "performance_score": 7.1},
+    {"emp_id": "EMP-022", "name": "Mark Thompson", "email": "mark.t@company.com", "department": "Engineering", "hire_date": "2019-05-15", "termination_date": "", "salary": 500, "manager_id": "EMP-001", "performance_score": 8.9},  # H6: salary too low (missing zeros)
+    {"emp_id": "EMP-023", "name": "Patricia Moore", "email": "patricia.m@company.com", "department": "HR", "hire_date": "2023-08-01", "termination_date": "", "salary": 70000, "manager_id": "EMP-005", "performance_score": 6.2},
+    {"emp_id": "EMP-024", "name": "Steven Harris", "email": "steven.h@company.com", "department": "Marketing", "hire_date": "2021-12-10", "termination_date": "", "salary": 77000, "manager_id": "EMP-002", "performance_score": 7.9},
+    {"emp_id": "EMP-025", "name": "Angela Martin", "email": "angela.m@company.com", "department": "Finance", "hire_date": "2020-02-20", "termination_date": "", "salary": 92000, "manager_id": "EMP-007", "performance_score": -2.0},  # H10: negative score
+    {"emp_id": "EMP-026", "name": "Brian Lewis", "email": "brian.l@company.com", "department": "Operations", "hire_date": "2022-04-01", "termination_date": "", "salary": 81000, "manager_id": "EMP-010", "performance_score": 7.4},
+    {"emp_id": "EMP-027", "name": "Michelle Walker", "email": "michelle.w@company.com", "department": "Sales", "hire_date": "2023-10-15", "termination_date": "", "salary": 69000, "manager_id": "EMP-004", "performance_score": 6.7},
+    {"emp_id": "EMP-028", "name": "Paul Robinson", "email": "paul.r@company.com", "department": "marketing", "hire_date": "2021-06-20", "termination_date": "", "salary": 76000, "manager_id": "EMP-002", "performance_score": 7.7},  # H12: lowercase department
+    {"emp_id": "EMP-029", "name": "Sandra Hall", "email": "sandra.h@company.com", "department": "Engineering", "hire_date": "2020-08-10", "termination_date": "", "salary": 94000, "manager_id": "EMP-001", "performance_score": 8.6},
+    {"emp_id": "EMP-030", "name": "Bob Smith", "email": "bob smith@company.com", "department": "Finance", "hire_date": "2022-12-01", "termination_date": "", "salary": 86000, "manager_id": "EMP-007", "performance_score": 7.0},  # H8: space in email
+    {"emp_id": "EMP-031", "name": "Diana Scott", "email": "diana.s@company.com", "department": "HR", "hire_date": "2023-03-15", "termination_date": "", "salary": 72000, "manager_id": "EMP-005", "performance_score": 6.9},
+    {"emp_id": "EMP-032", "name": "George Adams", "email": "george.a@company.com", "department": "Operations", "hire_date": "2021-09-01", "termination_date": "", "salary": 83000, "manager_id": "EMP-010", "performance_score": 7.8},
+    {"emp_id": "EMP-033", "name": "Kevin Taylor", "email": "kevin.t@company.com", "department": "Engineering", "hire_date": "2021-08-20", "termination_date": "", "salary": 93000, "manager_id": "EMP-001", "performance_score": 8.8},  # H15: semantic dup of row 8 (same name+email, diff score)
+    {"emp_id": "EMP-034", "name": "Helen King", "email": "helen.k@company.com", "department": "Sales", "hire_date": "2022-07-10", "termination_date": "", "salary": 71000, "manager_id": "EMP-004", "performance_score": 7.2},
+    {"emp_id": "EMP-035", "name": "Richard Wright", "email": "richard.w@company.com", "department": "Human Resources", "hire_date": "2020-11-20", "termination_date": "", "salary": 80000, "manager_id": "EMP-005", "performance_score": 8.0},  # H13: non-canonical dept
+    {"emp_id": "EMP-036", "name": "Nancy Lopez", "email": "nancy.l@company.com", "department": "Engineering", "hire_date": "2023-05-01", "termination_date": "", "salary": 87000, "manager_id": "EMP-001", "performance_score": 7.3},
+    {"emp_id": "EMP-037", "name": "Carl Hill", "email": "carl.h@company.com", "department": "Marketing", "hire_date": "2021-02-15", "termination_date": "", "salary": 74000, "manager_id": "EMP-002", "performance_score": 7.5},
+    {"emp_id": "EMP-038", "name": "Betty Young", "email": "betty.y@company.com", "department": "Finance", "hire_date": "2025-13-01", "termination_date": "", "salary": 88000, "manager_id": "EMP-007", "performance_score": 8.2},  # H18: invalid date (month 13)
+    {"emp_id": "EMP-039", "name": "Frank Allen", "email": "frank.a@company.com", "department": "Operations", "hire_date": "2022-01-20", "termination_date": "", "salary": 82000, "manager_id": "EMP-010", "performance_score": 7.6},
+]
+
+_VALID_DEPARTMENTS: Set[str] = {"Engineering", "Marketing", "Sales", "HR", "Finance", "Operations"}
+
+# Collect all valid emp_ids (excluding known duplicate rows 16 and 33)
+_VALID_EMP_IDS: Set[str] = {
+    row["emp_id"]
+    for i, row in enumerate(_HARD_DATA)
+    if i not in (16, 33)  # exclude duplicate rows
+}
+
+_HARD_ISSUES = [
+    Issue("H1", 5, "manager_id", "referential_integrity", "Manager EMP-099 does not exist in employee list",
+          validation_params={"valid_ids": _VALID_EMP_IDS}),
+    Issue("H2", 18, "manager_id", "referential_integrity", "Manager EMP-088 does not exist in employee list",
+          validation_params={"valid_ids": _VALID_EMP_IDS}),
+    Issue("H3", 7, "termination_date", "temporal_inconsistency", "Termination date 2023-01-10 is before hire date 2024-03-15"),
+    Issue("H4", 14, "termination_date", "temporal_inconsistency", "Termination date 2023-05-15 is before hire date 2023-06-01"),
+    Issue("H5", 2, "salary", "outlier", "Salary 15,000,000 is unreasonably high (expected $20K-$500K)",
+          validation_params={"low": 20000, "high": 500000}),
+    Issue("H6", 22, "salary", "outlier", "Salary 500 is unreasonably low (expected $20K-$500K)",
+          validation_params={"low": 20000, "high": 500000}),
+    Issue("H7", 10, "email", "invalid_email", "Email 'alice.jones@' is missing domain"),
+    Issue("H8", 30, "email", "invalid_email", "Email 'bob smith@company.com' contains a space"),
+    Issue("H9", 8, "performance_score", "score_out_of_range", "Score 11.5 exceeds the 0-10 scale",
+          validation_params={"low": 0.0, "high": 10.0}),
+    Issue("H10", 25, "performance_score", "score_out_of_range", "Score -2.0 is negative (scale is 0-10)",
+          validation_params={"low": 0.0, "high": 10.0}),
+    Issue("H11", 12, "department", "inconsistent_format", "Department 'Engg' should match canonical name",
+          validation_params={"canonical_set": _VALID_DEPARTMENTS}),
+    Issue("H12", 28, "department", "inconsistent_format", "Department 'marketing' has incorrect casing",
+          validation_params={"canonical_set": _VALID_DEPARTMENTS}),
+    Issue("H13", 35, "department", "inconsistent_format", "Department 'Human Resources' should be canonical 'HR'",
+          validation_params={"canonical_set": _VALID_DEPARTMENTS}),
+    Issue(
+        "H14", 16, "", "duplicate_row", "Semantic duplicate of row 2 (same emp_id EMP-003, slightly different name)",
+        original_row_data={"emp_id": "EMP-003", "name": "Maria R. Garcia", "email": "maria.g@company.com", "department": "Engineering", "hire_date": "2021-01-10", "termination_date": "", "salary": 15000000, "manager_id": "EMP-001", "performance_score": 9.1},
+    ),
+    Issue(
+        "H15", 33, "", "duplicate_row", "Semantic duplicate of row 8 (same name and email as Kevin Taylor)",
+        original_row_data={"emp_id": "EMP-033", "name": "Kevin Taylor", "email": "kevin.t@company.com", "department": "Engineering", "hire_date": "2021-08-20", "termination_date": "", "salary": 93000, "manager_id": "EMP-001", "performance_score": 8.8},
+    ),
+    Issue("H16", 20, "termination_date", "temporal_inconsistency",
+          "Termination date 2024-06-30 is before hire date 2024-11-15"),
+    Issue("H17", 6, "name", "excess_whitespace", "Name '  Robert   Williams  ' has excess whitespace"),
+    Issue("H18", 38, "hire_date", "invalid_date", "Date '2025-13-01' has invalid month 13"),
+]
+
+HARD_TASK = TaskDefinition(
+    task_id="employee_records",
+    difficulty="hard",
+    description=(
+        "You are cleaning employee records for an HR system migration. Requirements: "
+        "emp_id must be unique, emails must be valid (user@domain.tld, no spaces), "
+        "departments must be one of: Engineering, Marketing, Sales, HR, Finance, Operations. "
+        "Dates must be valid YYYY-MM-DD. Termination dates must be after hire dates. "
+        "Salaries must be in range $20,000-$500,000. Performance scores must be 0.0-10.0. "
+        "Manager IDs must reference existing employees. Names must have no excess whitespace. "
+        "Remove duplicate employee entries."
+    ),
+    columns=["emp_id", "name", "email", "department", "hire_date", "termination_date", "salary", "manager_id", "performance_score"],
+    data=_HARD_DATA,
+    issues=_HARD_ISSUES,
+    max_steps=35,
+    column_descriptions={
+        "emp_id": "Unique employee ID (format: EMP-XXX)",
+        "name": "Full name (no excess whitespace)",
+        "email": "Email (valid user@domain.tld, no spaces)",
+        "department": "Department (must be: Engineering, Marketing, Sales, HR, Finance, or Operations)",
+        "hire_date": "Hire date (valid YYYY-MM-DD)",
+        "termination_date": "Termination date (empty if active, must be after hire_date if set)",
+        "salary": "Annual salary USD (range: $20,000-$500,000)",
+        "manager_id": "Manager's emp_id (must reference existing employee, empty for top-level)",
+        "performance_score": "Performance score (0.0 to 10.0)",
+    },
+)
+
+
+# ---------------------------------------------------------------------------
+# Task registry
+# ---------------------------------------------------------------------------
+ALL_TASKS: Dict[str, TaskDefinition] = {
+    "customer_contacts": EASY_TASK,
+    "sales_records": MEDIUM_TASK,
+    "employee_records": HARD_TASK,
+}
+
+
+def get_task(task_id: str) -> TaskDefinition:
+    """Get a deep copy of a task definition."""
+    if task_id not in ALL_TASKS:
+        raise ValueError(
+            f"Unknown task_id '{task_id}'. Available: {list(ALL_TASKS.keys())}"
+        )
+    return copy.deepcopy(ALL_TASKS[task_id])