evaluation/benchmark_tasks.py

"""
Benchmark task definitions — Interview + Real-World Hybrid problems.

Tasks are designed to test:
  - Edge case handling (empty inputs, None, single element)
  - Boundary conditions (off-by-one, integer overflow)
  - Data normalization (messy CSV, unicode, inconsistent types)
  - Algorithm correctness under adversarial inputs
  - Real-world data patterns (logs, intervals, phone numbers)

Each task has a description written for the generator agent and
a reference_test_code suite used to evaluate the final output independently
of the adversarial tests generated by the QA agent. Reference tests are
held-out ground truth — never given to the agent.
"""

from dataclasses import dataclass, field
from typing import Any


@dataclass
class BenchmarkTask:
    task_id: str
    category: str
    description: str
    # Reference tests: list of (input_args, expected_output) tuples
    # Used for final evaluation only — not given to the agent
    reference_tests: list[dict[str, Any]] = field(default_factory=list)
    # Held-out executable Python assert statements for ground-truth evaluation.
    # Never shown to the agent. Run against the agent's final code.
    reference_test_code: str = ""
    # Optional: known tricky aspects for result analysis
    known_difficulty: str = ""


BENCHMARK_TASKS: list[BenchmarkTask] = [
    BenchmarkTask(
        task_id="interval_merge_001",
        category="interval_merging",
        description="""
Write a Python function `merge_intervals(intervals: list[list[int]]) -> list[list[int]]`
that merges all overlapping intervals and returns a sorted list of non-overlapping intervals.

Requirements:
- Input: list of [start, end] pairs (integers). May be unsorted.
- Output: merged, sorted list of [start, end] pairs.
- Handle empty list: return [].
- Handle single interval: return it unchanged.
- Intervals are inclusive on both ends: [1,3] and [3,5] overlap.
- Do not modify the input list.
""".strip(),
        known_difficulty="Off-by-one on touching intervals [1,3],[3,5] → must merge to [1,5]",
        reference_test_code="""
# Reference tests for merge_intervals
assert merge_intervals([]) == []
assert merge_intervals([[1, 3]]) == [[1, 3]]
assert merge_intervals([[1, 3], [3, 5]]) == [[1, 5]]  # touching intervals merge
assert merge_intervals([[1, 4], [2, 3]]) == [[1, 4]]  # contained
assert merge_intervals([[1, 3], [2, 6], [8, 10]]) == [[1, 6], [8, 10]]
assert merge_intervals([[6, 8], [1, 3], [2, 6]]) == [[1, 8]]  # unsorted input
assert merge_intervals([[1, 2], [3, 4]]) == [[1, 2], [3, 4]]  # non-overlapping, gap of 1
assert merge_intervals([[1, 5], [2, 3], [4, 8]]) == [[1, 8]]  # multiple merges
assert merge_intervals([[0, 0], [0, 0]]) == [[0, 0]]  # duplicate zero-length
original = [[1, 3], [2, 6]]
_ = merge_intervals(original)
assert original == [[1, 3], [2, 6]]  # input not modified
""".strip(),
    ),

    BenchmarkTask(
        task_id="csv_normalize_001",
        category="data_normalization",
        description="""
Write a Python function `normalize_csv_row(row: dict) -> dict` that normalizes a
single CSV row with messy data.

Normalization rules:
- Strip leading/trailing whitespace from all string values.
- Convert 'phone' field to digits only (remove spaces, dashes, parentheses, dots).
  If phone is empty or None after stripping, set to empty string.
- Convert 'age' field to int. If invalid or missing, set to -1.
- Convert 'email' to lowercase.
- Convert 'name' to title case.
- All other string fields: strip only.

Input dict may have missing keys — handle gracefully with defaults.
Return a new dict (do not modify the input).
""".strip(),
        known_difficulty="Missing keys, None values, partial phone strings",
        reference_test_code="""
# Reference tests for normalize_csv_row
result = normalize_csv_row({"name": " john doe ", "email": "JOHN@EXAMPLE.COM", "age": "30", "phone": "(555) 123-4567"})
assert result["name"] == "John Doe"
assert result["email"] == "john@example.com"
assert result["age"] == 30
assert result["phone"] == "5551234567"

# Missing keys → defaults
result = normalize_csv_row({})
assert result.get("age", -1) == -1
assert result.get("phone", "") == ""

# None phone
result = normalize_csv_row({"phone": None})
assert result["phone"] == ""

# Invalid age
result = normalize_csv_row({"age": "not_a_number"})
assert result["age"] == -1

# Phone with dots and dashes
result = normalize_csv_row({"phone": "555.123-4567"})
assert result["phone"] == "5551234567"

# Empty phone string
result = normalize_csv_row({"phone": "   "})
assert result["phone"] == ""

# Input not modified
original = {"name": "jane", "age": "25"}
_ = normalize_csv_row(original)
assert original == {"name": "jane", "age": "25"}

# Other string fields: strip only
result = normalize_csv_row({"city": "  Seattle  "})
assert result["city"] == "Seattle"
""".strip(),
    ),

    BenchmarkTask(
        task_id="log_dedup_001",
        category="log_processing",
        description="""
Write a Python function `deduplicate_logs(logs: list[str]) -> list[str]` that:
- Removes exact duplicate log lines.
- Preserves the original order of FIRST occurrence.
- Strips trailing whitespace from each line before comparing.
- Returns an empty list for empty input.
- Does not modify the input list.
""".strip(),
        known_difficulty="Order preservation, whitespace stripping before dedup",
        reference_test_code="""
# Reference tests for deduplicate_logs
assert deduplicate_logs([]) == []
assert deduplicate_logs(["a", "b", "c"]) == ["a", "b", "c"]
assert deduplicate_logs(["a", "a", "a"]) == ["a"]
assert deduplicate_logs(["a", "b", "a"]) == ["a", "b"]  # first occurrence preserved
assert deduplicate_logs(["a  ", "a"]) == ["a"]  # trailing whitespace stripped before compare
assert deduplicate_logs(["a", "b", "c", "b", "a"]) == ["a", "b", "c"]

# Order of first occurrence
result = deduplicate_logs(["z", "a", "z", "b", "a"])
assert result == ["z", "a", "b"]

# Input not modified
original = ["x", "x", "y"]
_ = deduplicate_logs(original)
assert original == ["x", "x", "y"]

# Single element
assert deduplicate_logs(["only"]) == ["only"]

# Whitespace-only lines treated as duplicates after strip
assert deduplicate_logs(["msg  ", "msg"]) == ["msg"]
""".strip(),
    ),

    BenchmarkTask(
        task_id="flatten_nested_001",
        category="data_transformation",
        description="""
Write a Python function `flatten(nested: Any) -> list` that recursively flattens
a nested structure of lists and tuples into a flat list of non-container values.

Requirements:
- Flattens arbitrarily deep nesting.
- Treats strings as scalar values (do NOT flatten characters from strings).
- Handles None as a scalar value (include it in output).
- Handles empty lists and tuples (contribute nothing to output).
- Works with mixed nesting of lists and tuples.
- Input type hint: Any (can be a scalar, list, or tuple at any level).
""".strip(),
        known_difficulty="Strings must not be iterated, None must be preserved",
        reference_test_code="""
# Reference tests for flatten
assert flatten([]) == []
assert flatten(()) == []
assert flatten([1, 2, 3]) == [1, 2, 3]
assert flatten([1, [2, 3], 4]) == [1, 2, 3, 4]
assert flatten([[1, [2]], [3, [4, [5]]]]) == [1, 2, 3, 4, 5]
assert flatten(["hello", "world"]) == ["hello", "world"]  # strings not flattened
assert flatten(["ab", ["cd"]]) == ["ab", "cd"]
assert flatten([1, None, 2]) == [1, None, 2]  # None preserved
assert flatten([None]) == [None]
assert flatten((1, (2, 3))) == [1, 2, 3]  # tuples flattened
assert flatten([1, (2, [3])]) == [1, 2, 3]  # mixed
assert flatten(42) == [42]  # scalar input
assert flatten("hello") == ["hello"]  # string scalar not flattened
assert flatten([[], [], []]) == []  # empty nested
""".strip(),
    ),

    BenchmarkTask(
        task_id="word_frequency_001",
        category="text_processing",
        description="""
Write a Python function `top_k_words(text: str, k: int) -> list[tuple[str, int]]`
that returns the k most frequent words in text, sorted by frequency descending.
Ties are broken alphabetically (ascending).

Requirements:
- Case-insensitive: 'The' and 'the' count as the same word.
- Strip punctuation from word boundaries (., ! ? ; : , etc.) but preserve apostrophes within words.
- Ignore empty strings after stripping.
- Return list of (word, count) tuples.
- If k > unique words, return all unique words.
- Empty text returns [].
""".strip(),
        known_difficulty="Punctuation stripping, tie-breaking by alphabetical order",
        reference_test_code="""
# Reference tests for top_k_words
assert top_k_words("", 5) == []
assert top_k_words("hello hello world", 1) == [("hello", 2)]
assert top_k_words("hello hello world", 2) == [("hello", 2), ("world", 1)]

# Case insensitive
result = top_k_words("The the THE", 1)
assert result == [("the", 3)]

# Punctuation stripped
result = top_k_words("hello, world! hello.", 2)
assert result[0] == ("hello", 2)

# Alphabetical tie-breaking
result = top_k_words("cat bat cat bat dog", 2)
assert result[0][1] == 2  # tied at 2
words_at_top = [r[0] for r in result if r[1] == 2]
assert words_at_top == sorted(words_at_top)  # alphabetical

# k > unique words → return all
result = top_k_words("a b c", 10)
assert len(result) == 3

# Apostrophes preserved within words
result = top_k_words("don't don't", 1)
assert result[0][0] == "don't"
assert result[0][1] == 2
""".strip(),
    ),

    BenchmarkTask(
        task_id="range_compression_001",
        category="interval_merging",
        description="""
Write a Python function `compress_ranges(numbers: list[int]) -> list[str]`
that compresses a list of integers into range strings.

Examples:
  [1,2,3,5,7,8,9] → ["1-3", "5", "7-9"]
  [1] → ["1"]
  [] → []
  [1,1,2] → ["1-2"]  (duplicates treated as single occurrences)

Requirements:
- Sort input before processing.
- Deduplicate before processing.
- Consecutive integers form a range ("start-end").
- Single integers are represented as their string value.
- Return list of strings in ascending order.
""".strip(),
        known_difficulty="Deduplication before ranging, single-element ranges",
        reference_test_code="""
# Reference tests for compress_ranges
assert compress_ranges([]) == []
assert compress_ranges([1]) == ["1"]
assert compress_ranges([1, 2, 3]) == ["1-3"]
assert compress_ranges([1, 2, 3, 5, 7, 8, 9]) == ["1-3", "5", "7-9"]
assert compress_ranges([1, 1, 2]) == ["1-2"]  # dedup before ranging
assert compress_ranges([5, 3, 1, 2, 4]) == ["1-5"]  # unsorted input
assert compress_ranges([1, 3, 5]) == ["1", "3", "5"]  # no consecutive
assert compress_ranges([1, 2, 4, 5, 7]) == ["1-2", "4-5", "7"]
assert compress_ranges([10]) == ["10"]
assert compress_ranges([-3, -2, -1, 0]) == ["-3-0"]  # negative numbers
assert compress_ranges([1, 1, 1, 1]) == ["1"]  # all same
""".strip(),
    ),

    BenchmarkTask(
        task_id="safe_divide_001",
        category="boundary_conditions",
        description="""
Write a Python function `safe_divide(numerator: float, denominator: float) -> float | None`
that performs division with full edge case handling.

Rules:
- Return None if denominator is 0 or -0.0.
- Return None if either argument is float('inf') or float('-inf').
- Return None if either argument is float('nan').
- Otherwise return numerator / denominator as float.
- Do not raise any exceptions.
""".strip(),
        known_difficulty="IEEE 754 edge cases: -0.0, inf, nan",
        reference_test_code="""
import math

# Reference tests for safe_divide
assert safe_divide(10.0, 2.0) == 5.0
assert safe_divide(0.0, 5.0) == 0.0
assert safe_divide(10.0, 0) is None
assert safe_divide(10.0, -0.0) is None
assert safe_divide(float('inf'), 2.0) is None
assert safe_divide(2.0, float('inf')) is None
assert safe_divide(float('-inf'), 2.0) is None
assert safe_divide(2.0, float('-inf')) is None
assert safe_divide(float('nan'), 2.0) is None
assert safe_divide(2.0, float('nan')) is None
assert safe_divide(float('nan'), float('nan')) is None
assert safe_divide(-10.0, 2.0) == -5.0
assert safe_divide(1.0, 3.0) is not None  # normal division works
result = safe_divide(1.0, 3.0)
assert result is not None and abs(result - 1/3) < 1e-9
""".strip(),
    ),

    BenchmarkTask(
        task_id="anagram_groups_001",
        category="text_processing",
        description="""
Write a Python function `group_anagrams(words: list[str]) -> list[list[str]]`
that groups words that are anagrams of each other.

Requirements:
- Case-insensitive comparison: 'Listen' and 'silent' are anagrams.
- Each group sorted alphabetically internally.
- Output list sorted by first element of each group alphabetically.
- Empty input returns [].
- Single-word input returns [[word]].
- Non-alpha characters count in anagram comparison ('eat!' ≠ 'eat').
""".strip(),
        known_difficulty="Case normalization, deterministic output ordering",
        reference_test_code="""
# Reference tests for group_anagrams
assert group_anagrams([]) == []
assert group_anagrams(["cat"]) == [["cat"]]

# Basic anagram grouping
result = group_anagrams(["eat", "tea", "tan", "ate", "nat", "bat"])
# Each group sorted alphabetically internally
for group in result:
    assert group == sorted(group, key=str.lower)
# Groups sorted by first element
firsts = [g[0].lower() for g in result]
assert firsts == sorted(firsts)
# Correct groupings
flat = {tuple(sorted(g, key=str.lower)) for g in result}
assert ("ate", "eat", "tea") in flat
assert ("nat", "tan") in flat
assert ("bat",) in flat

# Case-insensitive
result = group_anagrams(["Listen", "silent", "hello"])
group_words = {tuple(sorted(g, key=str.lower)) for g in result}
assert ("Listen", "silent") in group_words or ("listen", "silent") in group_words

# Non-alpha chars distinguish words
result = group_anagrams(["eat", "eat!"])
assert len(result) == 2  # 'eat!' != 'eat'

# Single word
assert group_anagrams(["only"]) == [["only"]]
""".strip(),
    ),
]


def get_task_by_id(task_id: str) -> BenchmarkTask | None:
    for task in BENCHMARK_TASKS:
        if task.task_id == task_id:
            return task
    return None


def get_tasks_by_category(category: str) -> list[BenchmarkTask]:
    return [t for t in BENCHMARK_TASKS if t.category == category]