""" Benchmark task definitions — Interview + Real-World Hybrid problems. Tasks are designed to test: - Edge case handling (empty inputs, None, single element) - Boundary conditions (off-by-one, integer overflow) - Data normalization (messy CSV, unicode, inconsistent types) - Algorithm correctness under adversarial inputs - Real-world data patterns (logs, intervals, phone numbers) Each task has a description written for the generator agent and a reference_test_code suite used to evaluate the final output independently of the adversarial tests generated by the QA agent. Reference tests are held-out ground truth — never given to the agent. """ from dataclasses import dataclass, field from typing import Any @dataclass class BenchmarkTask: task_id: str category: str description: str # Reference tests: list of (input_args, expected_output) tuples # Used for final evaluation only — not given to the agent reference_tests: list[dict[str, Any]] = field(default_factory=list) # Held-out executable Python assert statements for ground-truth evaluation. # Never shown to the agent. Run against the agent's final code. reference_test_code: str = "" # Optional: known tricky aspects for result analysis known_difficulty: str = "" BENCHMARK_TASKS: list[BenchmarkTask] = [ BenchmarkTask( task_id="interval_merge_001", category="interval_merging", description=""" Write a Python function `merge_intervals(intervals: list[list[int]]) -> list[list[int]]` that merges all overlapping intervals and returns a sorted list of non-overlapping intervals. Requirements: - Input: list of [start, end] pairs (integers). May be unsorted. - Output: merged, sorted list of [start, end] pairs. - Handle empty list: return []. - Handle single interval: return it unchanged. - Intervals are inclusive on both ends: [1,3] and [3,5] overlap. - Do not modify the input list. """.strip(), known_difficulty="Off-by-one on touching intervals [1,3],[3,5] → must merge to [1,5]", reference_test_code=""" # Reference tests for merge_intervals assert merge_intervals([]) == [] assert merge_intervals([[1, 3]]) == [[1, 3]] assert merge_intervals([[1, 3], [3, 5]]) == [[1, 5]] # touching intervals merge assert merge_intervals([[1, 4], [2, 3]]) == [[1, 4]] # contained assert merge_intervals([[1, 3], [2, 6], [8, 10]]) == [[1, 6], [8, 10]] assert merge_intervals([[6, 8], [1, 3], [2, 6]]) == [[1, 8]] # unsorted input assert merge_intervals([[1, 2], [3, 4]]) == [[1, 2], [3, 4]] # non-overlapping, gap of 1 assert merge_intervals([[1, 5], [2, 3], [4, 8]]) == [[1, 8]] # multiple merges assert merge_intervals([[0, 0], [0, 0]]) == [[0, 0]] # duplicate zero-length original = [[1, 3], [2, 6]] _ = merge_intervals(original) assert original == [[1, 3], [2, 6]] # input not modified """.strip(), ), BenchmarkTask( task_id="csv_normalize_001", category="data_normalization", description=""" Write a Python function `normalize_csv_row(row: dict) -> dict` that normalizes a single CSV row with messy data. Normalization rules: - Strip leading/trailing whitespace from all string values. - Convert 'phone' field to digits only (remove spaces, dashes, parentheses, dots). If phone is empty or None after stripping, set to empty string. - Convert 'age' field to int. If invalid or missing, set to -1. - Convert 'email' to lowercase. - Convert 'name' to title case. - All other string fields: strip only. Input dict may have missing keys — handle gracefully with defaults. Return a new dict (do not modify the input). """.strip(), known_difficulty="Missing keys, None values, partial phone strings", reference_test_code=""" # Reference tests for normalize_csv_row result = normalize_csv_row({"name": " john doe ", "email": "JOHN@EXAMPLE.COM", "age": "30", "phone": "(555) 123-4567"}) assert result["name"] == "John Doe" assert result["email"] == "john@example.com" assert result["age"] == 30 assert result["phone"] == "5551234567" # Missing keys → defaults result = normalize_csv_row({}) assert result.get("age", -1) == -1 assert result.get("phone", "") == "" # None phone result = normalize_csv_row({"phone": None}) assert result["phone"] == "" # Invalid age result = normalize_csv_row({"age": "not_a_number"}) assert result["age"] == -1 # Phone with dots and dashes result = normalize_csv_row({"phone": "555.123-4567"}) assert result["phone"] == "5551234567" # Empty phone string result = normalize_csv_row({"phone": " "}) assert result["phone"] == "" # Input not modified original = {"name": "jane", "age": "25"} _ = normalize_csv_row(original) assert original == {"name": "jane", "age": "25"} # Other string fields: strip only result = normalize_csv_row({"city": " Seattle "}) assert result["city"] == "Seattle" """.strip(), ), BenchmarkTask( task_id="log_dedup_001", category="log_processing", description=""" Write a Python function `deduplicate_logs(logs: list[str]) -> list[str]` that: - Removes exact duplicate log lines. - Preserves the original order of FIRST occurrence. - Strips trailing whitespace from each line before comparing. - Returns an empty list for empty input. - Does not modify the input list. """.strip(), known_difficulty="Order preservation, whitespace stripping before dedup", reference_test_code=""" # Reference tests for deduplicate_logs assert deduplicate_logs([]) == [] assert deduplicate_logs(["a", "b", "c"]) == ["a", "b", "c"] assert deduplicate_logs(["a", "a", "a"]) == ["a"] assert deduplicate_logs(["a", "b", "a"]) == ["a", "b"] # first occurrence preserved assert deduplicate_logs(["a ", "a"]) == ["a"] # trailing whitespace stripped before compare assert deduplicate_logs(["a", "b", "c", "b", "a"]) == ["a", "b", "c"] # Order of first occurrence result = deduplicate_logs(["z", "a", "z", "b", "a"]) assert result == ["z", "a", "b"] # Input not modified original = ["x", "x", "y"] _ = deduplicate_logs(original) assert original == ["x", "x", "y"] # Single element assert deduplicate_logs(["only"]) == ["only"] # Whitespace-only lines treated as duplicates after strip assert deduplicate_logs(["msg ", "msg"]) == ["msg"] """.strip(), ), BenchmarkTask( task_id="flatten_nested_001", category="data_transformation", description=""" Write a Python function `flatten(nested: Any) -> list` that recursively flattens a nested structure of lists and tuples into a flat list of non-container values. Requirements: - Flattens arbitrarily deep nesting. - Treats strings as scalar values (do NOT flatten characters from strings). - Handles None as a scalar value (include it in output). - Handles empty lists and tuples (contribute nothing to output). - Works with mixed nesting of lists and tuples. - Input type hint: Any (can be a scalar, list, or tuple at any level). """.strip(), known_difficulty="Strings must not be iterated, None must be preserved", reference_test_code=""" # Reference tests for flatten assert flatten([]) == [] assert flatten(()) == [] assert flatten([1, 2, 3]) == [1, 2, 3] assert flatten([1, [2, 3], 4]) == [1, 2, 3, 4] assert flatten([[1, [2]], [3, [4, [5]]]]) == [1, 2, 3, 4, 5] assert flatten(["hello", "world"]) == ["hello", "world"] # strings not flattened assert flatten(["ab", ["cd"]]) == ["ab", "cd"] assert flatten([1, None, 2]) == [1, None, 2] # None preserved assert flatten([None]) == [None] assert flatten((1, (2, 3))) == [1, 2, 3] # tuples flattened assert flatten([1, (2, [3])]) == [1, 2, 3] # mixed assert flatten(42) == [42] # scalar input assert flatten("hello") == ["hello"] # string scalar not flattened assert flatten([[], [], []]) == [] # empty nested """.strip(), ), BenchmarkTask( task_id="word_frequency_001", category="text_processing", description=""" Write a Python function `top_k_words(text: str, k: int) -> list[tuple[str, int]]` that returns the k most frequent words in text, sorted by frequency descending. Ties are broken alphabetically (ascending). Requirements: - Case-insensitive: 'The' and 'the' count as the same word. - Strip punctuation from word boundaries (., ! ? ; : , etc.) but preserve apostrophes within words. - Ignore empty strings after stripping. - Return list of (word, count) tuples. - If k > unique words, return all unique words. - Empty text returns []. """.strip(), known_difficulty="Punctuation stripping, tie-breaking by alphabetical order", reference_test_code=""" # Reference tests for top_k_words assert top_k_words("", 5) == [] assert top_k_words("hello hello world", 1) == [("hello", 2)] assert top_k_words("hello hello world", 2) == [("hello", 2), ("world", 1)] # Case insensitive result = top_k_words("The the THE", 1) assert result == [("the", 3)] # Punctuation stripped result = top_k_words("hello, world! hello.", 2) assert result[0] == ("hello", 2) # Alphabetical tie-breaking result = top_k_words("cat bat cat bat dog", 2) assert result[0][1] == 2 # tied at 2 words_at_top = [r[0] for r in result if r[1] == 2] assert words_at_top == sorted(words_at_top) # alphabetical # k > unique words → return all result = top_k_words("a b c", 10) assert len(result) == 3 # Apostrophes preserved within words result = top_k_words("don't don't", 1) assert result[0][0] == "don't" assert result[0][1] == 2 """.strip(), ), BenchmarkTask( task_id="range_compression_001", category="interval_merging", description=""" Write a Python function `compress_ranges(numbers: list[int]) -> list[str]` that compresses a list of integers into range strings. Examples: [1,2,3,5,7,8,9] → ["1-3", "5", "7-9"] [1] → ["1"] [] → [] [1,1,2] → ["1-2"] (duplicates treated as single occurrences) Requirements: - Sort input before processing. - Deduplicate before processing. - Consecutive integers form a range ("start-end"). - Single integers are represented as their string value. - Return list of strings in ascending order. """.strip(), known_difficulty="Deduplication before ranging, single-element ranges", reference_test_code=""" # Reference tests for compress_ranges assert compress_ranges([]) == [] assert compress_ranges([1]) == ["1"] assert compress_ranges([1, 2, 3]) == ["1-3"] assert compress_ranges([1, 2, 3, 5, 7, 8, 9]) == ["1-3", "5", "7-9"] assert compress_ranges([1, 1, 2]) == ["1-2"] # dedup before ranging assert compress_ranges([5, 3, 1, 2, 4]) == ["1-5"] # unsorted input assert compress_ranges([1, 3, 5]) == ["1", "3", "5"] # no consecutive assert compress_ranges([1, 2, 4, 5, 7]) == ["1-2", "4-5", "7"] assert compress_ranges([10]) == ["10"] assert compress_ranges([-3, -2, -1, 0]) == ["-3-0"] # negative numbers assert compress_ranges([1, 1, 1, 1]) == ["1"] # all same """.strip(), ), BenchmarkTask( task_id="safe_divide_001", category="boundary_conditions", description=""" Write a Python function `safe_divide(numerator: float, denominator: float) -> float | None` that performs division with full edge case handling. Rules: - Return None if denominator is 0 or -0.0. - Return None if either argument is float('inf') or float('-inf'). - Return None if either argument is float('nan'). - Otherwise return numerator / denominator as float. - Do not raise any exceptions. """.strip(), known_difficulty="IEEE 754 edge cases: -0.0, inf, nan", reference_test_code=""" import math # Reference tests for safe_divide assert safe_divide(10.0, 2.0) == 5.0 assert safe_divide(0.0, 5.0) == 0.0 assert safe_divide(10.0, 0) is None assert safe_divide(10.0, -0.0) is None assert safe_divide(float('inf'), 2.0) is None assert safe_divide(2.0, float('inf')) is None assert safe_divide(float('-inf'), 2.0) is None assert safe_divide(2.0, float('-inf')) is None assert safe_divide(float('nan'), 2.0) is None assert safe_divide(2.0, float('nan')) is None assert safe_divide(float('nan'), float('nan')) is None assert safe_divide(-10.0, 2.0) == -5.0 assert safe_divide(1.0, 3.0) is not None # normal division works result = safe_divide(1.0, 3.0) assert result is not None and abs(result - 1/3) < 1e-9 """.strip(), ), BenchmarkTask( task_id="anagram_groups_001", category="text_processing", description=""" Write a Python function `group_anagrams(words: list[str]) -> list[list[str]]` that groups words that are anagrams of each other. Requirements: - Case-insensitive comparison: 'Listen' and 'silent' are anagrams. - Each group sorted alphabetically internally. - Output list sorted by first element of each group alphabetically. - Empty input returns []. - Single-word input returns [[word]]. - Non-alpha characters count in anagram comparison ('eat!' ≠ 'eat'). """.strip(), known_difficulty="Case normalization, deterministic output ordering", reference_test_code=""" # Reference tests for group_anagrams assert group_anagrams([]) == [] assert group_anagrams(["cat"]) == [["cat"]] # Basic anagram grouping result = group_anagrams(["eat", "tea", "tan", "ate", "nat", "bat"]) # Each group sorted alphabetically internally for group in result: assert group == sorted(group, key=str.lower) # Groups sorted by first element firsts = [g[0].lower() for g in result] assert firsts == sorted(firsts) # Correct groupings flat = {tuple(sorted(g, key=str.lower)) for g in result} assert ("ate", "eat", "tea") in flat assert ("nat", "tan") in flat assert ("bat",) in flat # Case-insensitive result = group_anagrams(["Listen", "silent", "hello"]) group_words = {tuple(sorted(g, key=str.lower)) for g in result} assert ("Listen", "silent") in group_words or ("listen", "silent") in group_words # Non-alpha chars distinguish words result = group_anagrams(["eat", "eat!"]) assert len(result) == 2 # 'eat!' != 'eat' # Single word assert group_anagrams(["only"]) == [["only"]] """.strip(), ), ] def get_task_by_id(task_id: str) -> BenchmarkTask | None: for task in BENCHMARK_TASKS: if task.task_id == task_id: return task return None def get_tasks_by_category(category: str) -> list[BenchmarkTask]: return [t for t in BENCHMARK_TASKS if t.category == category]