scripts/build_confirmatory_tasks.py

"""Deterministically materialize held-out GitLab Runner task manifests and patches."""

from __future__ import annotations

import argparse
import json
from pathlib import Path
import re
import subprocess
from typing import Iterable

from tree_sitter import Language, Node, Parser
import tree_sitter_go


GO_LANGUAGE = Language(tree_sitter_go.language())


TASKS = (
    (
        "d870071152bd6ade567ac346624bc52acb19945c",
        "AWS Secrets Manager supports server-wide defaults and per-secret overrides. Per-secret RoleARN and RoleSessionName values are currently ignored when server defaults exist. Make each per-secret setting take precedence while retaining the server value as its fallback.",
    ),
    (
        "b6dedc95bd2b91fea1ac80e15d93078e45beeb6e",
        "When a runner cache type is misspelled or unsupported, the error does not tell users which adapters are available. Include the deterministically sorted registered adapter names in unknown-type errors for both cache and credential adapters.",
    ),
    (
        "91dcb6c16becdaf163fe7ab3974596d582d87244",
        "Closing a build-log timestamper more than once can emit its buffered partial line repeatedly, and a later write can pick up stale buffered data. Make Logger.Close idempotent without losing the first flush.",
    ),
    (
        "eeba44f142de0df780f639593fc567f01060ce6f",
        "Concrete execution classifies cache.policy before expanding CI variables, so values such as $MY_POLICY fall into the default branch and silently skip cache work. Expand the policy before classifying both cache extraction and archive behavior.",
    ),
    (
        "f07534b7a2031fe140be164b5476cc7dc428023f",
        "Concrete artifact upload forwards expire_in literally even when it contains a CI variable, unlike other artifact fields. Resolve the value using the job environment before passing --expire-in to the uploader.",
    ),
    (
        "e05f89e3ec2edf079e52d0085aeb4fe5911b143a",
        "In concrete execution, a user-cancelled script returns ErrJobCanceled but loses context.Canceled from its unwrap chain. Preserve the cancellation cause so step-runner classifies the result as cancelled instead of an unknown failure.",
    ),
    (
        "b4b74fa3d7c867c53efdb254893564c5119526a5",
        "The concrete runner emits trace section markers even when the GitLab server does not advertise trace_sections, exposing raw marker text on older servers. Propagate the feature bit and emit markers only when supported.",
    ),
    (
        "002579e259d9a473ae5c92db38899a5687fcf03b",
        "Concrete artifact upload still emits and invokes the upload section when the runner BaseURL is empty, while the abstract shell skips that stage. Skip only the artifact upload section in this case without affecting cache archive behavior.",
    ),
    (
        "c9632925cb0c5acd81502a7ab9bd381f72189e05",
        "The S3v2 cache presigner rejects HTTP HEAD, causing cache flows that check object existence to fail with an unsupported-method error. Add HEAD presigning while preserving errors for genuinely unsupported methods.",
    ),
    (
        "3b1d17363021ef29c5aee6363d709887b6086cce",
        "Kubernetes expands $(NAME) references inside environment-variable values before a container starts, which can corrupt literal dollar-containing CI values and secrets. Escape every dollar sign when constructing Kubernetes EnvVar values while leaving other text unchanged.",
    ),
    (
        "610441433600e1392f9cef3ddfa174c8d1876d86",
        "The artifacts-uploader command starts with zero-valued upload and response-header timeouts unless flags are explicitly supplied. Initialize both options from the runner's documented artifact timeout defaults.",
    ),
    (
        "86c126f852578e9dc91959619981f763e669c12c",
        "Creating a gzip artifact with a non-Latin-1 filename can fail because the filename is assigned directly to the gzip header. Encode unsafe filenames with the same reversible path-sanitization scheme already used for the header comment.",
    ),
    (
        "1a6fb708e5719cce74c7b0604c707780e5b1fcd5",
        "Hash-based cache keys all share one project-level object-store prefix, which can create a hot S3 partition under parallel load. When hashed keys are enabled, add a shard derived from the first two hash characters consistently across cache path construction and callers.",
    ),
    (
        "213001f5e9e25c25cff80c131f312649b70f5825",
        "Concrete mode forces after_script to allow failure before AFTER_SCRIPT_IGNORE_ERRORS is consulted, making the false setting ineffective. Let the runner-level policy handle after_script errors while preserving the default ignore-errors behavior.",
    ),
    (
        "32a34faf2b816d51e22a3f4dd75e20c17c23619c",
        "A panic inside a step-runner gRPC handler surfaces as codes.Internal without a job status, so it escapes the existing ErrorKind classification. Keep gRPC knowledge in the steps client and map this internal client error to ScriptFailure.",
    ),
)

END_TO_END_TASKS = {
    "TASK_CR_001", "TASK_CR_002", "TASK_CR_003", "TASK_CR_005", "TASK_CR_006",
    "TASK_CR_007", "TASK_CR_008", "TASK_CR_009", "TASK_CR_012", "TASK_CR_013",
}


def git(repository: Path, *arguments: str) -> str:
    result = subprocess.run(
        ["git", *arguments],
        cwd=repository,
        check=False,
        capture_output=True,
        text=True,
        timeout=120,
    )
    if result.returncode:
        raise RuntimeError(result.stderr.strip() or f"git {' '.join(arguments)} failed")
    return result.stdout


def changed_paths(repository: Path, parent: str, commit: str) -> tuple[list[str], list[str]]:
    paths = [
        item
        for item in git(repository, "diff", "--name-only", parent, commit, "--", "*.go").splitlines()
        if item
    ]
    source = [item for item in paths if not item.endswith("_test.go")]
    tests = [item for item in paths if item.endswith("_test.go")]
    if not source or not tests:
        raise RuntimeError(f"{commit} must change source and tests")
    return source, tests


HUNK = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@")


def changed_new_lines(repository: Path, parent: str, commit: str, path: str) -> list[tuple[int, int]]:
    ranges: list[tuple[int, int]] = []
    for line in git(repository, "diff", "--unified=0", parent, commit, "--", path).splitlines():
        match = HUNK.match(line)
        if match:
            start = int(match.group(1))
            count = int(match.group(2) or "1")
            ranges.append((start, start + max(count, 1) - 1))
    return ranges


def walk(node: Node) -> Iterable[Node]:
    yield node
    for child in node.children:
        yield from walk(child)


def declaration_name(node: Node, source: bytes) -> str | None:
    direct = node.child_by_field_name("name")
    if direct is not None:
        return source[direct.start_byte : direct.end_byte].decode("utf-8", errors="replace")
    for descendant in walk(node):
        named = descendant.child_by_field_name("name")
        if named is not None:
            return source[named.start_byte : named.end_byte].decode("utf-8", errors="replace")
    return None


def changed_symbols(
    repository: Path,
    commit: str,
    paths: list[str],
    ranges_by_path: dict[str, list[tuple[int, int]]],
) -> list[str]:
    # Retain the Language object for the parser lifetime. Constructing it as a
    # temporary can release the underlying capsule while traversal is active.
    parser = Parser(GO_LANGUAGE)
    declarations = {
        "function_declaration",
        "method_declaration",
        "type_declaration",
        "const_declaration",
        "var_declaration",
    }
    symbols: list[str] = []
    for path in paths:
        source = git(repository, "show", f"{commit}:{path}").encode("utf-8")
        tree = parser.parse(source)
        matches: list[str] = []
        for node in walk(tree.root_node):
            if node.type not in declarations:
                continue
            # Tuple indexing avoids a tree-sitter 0.26 macOS binding bug in
            # the Point.row descriptor that can segfault during cleanup.
            start, end = node.start_point[0] + 1, node.end_point[0] + 1
            if not any(start <= changed_end and changed_start <= end for changed_start, changed_end in ranges_by_path[path]):
                continue
            name = declaration_name(node, source)
            if name:
                matches.append(f"{path}::{name}")
        symbols.extend(dict.fromkeys(matches or [f"{path}::<file_scope>"]))
    return symbols


def toml_string(value: str) -> str:
    return json.dumps(value, ensure_ascii=False)


def toml_array(values: Iterable[str]) -> str:
    return "[" + ", ".join(toml_string(value) for value in values) + "]"


def build(root: Path, repository: Path, write: bool) -> None:
    manifest_dir = root / "tasks" / "manifests"
    patch_dir = root / "tasks" / "patches"
    if write:
        patch_dir.mkdir(parents=True, exist_ok=True)
    task_ids: list[str] = []
    for number, (commit, statement) in enumerate(TASKS, start=1):
        task_id = f"TASK_CR_{number:03d}"
        task_ids.append(task_id)
        parent = git(repository, "rev-parse", f"{commit}^1").strip()
        source_paths, test_paths = changed_paths(repository, parent, commit)
        ranges = {
            path: changed_new_lines(repository, parent, commit, path) for path in source_paths
        }
        symbols = changed_symbols(repository, commit, source_paths, ranges)
        packages = sorted({str(Path(path).parent) for path in test_paths})
        commands = [f"go test ./{package} -count=1" for package in packages]
        source_patch_name = f"{task_id}_source.patch"
        test_patch_name = f"{task_id}_tests.patch"
        source_patch = git(repository, "diff", "--binary", parent, commit, "--", *source_paths)
        test_patch = git(repository, "diff", "--binary", parent, commit, "--", *test_paths)
        manifest = "\n".join(
            (
                "schema_version = 1",
                f"task_id = {toml_string(task_id)}",
                'repository_url = "https://gitlab.com/gitlab-org/gitlab-runner.git"',
                f"base_commit = {toml_string(parent)}",
                f"gold_commit = {toml_string(commit)}",
                'language = "go"',
                f"statement = {toml_string(statement)}",
                f"gold_patch = {toml_string('patches/' + source_patch_name)}",
                f"test_patch = {toml_string('patches/' + test_patch_name)}",
                f"gold_files = {toml_array(source_paths)}",
                f"gold_symbols = {toml_array(symbols)}",
                f"fail_to_pass_tests = {toml_array(commands)}",
                f"pass_to_pass_tests = {toml_array(commands)}",
                f"difficulty = {toml_string('single_file' if len(source_paths) == 1 else 'multi_file')}",
                f"provenance = {toml_string('GitLab Runner commit ' + commit + '; held-out test-backed fix frozen before confirmatory evaluation.')}",
                f"validation_status = {toml_string('end_to_end_ready' if task_id in END_TO_END_TASKS else 'retrieval_ready')}",
                "",
            )
        )
        if write:
            (manifest_dir / f"{task_id}.toml").write_text(manifest, encoding="utf-8")
            (patch_dir / source_patch_name).write_text(source_patch, encoding="utf-8")
            (patch_dir / test_patch_name).write_text(test_patch, encoding="utf-8")
        print(f"{task_id} {commit[:12]} source={len(source_paths)} tests={len(test_paths)} symbols={len(symbols)}")

    if write:
        frozen = "# Frozen before confirmatory execution on 2026-07-18.\n" + "\n".join(task_ids) + "\n"
        for split in ("retrieval_confirmatory", "localization_confirmatory", "robustness"):
            (root / "tasks" / "splits" / f"{split}.txt").write_text(frozen, encoding="utf-8")
        end_to_end = (
            "# Frozen after fail-to-pass/pass validation on 2026-07-18.\n"
            + "\n".join(item for item in task_ids if item in END_TO_END_TASKS)
            + "\n"
        )
        (root / "tasks" / "splits" / "end_to_end_confirmatory.txt").write_text(
            end_to_end, encoding="utf-8"
        )


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1])
    parser.add_argument("--repository", type=Path, default=Path("data/repos/gitlab-runner"))
    parser.add_argument("--write", action="store_true")
    args = parser.parse_args()
    build(args.root.resolve(), args.repository.resolve(), args.write)


if __name__ == "__main__":
    main()