vehicle_routing_time_windows.json

{
  "task_id": "vehicle_routing_time_windows",
  "name": "Vehicle Routing Time Windows",
  "category": "Combinatorial Optimization",
  "base_image": "python",
  "platform": "linux/amd64",
  "internet": false,
  "cwd": "/home/workspace/task_cvrptw_905da6dd/workspace",
  "submit_paths": [
    "src/",
    "include/",
    "Makefile"
  ],
  "submit_exclude": [
    "instances/",
    "internal/",
    "__MACOSX/",
    ".DS_Store"
  ],
  "work": {
    "image_tag": "53214fda231b",
    "specs_dir": "/home/workspace/task_cvrptw_905da6dd/workspace",
    "agent_query": "Read README.md, then optimize the C++17 CVRPTW solver. Modify only src/, include/, and Makefile. Preserve the cvrptw_solver binary name and CLI: ./cvrptw_solver <input.txt> <output.txt> <time_budget_sec>. Do not read hidden instances, internal judge files, BKS tables, reference solutions, or evaluator internals. Hidden judge submissions have a 20-minute total wall-clock budget; keep the solver efficient."
  },
  "judge": {
    "image_tag": "d5abb8bb08d2",
    "eval_cmd": "cd /home/workspace/task_cvrptw_905da6dd/workspace && {\npython - <<'PY'\nimport csv, json, os, re, subprocess, time, traceback\nfrom pathlib import Path\n\nROOT = Path('/home/workspace/task_cvrptw_905da6dd')\nTOTAL_JUDGE_TIMEOUT_SEC = 20 * 60\nCOMPILE_TIMEOUT_SEC = 300\nPER_INSTANCE_BUDGET_SEC = 60\nVERIFIER_TIMEOUT_SEC = 30\nDEADLINE_GRACE_SEC = 5\n\nstarted = time.monotonic()\ndeadline = started + TOTAL_JUDGE_TIMEOUT_SEC\ncompile_output = ''\ndetails = []\nmetrics = {\n    'judge_time_limit_sec': TOTAL_JUDGE_TIMEOUT_SEC,\n    'per_instance_budget_sec': PER_INSTANCE_BUDGET_SEC,\n    'compile_timeout_sec': COMPILE_TIMEOUT_SEC,\n    'score': 0.0,\n    'compile_returncode': None,\n    'cases': 0,\n    'timeout_count': 0,\n    'no_pass_count': 0,\n    'judge_time_exceeded': False,\n    'compile_timed_out': False,\n}\n\n\ndef remaining() -> float:\n    return deadline - time.monotonic()\n\n\ndef safe_text(x) -> str:\n    if x is None:\n        return ''\n    if isinstance(x, bytes):\n        return x.decode(errors='replace')\n    return str(x)\n\n\ndef add_detail(name: str, status: str, score: float = 0.0, message: str = '', **extra) -> None:\n    item = {'name': name, 'status': status, 'score': float(score), 'message': (message or '')[-1000:]}\n    item.update(extra)\n    details.append(item)\n\n\ndef compute(nv, td, bnv, btd):\n    gap_nv = max(0, nv - bnv)\n    gap_td = max(0.0, (td - btd) / btd * 100.0) if btd > 0 else 0.0\n    s = max(0.0, 100.0 - 30.0 * gap_nv - 0.5 * gap_td)\n    if nv < bnv:\n        s += 30.0 * (bnv - nv)\n    if nv <= bnv and td < btd:\n        s += min(30.0, (btd - td) / btd * 100.0)\n    return min(s, 160.0)\n\n\ndef load_bks():\n    bks = {}\n    for bp in [ROOT / 'internal/cases/bks_snapshot.csv']:\n        if bp.exists():\n            with open(bp, newline='') as f:\n                for row in csv.DictReader(f):\n                    bks[row['name']] = (int(row['nv']), float(row['td']))\n    return bks\n\n\ndef case_files():\n    case_dir = ROOT / 'internal/cases/raw'\n    return sorted(case_dir.glob('*.txt'))[:56] if case_dir.exists() else []\n\n\ndef emit(valid: bool, score: float, summary: str):\n    passed = sum(1 for d in details if d.get('status') == 'PASSED')\n    failed = sum(1 for d in details if d.get('status') == 'FAILED')\n    errors = sum(1 for d in details if d.get('status') == 'ERROR')\n    total = len(details) or 1\n    metrics['score'] = float(score)\n    metrics['cases'] = metrics.get('cases') or len([d for d in details if d.get('name') != 'compile'])\n    metrics['timeout_count'] = sum(1 for d in details if d.get('reason') in ('solver_timeout', 'judge_time_limit', 'verifier_timeout'))\n    metrics['no_pass_count'] = failed\n    metrics['runtime_seconds'] = time.monotonic() - started\n    if metrics['timeout_count']:\n        metrics['judge_time_exceeded'] = any(d.get('reason') == 'judge_time_limit' for d in details)\n    report = {\n        'valid': valid,\n        'score': float(score),\n        'pass_rate': passed / total if total else 0.0,\n        'total_tests': total,\n        'passed': passed,\n        'failed': failed,\n        'errors': errors,\n        'summary': summary,\n        'details': details[:100],\n        'metrics': metrics,\n    }\n    print('>>>>> Start Structured Result')\n    print(json.dumps(report, ensure_ascii=False))\n    print('>>>>> End Structured Result')\n\n\ntry:\n    bks = load_bks()\n    cases = case_files()\n    metrics['cases'] = len(cases)\n\n    compile_timeout = max(1, min(COMPILE_TIMEOUT_SEC, int(remaining() - DEADLINE_GRACE_SEC)))\n    if compile_timeout <= 0:\n        for inp in cases or [Path('compile')]:\n            add_detail(inp.stem, 'ERROR', 0.0, 'judge global time limit reached before compilation', reason='judge_time_limit')\n        emit(False, 0.0, f'Judge timed out before compilation: 0 passed, {len(details)} timed out/no result')\n    else:\n        try:\n            proc = subprocess.run(\n                ['bash', '-lc', 'rm -f cvrptw_solver tools/verify && make -j4'],\n                text=True,\n                capture_output=True,\n                timeout=compile_timeout,\n            )\n            compile_output = (proc.stdout or '') + (proc.stderr or '')\n            metrics['compile_returncode'] = proc.returncode\n            compiled = proc.returncode == 0 and Path('cvrptw_solver').exists()\n        except subprocess.TimeoutExpired as ex:\n            compile_output = safe_text(ex.stdout) + safe_text(ex.stderr) + '\\nCOMPILE_TIMEOUT'\n            metrics['compile_returncode'] = -9\n            metrics['compile_timed_out'] = True\n            compiled = False\n\n        print(compile_output, end='')\n\n        if not compiled:\n            status = 'ERROR' if metrics['compile_timed_out'] else 'FAILED'\n            reason = 'compile_timeout' if metrics['compile_timed_out'] else 'compile_failed'\n            msg = compile_output[-1000:] or reason\n            if cases:\n                for inp in cases:\n                    add_detail(inp.stem, status, 0.0, msg, reason=reason)\n            else:\n                add_detail('compile', status, 0.0, msg, reason=reason)\n            emit(False, 0.0, f'Compilation did not pass ({reason}); score=0. Timeouts: {metrics[\"compile_timed_out\"]}')\n        else:\n            bks = bks or {}\n            score_sum = 0.0\n            for idx, inp in enumerate(cases):\n                name = inp.stem\n                rem = remaining()\n                if rem <= DEADLINE_GRACE_SEC + 1:\n                    add_detail(name, 'ERROR', 0.0, 'judge global time limit reached before this instance could finish', reason='judge_time_limit')\n                    continue\n\n                budget = min(PER_INSTANCE_BUDGET_SEC, max(1, int(rem - DEADLINE_GRACE_SEC - 1)))\n                out = Path(f'out_{name}.sol')\n                try:\n                    out.unlink(missing_ok=True)\n                except OSError:\n                    pass\n\n                try:\n                    run = subprocess.run(\n                        ['./cvrptw_solver', str(inp), str(out), str(budget)],\n                        text=True,\n                        capture_output=True,\n                        timeout=budget + DEADLINE_GRACE_SEC,\n                    )\n                    run_txt = (run.stdout or '') + (run.stderr or '')\n                    solver_timed_out = False\n                except subprocess.TimeoutExpired as ex:\n                    run = None\n                    run_txt = safe_text(ex.stdout) + safe_text(ex.stderr) + '\\nSOLVER_TIMEOUT'\n                    solver_timed_out = True\n\n                if solver_timed_out:\n                    add_detail(name, 'ERROR', 0.0, run_txt, reason='solver_timeout', budget_sec=budget)\n                    continue\n                if run is None or run.returncode != 0:\n                    rc = getattr(run, 'returncode', 'unknown')\n                    add_detail(name, 'FAILED', 0.0, f'solver exited without a passing solution: exit_code={rc}\\n{run_txt}', reason='no_pass', budget_sec=budget)\n                    continue\n                if not out.exists():\n                    add_detail(name, 'FAILED', 0.0, 'solver produced no output file', reason='no_pass', budget_sec=budget)\n                    continue\n\n                v_rem = remaining()\n                if v_rem <= 2:\n                    add_detail(name, 'ERROR', 0.0, 'judge global time limit reached before verification', reason='judge_time_limit', budget_sec=budget)\n                    continue\n                v_timeout = max(1, min(VERIFIER_TIMEOUT_SEC, int(v_rem - 1)))\n                try:\n                    ver = subprocess.run(\n                        ['./tools/verify', str(inp), str(out)],\n                        text=True,\n                        capture_output=True,\n                        timeout=v_timeout,\n                    )\n                    ver_txt = (ver.stdout or '') + (ver.stderr or '')\n                    verifier_timed_out = False\n                except subprocess.TimeoutExpired as ex:\n                    ver = None\n                    ver_txt = safe_text(ex.stdout) + safe_text(ex.stderr) + '\\nVERIFIER_TIMEOUT'\n                    verifier_timed_out = True\n\n                if verifier_timed_out:\n                    add_detail(name, 'ERROR', 0.0, ver_txt, reason='verifier_timeout', budget_sec=budget)\n                    continue\n\n                feasible = ver is not None and ver.returncode == 0 and 'feasible: true' in (ver.stdout or '')\n                nv_m = re.search(r'NV:\\s*(\\d+)', ver.stdout if ver else '')\n                td_m = re.search(r'TD:\\s*([0-9.]+)', ver.stdout if ver else '')\n                nv = int(nv_m.group(1)) if nv_m else 10**9\n                td = float(td_m.group(1)) if td_m else 1e18\n                bnv, btd = bks.get(name, (nv, td))\n                si = compute(nv, td, bnv, btd) if feasible else 0.0\n                score_sum += si\n                if feasible:\n                    msg = f'feasible; NV={nv}; TD={td}; BKS_NV={bnv}; BKS_TD={btd}; budget_sec={budget}'\n                    add_detail(name, 'PASSED', si, msg, reason='passed', budget_sec=budget, NV=nv, TD=td, BKS_NV=bnv, BKS_TD=btd)\n                else:\n                    msg = (run_txt + '\\n' + ver_txt)[-1000:] or 'solution did not pass feasibility/cost verification'\n                    add_detail(name, 'FAILED', 0.0, msg, reason='no_pass', budget_sec=budget, NV=nv, TD=td, BKS_NV=bnv, BKS_TD=btd)\n\n            total_cases = len(cases) or 1\n            final_score = score_sum / total_cases * 100.0\n            pass_cnt = sum(1 for d in details if d.get('status') == 'PASSED')\n            fail_cnt = sum(1 for d in details if d.get('status') == 'FAILED')\n            timeout_cnt = sum(1 for d in details if d.get('reason') in ('solver_timeout', 'judge_time_limit', 'verifier_timeout'))\n            summary = (\n                f'Score: {final_score:.6f} over {len(cases)} cases; '\n                f'passed={pass_cnt}, no_pass={fail_cnt}, timeout={timeout_cnt}, '\n                f'runtime={time.monotonic() - started:.2f}s/{TOTAL_JUDGE_TIMEOUT_SEC}s'\n            )\n            emit(True, final_score, summary)\nexcept Exception:\n    err = traceback.format_exc()\n    print(err)\n    if not details:\n        add_detail('judge_exception', 'ERROR', 0.0, err, reason='judge_exception')\n    emit(False, 0.0, 'Judge exception before normal completion; see details')\nPY\n}",
    "eval_timeout": 1200,
    "parser": "structured_json",
    "score_direction": "maximize",
    "selection": "score_first",
    "rescale": {
      "kind": "linear",
      "lower": 0.0,
      "upper": 10000.0
    }
  }
}