smt_solver.json

{
  "task_id": "smt_solver",
  "name": "Smt Solver",
  "category": "Combinatorial Optimization",
  "base_image": "python310",
  "platform": "linux/amd64",
  "internet": false,
  "cwd": "/home/workspace/smt_solver",
  "submit_paths": [
    "build.sh",
    "solve",
    "src/"
  ],
  "submit_exclude": [
    "__pycache__/",
    "*.pyc",
    ".pytest_cache/",
    "benchmarks/",
    "hidden_benchmarks/",
    "expected_answers.json",
    "selection_manifest.json",
    "public_score.json",
    "hidden_score.json",
    "*.log"
  ],
  "work": {
    "image_tag": "ce487a98870f",
    "specs_dir": "/home/workspace/smt_solver",
    "agent_query": "Implement an SMT solver from scratch. Read README.md and smtlib_spec.md. Provide build.sh, solve, and src/. The evaluator calls ./solve path/to/input.smt2 and expects unsat or unknown as exactly one line, and sat followed by a JSON model witness that validates the assertions. The benchmark logics include QF_UF, QF_LRA, QF_LIA, and QF_UFLRA. Use the compact benchmarks/ develop set and scripts/evaluate.py for local feedback. Official hidden evaluation uses a 60 second per-case timeout, 16 evaluator workers, perturbation with a private salt, and a 7200 second whole-run wall-clock budget. Do not call external SMT solvers such as z3/cvc5, online services, or hard-code benchmark answers."
  },
  "judge": {
    "image_tag": "fdce54253c61",
    "eval_cmd": "python3 - <<'PY'\nimport pathlib, shutil, zipfile\nzip_path = pathlib.Path('/home/workspace/smt_judge/SMT-hidden_benchmarks.zip')\ndest = pathlib.Path('/home/workspace/smt_judge/hidden_benchmarks')\nif dest.exists():\n    shutil.rmtree(dest)\ndest.mkdir(parents=True, exist_ok=True)\nwith zipfile.ZipFile(zip_path) as z:\n    z.extractall(dest)\nPY\ncd /home/workspace/smt_solver && rm -rf benchmarks hidden_benchmarks scripts expected_answers.json selection_manifest.json hidden_score.json public_score.json; chmod +x build.sh solve; build_status=0; ./build.sh >/tmp/smt_build.out 2>/tmp/smt_build.err || build_status=$?; if [ \"$build_status\" -eq 0 ]; then python3 /home/workspace/smt_judge/scripts/evaluate.py --solver ./solve --benchmarks /home/workspace/smt_judge/hidden_benchmarks --expected /home/workspace/smt_judge/hidden_benchmarks/expected_answers.json --manifest /home/workspace/smt_judge/hidden_benchmarks/selection_manifest.json --timeout 60 --parallel 16 --perturb --perturb-salt hidden-sebench --submission-root /home/workspace/smt_solver --output /tmp/smt_hidden_score.json >/tmp/smt_eval.out 2>/tmp/smt_eval.err; eval_status=$?; else eval_status=$build_status; fi; if [ -f /tmp/smt_build.out ]; then cat /tmp/smt_build.out; fi; if [ -f /tmp/smt_build.err ]; then cat /tmp/smt_build.err; fi; if [ -f /tmp/smt_eval.out ]; then cat /tmp/smt_eval.out; fi; if [ -f /tmp/smt_eval.err ]; then cat /tmp/smt_eval.err; fi; cat > /tmp/sebench_parse_result.py <<'PY'\nimport json, pathlib\np = pathlib.Path('/tmp/smt_hidden_score.json')\nraw = json.loads(p.read_text()) if p.exists() else {'final_score': 0.0, 'logic_summary': {}}\nraw_score = float(raw.get('final_score', 0.0) or 0.0)\nscore = raw_score / 100.0 if raw_score > 1.0 else raw_score\nby = raw.get('logic_summary', {}) or {}\ndef clean(g):\n    out = {}\n    for k, item in sorted(g.items()):\n        t = int(item.get('total', 0) or 0); c = int(item.get('correct', 0) or 0)\n        out[k] = {'total': t, 'correct': c, 'wrong': int(item.get('wrong', max(0, t-c)) or 0), 'timeout': int(item.get('timeout', 0) or 0), 'unknown': int(item.get('unknown', 0) or 0), 'error': int(item.get('error', 0) or 0), 'no_reference': int(item.get('no_reference', 0) or 0), 'invalid_model': int(item.get('invalid_model', 0) or 0), 'score': float(item.get('score', 0.0) or 0.0)}\n    return out\nby = clean(by)\ntotal = sum(v['total'] for v in by.values()); correct = sum(v['correct'] for v in by.values())\nwrong = max(0, total - correct)\nresult = {'valid': True, 'score': score, 'pass_rate': correct/total if total else 0.0, 'total_tests': total, 'passed': correct, 'failed': wrong, 'errors': 0, 'summary': 'score={:.6f} ({:.2f}/100), correct={}/{}'.format(score, 100*score, correct, total), 'details': [], 'metrics': {'score': score, 'score_percent': 100*score, 'total': total, 'correct': correct, 'wrong_or_failed': wrong, 'by_logic': by}}\nprint('>>>>> Start Structured Result')\nprint(json.dumps(result, ensure_ascii=False))\nprint('>>>>> End Structured Result')\nPY\npython3 /tmp/sebench_parse_result.py",
    "eval_timeout": 7200,
    "parser": "structured_json",
    "score_direction": "maximize",
    "selection": "score_first",
    "rescale": {
      "kind": "linear",
      "lower": 0.0,
      "upper": 1.0
    }
  }
}