File size: 8,508 Bytes
4e1037f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 | """Claude Code CLI provider — subscription-metered frontier slot.
Drives the locally-installed `claude` CLI in print mode (`-p`) and reads its JSON
envelope. Like GraceKelly and grok_cli this rides a subscription rather than an
API key: no bill, but the run draws on the user's plan limits. The envelope reports
`total_cost_usd`, so the draw is measurable rather than guessed — a BIRD generation
prompt (~6.7k fresh tokens, ~155 out) costs about $0.03 of plan-equivalent, so an
n=200 run is roughly $6.
Why this exists: Sonnet's only previous number on this benchmark (58.5%, closed as
"worse than codestral") was measured through the browser bridge **on the compact
prompt**, not the product prompt. That is a confound, not a result. This slot lets a
frontier model run the same config E as every other generator, so the comparison is
finally like-for-like.
Two details that matter:
- **`--system-prompt` replaces the harness prompt** (`--append-system-prompt` would
add to it). Replacing it strips the agent persona, so the model answers as a
completion rather than as an agent. grok_cli cannot do this — there the scaffolding
is unavoidable.
- **One turn is still not enough.** Even toolless and de-personified, the model
sometimes reaches for a tool, gets denied, and spends the turn on it — the CLI then
returns `error_max_turns` and the question scores as a pipeline exception. Same trap
as grok_cli, different CLI: *disallowing tools stops an agent from touching the
machine, it does not stop it from behaving like an agent.* Give it turns to spare.
- **The prompt goes over stdin**, not argv: it runs ~40k characters, past the Windows
command-line limit.
"""
from __future__ import annotations
import json
import subprocess
import tempfile
import time
from nl_sql.llm.providers.base import (
GenerateRequest,
GenerateResponse,
ProviderError,
)
# A generator has no business touching the machine, and a stray tool turn costs
# plan quota. Names are the Claude Code tool names, not the grok ones.
_DISALLOWED_TOOLS = "Bash,Read,Write,Edit,Glob,Grep,WebFetch,WebSearch,Task,TodoWrite,NotebookEdit"
# A dropped process spawn is not an answer. See the retry loop in generate().
_SPAWN_ATTEMPTS = 3
_SPAWN_BACKOFF_SECONDS = 2.0
class ClaudeCliProvider:
name: str = "claude_cli"
def __init__(
self,
cli_path: str = "claude",
model: str = "claude-sonnet-5",
timeout_seconds: float = 600.0,
max_turns: int = 4,
effort: str | None = None,
) -> None:
self.model = model
# Read by the cache wrapper: two runs of the same model at different
# efforts are different experiments and must not share a cache entry.
self.effort = effort
self._cli_path = cli_path
self._timeout = timeout_seconds
self._max_turns = max_turns
self._cwd: str | None = None
def _argv(self, req: GenerateRequest) -> list[str]:
argv = [
self._cli_path,
"-p",
"--model",
self.model,
"--output-format",
"json",
"--max-turns",
str(self._max_turns),
"--disallowedTools",
_DISALLOWED_TOOLS,
]
if self.effort:
# The spawned CLI does not inherit the caller's session effort, so an
# effort ablation only happens if it is passed explicitly here.
argv += ["--effort", self.effort]
if req.system:
# Replace, not append: this is what strips the agent persona.
argv += ["--system-prompt", req.system]
return argv
def _workdir(self) -> str:
"""One scratch directory for the whole run, created once.
It used to be a fresh `TemporaryDirectory` per call, and that was the bug behind
the "claude CLI not found" misses. The CLI outlives `subprocess.run` — cmd.exe
hands off to node.exe, which keeps the cwd handle open a moment longer — so the
context manager's cleanup hit `WinError 32: being used by another process`, and
the NEXT call then tried to spawn into a directory Windows was still tearing
down and got `WinError 2: cannot find the file`. Python raises that as
FileNotFoundError, and this module blamed the CLI path in the message, which is
why the failure read as "the binary is missing" when the binary was fine and the
cwd was not. Retrying could not help: each attempt rebuilt the same race.
A directory that is created once and never deleted mid-run has no race to lose.
It stays a scratch dir, so the model still cannot see the repository (no
CLAUDE.md, no source) — which is the only reason cwd is redirected at all.
"""
if self._cwd is None:
self._cwd = tempfile.mkdtemp(prefix="claude-gen-")
return self._cwd
def _run_once(self, req: GenerateRequest) -> subprocess.CompletedProcess[str]:
return subprocess.run(
self._argv(req),
input=req.prompt,
capture_output=True,
text=True,
encoding="utf-8",
timeout=self._timeout,
cwd=self._workdir(),
check=False,
)
def generate(self, req: GenerateRequest) -> GenerateResponse:
started = time.perf_counter()
proc: subprocess.CompletedProcess[str] | None = None
last_error = ""
# A spawn can still fail transiently (the machine is busy, the shim is slow), so
# the retry stays — but it is no longer papering over a race we caused ourselves.
for attempt in range(_SPAWN_ATTEMPTS):
try:
proc = self._run_once(req)
except FileNotFoundError as exc:
# Could be the CLI, could be the cwd. Say both rather than guess — the
# old message asserted the first and sent us hunting the wrong bug.
last_error = f"spawn failed (cli={self._cli_path!r}, cwd={self._cwd!r}): {exc}"
proc = None
except OSError as exc:
last_error = f"claude CLI failed to spawn: {exc}"
proc = None
except subprocess.TimeoutExpired as exc:
raise ProviderError(
f"claude CLI timed out after {self._timeout}s for model={self.model}"
) from exc
if proc is not None and proc.stdout and proc.stdout.lstrip().startswith("{"):
break
if proc is not None:
last_error = (proc.stderr or proc.stdout or "").strip()[:200]
proc = None
if attempt < _SPAWN_ATTEMPTS - 1:
time.sleep(_SPAWN_BACKOFF_SECONDS * (attempt + 1))
if proc is None:
raise ProviderError(
f"claude CLI produced no usable output after {_SPAWN_ATTEMPTS} attempts: "
f"{last_error}"
)
stdout = proc.stdout or ""
try:
payload = json.loads(stdout)
except ValueError as exc:
tail = (proc.stderr or stdout).strip()[:300]
raise ProviderError(
f"claude CLI returned no JSON (rc={proc.returncode}): {tail}"
) from exc
if payload.get("is_error"):
reason = payload.get("result") or payload.get("subtype") or "unknown"
raise ProviderError(f"claude CLI error: {str(reason)[:300]}")
usage = payload.get("usage") or {}
model_usage = payload.get("modelUsage") or {}
served = next(iter(model_usage), self.model)
# Cached prefix reads are input the model saw; counting only `input_tokens`
# would report ~2 tokens per call and make the run look free.
input_tokens = (
int(usage.get("input_tokens") or 0)
+ int(usage.get("cache_read_input_tokens") or 0)
+ int(usage.get("cache_creation_input_tokens") or 0)
)
return GenerateResponse(
text=str(payload.get("result") or ""),
model=served,
input_tokens=input_tokens,
output_tokens=int(usage.get("output_tokens") or 0),
latency_ms=(time.perf_counter() - started) * 1000.0,
)
|