Spaces:
Runtime error
Runtime error
File size: 2,652 Bytes
8c486a8 53ccc8a 8c486a8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 | """Check 8: Difficulty calibration — golden path steps within tier target."""
from __future__ import annotations
from open_range.protocols import CheckResult, ContainerSet, SnapshotSpec
# Tier -> target golden-path step count.
TIER_TARGETS: dict[int, int] = {
1: 8,
2: 15,
3: 25,
4: 35,
5: 50,
}
TOLERANCE = 0.20 # +/- 20%
class DifficultyCheck:
"""Verify golden-path step count is within +-20% of the tier target.
Also rejects:
- Single-step golden paths (trivial).
- Duplicate consecutive commands (likely builder hallucination).
- Vuln count outside manifest ``difficulty.min_vulns / max_vulns``.
"""
async def check(self, snapshot: SnapshotSpec, containers: ContainerSet) -> CheckResult:
issues: list[str] = []
n_steps = len(snapshot.golden_path)
# --- Step count vs tier target ----------------------------------------
# Tier can come from topology metadata or default to 1.
tier: int = snapshot.topology.get("tier", 1)
target = TIER_TARGETS.get(tier, TIER_TARGETS[1])
lo = int(target * (1 - TOLERANCE))
hi = round(target * (1 + TOLERANCE))
if n_steps < lo or n_steps > hi:
issues.append(
f"golden path has {n_steps} steps; tier {tier} target "
f"is {target} (allowed {lo}-{hi})"
)
# --- Trivial path -----------------------------------------------------
if n_steps <= 1:
issues.append("golden path has <= 1 step (trivial)")
# --- Duplicate consecutive commands -----------------------------------
cmds = [s.command for s in snapshot.golden_path]
for i in range(1, len(cmds)):
if cmds[i] == cmds[i - 1]:
issues.append(f"duplicate consecutive command at steps {i} and {i + 1}")
# --- Vuln count bounds ------------------------------------------------
difficulty = snapshot.topology.get("difficulty", {})
min_v = difficulty.get("min_vulns")
max_v = difficulty.get("max_vulns")
n_vulns = len(snapshot.truth_graph.vulns)
if min_v is not None and n_vulns < min_v:
issues.append(f"only {n_vulns} vuln(s); minimum is {min_v}")
if max_v is not None and n_vulns > max_v:
issues.append(f"{n_vulns} vuln(s); maximum is {max_v}")
passed = len(issues) == 0
return CheckResult(
name="difficulty",
passed=passed,
details={"steps": n_steps, "tier": tier, "target": target, "issues": issues},
error="" if passed else "; ".join(issues),
)
|