File size: 14,325 Bytes
e172b08
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d9f1a08
6295ee4
d9f1a08
6295ee4
a1aca76
e172b08
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
75a3e57
e172b08
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
75a3e57
 
e172b08
 
75a3e57
 
e172b08
 
 
75a3e57
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e172b08
 
 
 
 
6295ee4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4187246
488c3c9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4187246
 
 
 
 
 
 
 
 
 
 
488c3c9
 
4187246
 
 
 
e172b08
 
 
4187246
 
e172b08
4187246
e172b08
 
 
6295ee4
 
 
e172b08
 
 
 
 
 
75a3e57
 
e172b08
 
 
 
 
 
 
 
 
 
 
 
d9f1a08
fb12f93
e172b08
 
 
75a3e57
 
 
 
fb12f93
 
 
e172b08
 
 
 
 
6295ee4
 
 
 
 
ae3d74f
 
 
e172b08
75a3e57
 
 
 
 
 
 
 
 
e172b08
 
6295ee4
 
 
 
 
75a3e57
 
 
 
73db23b
 
 
 
 
 
 
 
 
 
 
75a3e57
fb12f93
75a3e57
e172b08
1d13196
 
 
 
 
 
 
6295ee4
 
fb12f93
 
75a3e57
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
"""ViKernel Open Red-Team Challenge: HF OAuth login, submissions and leaderboard.
Lessons applied: the Space runs inside an iframe on huggingface.co, so the session cookie is SameSite=None; Secure,
login links open with target=_top, and every OAuth failure returns a distinct ?login=<stage> code."""
import base64, hashlib, hmac, json, os, secrets, time, urllib.parse, urllib.request

PROVIDER = os.environ.get("OPENID_PROVIDER_URL", "https://huggingface.co").rstrip("/")
CLIENT_ID = os.environ.get("OAUTH_CLIENT_ID", "")
CLIENT_SECRET = os.environ.get("OAUTH_CLIENT_SECRET", "")
HOST = os.environ.get("SPACE_HOST", "")
COOKIE_KEY = (os.environ.get("COOKIE_KEY") or secrets.token_hex(32)).encode()
LEDGER = os.environ.get("LEDGER_REPO", "FINAL-Bench/vk-redteam-ledger")
LEDGER_TOKEN = os.environ.get("LEDGER_TOKEN", "")
DAILY = int(os.environ.get("RT_DAILY_LIMIT", "5"))
EXCLUDE = {x.strip().lower() for x in os.environ.get("RT_EXCLUDE", "SeaWolf-AI,FINAL-Bench,VIDraft,ginigen-ai").split(",")}
POINTS = {"S": 100, "A": 40, "B": 5}
B_CAP = 50
CATS = {"S-exec": "S", "S-kill": "S", "S-audit": "S", "S-inject": "S", "A-leak": "A", "B-shell": "B"}
# three seasons run at the same time; each category belongs to exactly one season
SEASON_OF = {"S-exec": "s1", "B-shell": "s1", "A-leak": "s2", "S-inject": "s3", "S-kill": "s4", "S-audit": "s4"}
# (start day offset, length in days): seasons 1 and 2 run together, season 3 opens after them
WINDOWS = {"s1": (0, 60), "s2": (0, 60), "s3": (0, 60), "s4": (60, 30)}
_states = {}
_cache = {"t": 0, "board": None}
_daily = {}


def _sign(v):
    return v + "." + hmac.new(COOKIE_KEY, v.encode(), hashlib.sha256).hexdigest()[:32]


def _unsign(s):
    if not s or "." not in s:
        return None
    v, sig = s.rsplit(".", 1)
    return v if hmac.compare_digest(sig, hmac.new(COOKIE_KEY, v.encode(), hashlib.sha256).hexdigest()[:32]) else None


def user_from(headers):
    for part in (headers.get("Cookie") or "").split(";"):
        k, _, v = part.strip().partition("=")
        if k == "rt_user":
            raw = _unsign(urllib.parse.unquote(v))
            if raw:
                name, _, exp = raw.rpartition("|")
                if exp.isdigit() and int(exp) > time.time():
                    return name
    return None


def cookie_for(name):
    val = urllib.parse.quote(_sign(name + "|" + str(int(time.time()) + 7 * 86400)))
    return "rt_user=%s; Path=/; Max-Age=604800; HttpOnly; Secure; SameSite=None" % val


def login_url():
    st = secrets.token_urlsafe(16)
    _states[st] = time.time()
    for k in [k for k, t in _states.items() if time.time() - t > 900]:
        _states.pop(k, None)
    q = urllib.parse.urlencode({"client_id": CLIENT_ID, "redirect_uri": "https://%s/oauth/callback" % HOST,
                                "scope": "openid profile email", "state": st, "response_type": "code"})
    return PROVIDER + "/oauth/authorize?" + q


def callback(query):
    q = urllib.parse.parse_qs(query)
    code, st = (q.get("code") or [""])[0], (q.get("state") or [""])[0]
    if not code:
        return None, "nocode"
    if st not in _states:
        return None, "state"
    _states.pop(st, None)
    try:
        body = urllib.parse.urlencode({"grant_type": "authorization_code", "code": code,
                                       "redirect_uri": "https://%s/oauth/callback" % HOST, "client_id": CLIENT_ID}).encode()
        auth = base64.b64encode(("%s:%s" % (CLIENT_ID, CLIENT_SECRET)).encode()).decode()
        r = urllib.request.urlopen(urllib.request.Request(PROVIDER + "/oauth/token", data=body,
                                   headers={"Authorization": "Basic " + auth, "Content-Type": "application/x-www-form-urlencoded"}), timeout=15)
        tok = json.loads(r.read())["access_token"]
    except Exception:  # noqa: BLE001
        return None, "exchange"
    try:
        r = urllib.request.urlopen(urllib.request.Request(PROVIDER + "/oauth/userinfo", headers={"Authorization": "Bearer " + tok}), timeout=15)
        info = json.loads(r.read())
        name = info.get("preferred_username")
    except Exception:  # noqa: BLE001
        return None, "userinfo"
    if name:
        _save_participant(name, info.get("email"), info.get("email_verified"))
    return (name, None) if name else (None, "userinfo")


_has_email = set()


def _save_participant(name, email, verified):
    """Record the account email in the private ledger (participants/<user>.json). Used for judgement and prize notices."""
    if not (LEDGER_TOKEN and email):
        return
    try:
        api = _api()
        path = "participants/%s.json" % name
        now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
        rec = {"user": name, "first_login": now}
        try:
            rec.update(json.loads(open(api.hf_hub_download(LEDGER, path, repo_type="dataset"), encoding="utf-8").read()))
        except Exception:  # noqa: BLE001
            pass
        if rec.get("email") == email and rec.get("last_login", "")[:10] == now[:10]:
            _has_email.add(name.lower())
            return
        rec.update(email=email, email_verified=bool(verified), last_login=now)
        api.upload_file(path_or_fileobj=json.dumps(rec, ensure_ascii=False, indent=1).encode(), path_in_repo=path, repo_id=LEDGER,
                        repo_type="dataset", commit_message="participant " + name)
        _has_email.add(name.lower())
    except Exception:  # noqa: BLE001
        pass


def has_email(user):
    if user.lower() in _has_email:
        return True
    try:
        api = _api()
        rec = json.loads(open(api.hf_hub_download(LEDGER, "participants/%s.json" % user, repo_type="dataset"), encoding="utf-8").read())
        if rec.get("email"):
            _has_email.add(user.lower())
            return True
    except Exception:  # noqa: BLE001
        pass
    return False


def _api():
    from huggingface_hub import HfApi
    return HfApi(token=LEDGER_TOKEN)


def _start():
    if not SEASON_START:
        return None
    import calendar
    try:
        return calendar.timegm(time.strptime(SEASON_START, "%Y-%m-%dT%H:%M") if "T" in SEASON_START else time.strptime(SEASON_START, "%Y-%m-%d"))
    except ValueError:
        return None


def windows():
    st = _start()
    out = {}
    for k, (off, n) in WINDOWS.items():
        if st is None:
            out[k] = {"opens_at": None, "ends_at": None, "state": "soon"}
            continue
        o, e = st + off * 86400, st + (off + n) * 86400
        now = time.time()
        out[k] = {"opens_at": o, "ends_at": e, "state": "open" if o <= now < e else ("soon" if now < o else "closed")}
    return out


_subs_cache = {"t": 0, "files": []}
_acc_cache = {"t": 0, "names": set()}


def _accepted_names():
    """File names of submissions with an accepted judgement (cached for 2 minutes)."""
    if time.time() - _acc_cache["t"] > 120:
        names = set()
        api = _api()
        for f in api.list_repo_files(LEDGER, repo_type="dataset"):
            if f.startswith("judgements/") and f.endswith(".json"):
                try:
                    if json.loads(open(api.hf_hub_download(LEDGER, f, repo_type="dataset"), encoding="utf-8").read()).get("accepted"):
                        names.add(f.split("/", 1)[1])
                except Exception:  # noqa: BLE001
                    pass
        _acc_cache.update(t=time.time(), names=names)
    return _acc_cache["names"]


def _used_today(user):
    """Submissions this account already has in the ledger for the current UTC day (file name = <YYYYmmddHHMMSS>-<user>-<hex>)."""
    if not LEDGER_TOKEN:
        return 0
    try:
        if time.time() - _subs_cache["t"] > 30:
            _subs_cache.update(t=time.time(), files=[f for f in _api().list_repo_files(LEDGER, repo_type="dataset") if f.startswith("submissions/")])
        day = time.strftime("%Y%m%d", time.gmtime())
        tag = "-" + user.lower() + "-"
        mine = [f.split("/", 1)[1] for f in _subs_cache["files"] if f[12:20] == day and tag in f.lower()]
        return sum(1 for f in mine if f not in _accepted_names())  # accepted findings do not use up the daily limit
    except Exception:  # noqa: BLE001
        return 0


def submit(user, req):
    day = time.strftime("%Y-%m-%d", time.gmtime())
    key = (user.lower(), day)
    used = max(_daily.get(key, 0), _used_today(user))  # counted from the ledger, so a restart does not reset it
    if used >= DAILY:
        return 429, {"error": "daily limit reached (%d per day)" % DAILY}
    _daily[key] = used
    cat = str(req.get("category", ""))
    if cat not in CATS:
        return 400, {"error": "unknown category"}
    w = windows()[SEASON_OF[cat]]
    if SEASON_START and w["state"] != "open":
        return 403, {"error": "this season is not open"}
    rec = {"user": user, "category": cat, "session": str(req.get("session", ""))[:80], "audit_seq": str(req.get("audit_seq", ""))[:40],
           "input": str(req.get("input", ""))[:6000], "description": str(req.get("description", ""))[:4000],
           "submitted_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}
    sid = "%s-%s-%s" % (time.strftime("%Y%m%d%H%M%S", time.gmtime()), user, secrets.token_hex(3))
    if not LEDGER_TOKEN:
        return 503, {"error": "submissions not open yet"}
    if user.lower() not in EXCLUDE and not has_email(user):
        return 401, {"error": "Please sign in again before submitting. We now record your Hugging Face account email for judgement and prize notices.", "relogin": True}
    try:
        _api().upload_file(path_or_fileobj=json.dumps(rec, ensure_ascii=False, indent=1).encode(), path_in_repo="submissions/%s.json" % sid,
                           repo_id=LEDGER, repo_type="dataset", commit_message="submission " + sid)
    except Exception:  # noqa: BLE001
        return 503, {"error": "could not store the submission, please retry"}
    _daily[key] = _daily.get(key, 0) + 1
    return 200, {"ok": True, "id": sid, "left_today": DAILY - _daily[key]}


def leaderboard():
    if time.time() - _cache["t"] < 60 and _cache["board"] is not None:
        return _cache["board"]
    rows = {"s1": {}, "s2": {}, "s3": {}, "s4": {}}
    causes, hall, first, names, n_subs = {}, {}, {}, {}, 0
    if LEDGER_TOKEN:
        try:
            api = _api()
            try:
                names = json.loads(open(api.hf_hub_download(LEDGER, "causes.json", repo_type="dataset"), encoding="utf-8").read())
            except Exception:  # noqa: BLE001
                names = {}
            allfiles = api.list_repo_files(LEDGER, repo_type="dataset")
            n_subs = sum(1 for f in allfiles if f.startswith("submissions/") and f.endswith(".json"))
            for f in allfiles:
                if not (f.startswith("judgements/") and f.endswith(".json")):
                    continue
                j = json.loads(open(api.hf_hub_download(LEDGER, f, repo_type="dataset"), encoding="utf-8").read())
                if not j.get("accepted") or str(j.get("user", "")).lower() in EXCLUDE:
                    continue
                cat = j.get("category"); sea = SEASON_OF.get(cat); tier = CATS.get(cat, "")
                if not sea:
                    continue
                u = j["user"]
                r = rows[sea].setdefault(u, {"user": u, "S": 0, "A": 0, "B": 0, "points": 0, "reached": ""})
                # one B submission may cover several shell-check families; the judgement records how many were accepted
                n = max(1, min(int(j.get("families", 1) or 1), 10)) if tier == "B" else 1
                r[tier] = r.get(tier, 0) + n
                r["reached"] = max(r["reached"], j.get("submitted_at", ""))
                at = j.get("submitted_at", "")
                h = hall.setdefault(u, {"user": u, "S": 0, "A": 0, "B": 0, "causes": 0, "first": []})
                h[tier] = h.get(tier, 0) + n
                if at and (sea not in first or at < first[sea][0]):
                    first[sea] = (at, u)
                ck = j.get("cause")
                if ck and (ck not in causes or at < causes[ck]["at"]):
                    nm = names.get(ck, {})
                    causes[ck] = {"key": ck, "en": nm.get("en", ck), "ko": nm.get("ko", ck), "season": sea, "finder": u, "at": at, "fixed_at": j.get("fixed_at", "")}
        except Exception:  # noqa: BLE001
            pass
    board = {}
    for sea, rs in rows.items():
        for r in rs.values():
            r["points"] = r["S"] * POINTS["S"] + r["A"] * POINTS["A"] + min(r["B"] * POINTS["B"], B_CAP)
        board[sea] = sorted(rs.values(), key=lambda r: (-r["points"], r["reached"]))
    for c in causes.values():
        hall[c["finder"]]["causes"] += 1
    for sea, (_at, u) in first.items():
        hall[u]["first"].append(sea)
    # The "fixed causes" list must show every cause recorded in causes.json, not only the ones whose
    # judgement carried a cause key. Attributed causes keep their finder and date; the rest are listed by name.
    if names:
        all_causes = []
        for k, nm in names.items():
            nm = nm if isinstance(nm, dict) else {}
            all_causes.append(causes.get(k) or {"key": k, "en": nm.get("en", k), "ko": nm.get("ko", k),
                                                 "season": "", "finder": "", "at": "", "fixed_at": ""})
    else:
        all_causes = list(causes.values())
    extra = {"causes": sorted(all_causes, key=lambda c: (c["at"] == "", c["at"], c["key"])),
             "hall": sorted(hall.values(), key=lambda h: (-h["causes"], -(h["S"] + h["A"]), -h["B"], h["user"].lower())),
             "families": sum(h["B"] for h in hall.values()), "submissions": n_subs}
    _cache.update(t=time.time(), board=board, extra=extra)
    return board


SEASON_START = os.environ.get("RT_SEASON_START", "")  # YYYY-MM-DD, set on launch day
SEASON_DAYS = int(os.environ.get("RT_SEASON_DAYS", "90"))


def board_info():
    seasons = leaderboard()
    acc = sum(r["S"] + r["A"] + r["B"] for rs in seasons.values() for r in rs)
    extra = _cache.get("extra") or {"causes": [], "hall": [], "families": 0, "submissions": 0}
    return {"seasons": seasons, "windows": windows(), "submissions": extra.get("submissions", 0), "accepted": acc,
            "causes": extra["causes"], "hall": extra["hall"], "families": extra["families"]}