Spaces:
Sleeping
Sleeping
Hussien Haider commited on
Commit ·
f750f23
1
Parent(s): d00336f
- app/api/onevone.py +27 -1
- app/services/evaluator.py +82 -6
app/api/onevone.py
CHANGED
|
@@ -238,6 +238,29 @@ def _check_red_answer(challenge_row: dict, payload: str) -> bool:
|
|
| 238 |
return False
|
| 239 |
|
| 240 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 241 |
async def _ai_check_blue_answer(challenge_row: dict, challenge_type: str, payload: dict) -> bool:
|
| 242 |
"""Blue Team: delegate to the existing AI evaluators (code-fix, log-analysis, vuln-hunter).
|
| 243 |
|
|
@@ -621,9 +644,12 @@ async def submit_answer(match_id: str, req: SubmitRequest, user: dict = Depends(
|
|
| 621 |
if not is_blue_clientverdict:
|
| 622 |
challenge_row = await _load_challenge_row(match["challenge_id"], match["challenge_type"])
|
| 623 |
|
| 624 |
-
if match["challenge_type"] in ("crypto", "steganography"
|
| 625 |
# Red team: deterministic flag match. Server is the source of truth.
|
| 626 |
correct = _check_red_answer(challenge_row, str(req.submission or ""))
|
|
|
|
|
|
|
|
|
|
| 627 |
elif match["challenge_type"] in ("code-fixing", "log-analysis", "vulnerability-hunter"):
|
| 628 |
# Blue team: the local TrainingSession evaluation is the source of
|
| 629 |
# truth. The AI verifier is non-deterministic and re-evaluating on
|
|
|
|
| 238 |
return False
|
| 239 |
|
| 240 |
|
| 241 |
+
async def _ai_check_red_answer(challenge_row: dict, payload: str) -> bool:
|
| 242 |
+
"""Red Team (web-exploitation): AI-based evaluation via Mistral.
|
| 243 |
+
|
| 244 |
+
Falls back to deterministic flag match when Mistral is unavailable.
|
| 245 |
+
The evaluator returns True when Mistral judges the payload as a
|
| 246 |
+
successful exploit, OR when the flag matches exactly.
|
| 247 |
+
"""
|
| 248 |
+
# Quick deterministic check first
|
| 249 |
+
if _check_red_answer(challenge_row, payload):
|
| 250 |
+
return True
|
| 251 |
+
# Try AI evaluation
|
| 252 |
+
from app.services.evaluator import _grade_web_exploit_with_mistral
|
| 253 |
+
result = await _grade_web_exploit_with_mistral(
|
| 254 |
+
payload=payload,
|
| 255 |
+
vuln_type=challenge_row.get("vulnerability_type", ""),
|
| 256 |
+
vuln_description=challenge_row.get("vulnerability_description", ""),
|
| 257 |
+
http_request=challenge_row.get("http_request", ""),
|
| 258 |
+
http_response=challenge_row.get("http_response", ""),
|
| 259 |
+
expected_flag=challenge_row.get("flag_preview", ""),
|
| 260 |
+
)
|
| 261 |
+
return result.get("secured", False)
|
| 262 |
+
|
| 263 |
+
|
| 264 |
async def _ai_check_blue_answer(challenge_row: dict, challenge_type: str, payload: dict) -> bool:
|
| 265 |
"""Blue Team: delegate to the existing AI evaluators (code-fix, log-analysis, vuln-hunter).
|
| 266 |
|
|
|
|
| 644 |
if not is_blue_clientverdict:
|
| 645 |
challenge_row = await _load_challenge_row(match["challenge_id"], match["challenge_type"])
|
| 646 |
|
| 647 |
+
if match["challenge_type"] in ("crypto", "steganography"):
|
| 648 |
# Red team: deterministic flag match. Server is the source of truth.
|
| 649 |
correct = _check_red_answer(challenge_row, str(req.submission or ""))
|
| 650 |
+
elif match["challenge_type"] == "web-exploitation":
|
| 651 |
+
# Web exploitation: flag match + AI fallback
|
| 652 |
+
correct = await _ai_check_red_answer(challenge_row, str(req.submission or ""))
|
| 653 |
elif match["challenge_type"] in ("code-fixing", "log-analysis", "vulnerability-hunter"):
|
| 654 |
# Blue team: the local TrainingSession evaluation is the source of
|
| 655 |
# truth. The AI verifier is non-deterministic and re-evaluating on
|
app/services/evaluator.py
CHANGED
|
@@ -612,13 +612,83 @@ async def evaluate_vuln_hunter(req, background_tasks: BackgroundTasks) -> dict:
|
|
| 612 |
|
| 613 |
|
| 614 |
# --------------------------------------------------------------------------- #
|
| 615 |
-
# Web Exploitation — flag
|
| 616 |
# --------------------------------------------------------------------------- #
|
| 617 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 618 |
async def ai_evaluate_web_exploit(
|
| 619 |
challenge_id: str, payload: str, team_role: str = "red"
|
| 620 |
) -> dict:
|
| 621 |
-
"""Evaluate a web exploitation payload submission."""
|
| 622 |
row = await fetch_scenario_by_id(
|
| 623 |
team_role, challenge_id, challenge_type="web-exploitation"
|
| 624 |
)
|
|
@@ -652,10 +722,16 @@ async def ai_evaluate_web_exploit(
|
|
| 652 |
"secured": True,
|
| 653 |
"feedback": "إجابة صحيحة! تم استغلال الثغرة بنجاح.",
|
| 654 |
}
|
| 655 |
-
|
| 656 |
-
|
| 657 |
-
|
| 658 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 659 |
|
| 660 |
|
| 661 |
async def evaluate_web_exploit(req, background_tasks: BackgroundTasks) -> dict:
|
|
|
|
| 612 |
|
| 613 |
|
| 614 |
# --------------------------------------------------------------------------- #
|
| 615 |
+
# Web Exploitation — flag + AI evaluation (Red Team) #
|
| 616 |
# --------------------------------------------------------------------------- #
|
| 617 |
|
| 618 |
+
async def _grade_web_exploit_with_mistral(
|
| 619 |
+
payload: str,
|
| 620 |
+
vuln_type: str,
|
| 621 |
+
vuln_description: str,
|
| 622 |
+
http_request: str,
|
| 623 |
+
http_response: str,
|
| 624 |
+
expected_flag: str,
|
| 625 |
+
) -> dict:
|
| 626 |
+
"""Ask Mistral to evaluate whether the submitted payload successfully
|
| 627 |
+
exploits the web vulnerability described in the challenge.
|
| 628 |
+
|
| 629 |
+
Returns ``{"secured": True|False, "feedback": "..."}``. On any Mistral
|
| 630 |
+
error, returns a hard fail so the student can retry.
|
| 631 |
+
"""
|
| 632 |
+
if not MISTRAL_API_KEY:
|
| 633 |
+
return {"secured": False, "feedback": "Mistral غير مهيأ للتقييم"}
|
| 634 |
+
|
| 635 |
+
system_prompt = (
|
| 636 |
+
"أنت خبير أمن تطبيقات ويب مهمتك تقييم إجابة الطالب في تحدي استغلال ثغرات ويب. "
|
| 637 |
+
"الطالب يرسل حمولة (payload) تستهدف ثغرة معينة. مهمتك تحديد ما إذا كانت الحمولة "
|
| 638 |
+
"تستغل الثغرة بنجاح أم لا.\n\n"
|
| 639 |
+
"معايير التقييم:\n"
|
| 640 |
+
"- إذا كانت الحمولة تستغل الثغرة بشكل صحيح وتؤدي إلى نجاح الهجوم، فالإجابة صحيحة.\n"
|
| 641 |
+
"- إذا كانت الحمولة غير صحيحة أو لا تستغل الثغرة، فالإجابة خاطئة.\n"
|
| 642 |
+
"- الحمولة قد تكون مختلفه عن العلم المتوقع ولكنها ما زالت صحيحة إذا كانت تستغل الثغرة.\n\n"
|
| 643 |
+
"أرجع JSON فقط:\n"
|
| 644 |
+
'{"secured": true|false, "feedback": "..."}'
|
| 645 |
+
)
|
| 646 |
+
|
| 647 |
+
user_prompt = (
|
| 648 |
+
f"نوع الثغرة: {vuln_type}\n"
|
| 649 |
+
f"وصف الثغرة: {vuln_description}\n"
|
| 650 |
+
f"الطلب الأصلي (HTTP Request):\n{http_request}\n\n"
|
| 651 |
+
f"الاستجابة الأصلية (HTTP Response):\n{http_response}\n\n"
|
| 652 |
+
f"العلم المتوقع: {expected_flag}\n"
|
| 653 |
+
f"---\n"
|
| 654 |
+
f"حمولة الطالب: {payload}\n\n"
|
| 655 |
+
"هل هذه الحمولة تستغل الثغرة بنجاح؟ قيمها."
|
| 656 |
+
)
|
| 657 |
+
|
| 658 |
+
try:
|
| 659 |
+
async with httpx.AsyncClient(timeout=25) as client:
|
| 660 |
+
r = await client.post(
|
| 661 |
+
MISTRAL_API_URL,
|
| 662 |
+
json={
|
| 663 |
+
"model": MISTRAL_MODEL,
|
| 664 |
+
"messages": [
|
| 665 |
+
{"role": "system", "content": system_prompt},
|
| 666 |
+
{"role": "user", "content": user_prompt},
|
| 667 |
+
],
|
| 668 |
+
"temperature": 0.0,
|
| 669 |
+
"max_tokens": 400,
|
| 670 |
+
"response_format": {"type": "json_object"},
|
| 671 |
+
},
|
| 672 |
+
headers={
|
| 673 |
+
"Content-Type": "application/json",
|
| 674 |
+
"Authorization": f"Bearer {MISTRAL_API_KEY}",
|
| 675 |
+
},
|
| 676 |
+
)
|
| 677 |
+
if r.status_code != 200:
|
| 678 |
+
return {"secured": False, "feedback": f"Mistral HTTP {r.status_code}"}
|
| 679 |
+
content = r.json().get("choices", [{}])[0].get("message", {}).get("content", "")
|
| 680 |
+
data = parse_json_safe(content) or {}
|
| 681 |
+
secured = bool(data.get("secured", False))
|
| 682 |
+
feedback = str(data.get("feedback", "")).strip() or "تم التقييم بواسطة Mistral"
|
| 683 |
+
return {"secured": secured, "feedback": feedback}
|
| 684 |
+
except Exception as e:
|
| 685 |
+
return {"secured": False, "feedback": f"خطأ في التقييم: {e}"}
|
| 686 |
+
|
| 687 |
+
|
| 688 |
async def ai_evaluate_web_exploit(
|
| 689 |
challenge_id: str, payload: str, team_role: str = "red"
|
| 690 |
) -> dict:
|
| 691 |
+
"""Evaluate a web exploitation payload submission — flag match first, then AI."""
|
| 692 |
row = await fetch_scenario_by_id(
|
| 693 |
team_role, challenge_id, challenge_type="web-exploitation"
|
| 694 |
)
|
|
|
|
| 722 |
"secured": True,
|
| 723 |
"feedback": "إجابة صحيحة! تم استغلال الثغرة بنجاح.",
|
| 724 |
}
|
| 725 |
+
|
| 726 |
+
# Flag didn't match — fall back to AI semantic evaluation
|
| 727 |
+
return await _grade_web_exploit_with_mistral(
|
| 728 |
+
payload=payload,
|
| 729 |
+
vuln_type=row.get("vulnerability_type", ""),
|
| 730 |
+
vuln_description=row.get("vulnerability_description", ""),
|
| 731 |
+
http_request=row.get("http_request", ""),
|
| 732 |
+
http_response=row.get("http_response", ""),
|
| 733 |
+
expected_flag=expected,
|
| 734 |
+
)
|
| 735 |
|
| 736 |
|
| 737 |
async def evaluate_web_exploit(req, background_tasks: BackgroundTasks) -> dict:
|