Hussien Haider commited on
Commit
f750f23
·
1 Parent(s): d00336f
Files changed (2) hide show
  1. app/api/onevone.py +27 -1
  2. app/services/evaluator.py +82 -6
app/api/onevone.py CHANGED
@@ -238,6 +238,29 @@ def _check_red_answer(challenge_row: dict, payload: str) -> bool:
238
  return False
239
 
240
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
241
  async def _ai_check_blue_answer(challenge_row: dict, challenge_type: str, payload: dict) -> bool:
242
  """Blue Team: delegate to the existing AI evaluators (code-fix, log-analysis, vuln-hunter).
243
 
@@ -621,9 +644,12 @@ async def submit_answer(match_id: str, req: SubmitRequest, user: dict = Depends(
621
  if not is_blue_clientverdict:
622
  challenge_row = await _load_challenge_row(match["challenge_id"], match["challenge_type"])
623
 
624
- if match["challenge_type"] in ("crypto", "steganography", "web-exploitation"):
625
  # Red team: deterministic flag match. Server is the source of truth.
626
  correct = _check_red_answer(challenge_row, str(req.submission or ""))
 
 
 
627
  elif match["challenge_type"] in ("code-fixing", "log-analysis", "vulnerability-hunter"):
628
  # Blue team: the local TrainingSession evaluation is the source of
629
  # truth. The AI verifier is non-deterministic and re-evaluating on
 
238
  return False
239
 
240
 
241
+ async def _ai_check_red_answer(challenge_row: dict, payload: str) -> bool:
242
+ """Red Team (web-exploitation): AI-based evaluation via Mistral.
243
+
244
+ Falls back to deterministic flag match when Mistral is unavailable.
245
+ The evaluator returns True when Mistral judges the payload as a
246
+ successful exploit, OR when the flag matches exactly.
247
+ """
248
+ # Quick deterministic check first
249
+ if _check_red_answer(challenge_row, payload):
250
+ return True
251
+ # Try AI evaluation
252
+ from app.services.evaluator import _grade_web_exploit_with_mistral
253
+ result = await _grade_web_exploit_with_mistral(
254
+ payload=payload,
255
+ vuln_type=challenge_row.get("vulnerability_type", ""),
256
+ vuln_description=challenge_row.get("vulnerability_description", ""),
257
+ http_request=challenge_row.get("http_request", ""),
258
+ http_response=challenge_row.get("http_response", ""),
259
+ expected_flag=challenge_row.get("flag_preview", ""),
260
+ )
261
+ return result.get("secured", False)
262
+
263
+
264
  async def _ai_check_blue_answer(challenge_row: dict, challenge_type: str, payload: dict) -> bool:
265
  """Blue Team: delegate to the existing AI evaluators (code-fix, log-analysis, vuln-hunter).
266
 
 
644
  if not is_blue_clientverdict:
645
  challenge_row = await _load_challenge_row(match["challenge_id"], match["challenge_type"])
646
 
647
+ if match["challenge_type"] in ("crypto", "steganography"):
648
  # Red team: deterministic flag match. Server is the source of truth.
649
  correct = _check_red_answer(challenge_row, str(req.submission or ""))
650
+ elif match["challenge_type"] == "web-exploitation":
651
+ # Web exploitation: flag match + AI fallback
652
+ correct = await _ai_check_red_answer(challenge_row, str(req.submission or ""))
653
  elif match["challenge_type"] in ("code-fixing", "log-analysis", "vulnerability-hunter"):
654
  # Blue team: the local TrainingSession evaluation is the source of
655
  # truth. The AI verifier is non-deterministic and re-evaluating on
app/services/evaluator.py CHANGED
@@ -612,13 +612,83 @@ async def evaluate_vuln_hunter(req, background_tasks: BackgroundTasks) -> dict:
612
 
613
 
614
  # --------------------------------------------------------------------------- #
615
- # Web Exploitation — flag-based match (Red Team) #
616
  # --------------------------------------------------------------------------- #
617
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
618
  async def ai_evaluate_web_exploit(
619
  challenge_id: str, payload: str, team_role: str = "red"
620
  ) -> dict:
621
- """Evaluate a web exploitation payload submission."""
622
  row = await fetch_scenario_by_id(
623
  team_role, challenge_id, challenge_type="web-exploitation"
624
  )
@@ -652,10 +722,16 @@ async def ai_evaluate_web_exploit(
652
  "secured": True,
653
  "feedback": "إجابة صحيحة! تم استغلال الثغرة بنجاح.",
654
  }
655
- return {
656
- "secured": False,
657
- "feedback": "الإجابة غير صحيحة. تأكد من أنك استخدمت العلم الصحيح أو الحمولة المناسبة للثغرة.",
658
- }
 
 
 
 
 
 
659
 
660
 
661
  async def evaluate_web_exploit(req, background_tasks: BackgroundTasks) -> dict:
 
612
 
613
 
614
  # --------------------------------------------------------------------------- #
615
+ # Web Exploitation — flag + AI evaluation (Red Team) #
616
  # --------------------------------------------------------------------------- #
617
 
618
+ async def _grade_web_exploit_with_mistral(
619
+ payload: str,
620
+ vuln_type: str,
621
+ vuln_description: str,
622
+ http_request: str,
623
+ http_response: str,
624
+ expected_flag: str,
625
+ ) -> dict:
626
+ """Ask Mistral to evaluate whether the submitted payload successfully
627
+ exploits the web vulnerability described in the challenge.
628
+
629
+ Returns ``{"secured": True|False, "feedback": "..."}``. On any Mistral
630
+ error, returns a hard fail so the student can retry.
631
+ """
632
+ if not MISTRAL_API_KEY:
633
+ return {"secured": False, "feedback": "Mistral غير مهيأ للتقييم"}
634
+
635
+ system_prompt = (
636
+ "أنت خبير أمن تطبيقات ويب مهمتك تقييم إجابة الطالب في تحدي استغلال ثغرات ويب. "
637
+ "الطالب يرسل حمولة (payload) تستهدف ثغرة معينة. مهمتك تحديد ما إذا كانت الحمولة "
638
+ "تستغل الثغرة بنجاح أم لا.\n\n"
639
+ "معايير التقييم:\n"
640
+ "- إذا كانت الحمولة تستغل الثغرة بشكل صحيح وتؤدي إلى نجاح الهجوم، فالإجابة صحيحة.\n"
641
+ "- إذا كانت الحمولة غير صحيحة أو لا تستغل الثغرة، فالإجابة خاطئة.\n"
642
+ "- الحمولة قد تكون مختلفه عن العلم المتوقع ولكنها ما زالت صحيحة إذا كانت تستغل الثغرة.\n\n"
643
+ "أرجع JSON فقط:\n"
644
+ '{"secured": true|false, "feedback": "..."}'
645
+ )
646
+
647
+ user_prompt = (
648
+ f"نوع الثغرة: {vuln_type}\n"
649
+ f"وصف الثغرة: {vuln_description}\n"
650
+ f"الطلب الأصلي (HTTP Request):\n{http_request}\n\n"
651
+ f"الاستجابة الأصلية (HTTP Response):\n{http_response}\n\n"
652
+ f"العلم المتوقع: {expected_flag}\n"
653
+ f"---\n"
654
+ f"حمولة الطالب: {payload}\n\n"
655
+ "هل هذه الحمولة تستغل الثغرة بنجاح؟ قيمها."
656
+ )
657
+
658
+ try:
659
+ async with httpx.AsyncClient(timeout=25) as client:
660
+ r = await client.post(
661
+ MISTRAL_API_URL,
662
+ json={
663
+ "model": MISTRAL_MODEL,
664
+ "messages": [
665
+ {"role": "system", "content": system_prompt},
666
+ {"role": "user", "content": user_prompt},
667
+ ],
668
+ "temperature": 0.0,
669
+ "max_tokens": 400,
670
+ "response_format": {"type": "json_object"},
671
+ },
672
+ headers={
673
+ "Content-Type": "application/json",
674
+ "Authorization": f"Bearer {MISTRAL_API_KEY}",
675
+ },
676
+ )
677
+ if r.status_code != 200:
678
+ return {"secured": False, "feedback": f"Mistral HTTP {r.status_code}"}
679
+ content = r.json().get("choices", [{}])[0].get("message", {}).get("content", "")
680
+ data = parse_json_safe(content) or {}
681
+ secured = bool(data.get("secured", False))
682
+ feedback = str(data.get("feedback", "")).strip() or "تم التقييم بواسطة Mistral"
683
+ return {"secured": secured, "feedback": feedback}
684
+ except Exception as e:
685
+ return {"secured": False, "feedback": f"خطأ في التقييم: {e}"}
686
+
687
+
688
  async def ai_evaluate_web_exploit(
689
  challenge_id: str, payload: str, team_role: str = "red"
690
  ) -> dict:
691
+ """Evaluate a web exploitation payload submission — flag match first, then AI."""
692
  row = await fetch_scenario_by_id(
693
  team_role, challenge_id, challenge_type="web-exploitation"
694
  )
 
722
  "secured": True,
723
  "feedback": "إجابة صحيحة! تم استغلال الثغرة بنجاح.",
724
  }
725
+
726
+ # Flag didn't match — fall back to AI semantic evaluation
727
+ return await _grade_web_exploit_with_mistral(
728
+ payload=payload,
729
+ vuln_type=row.get("vulnerability_type", ""),
730
+ vuln_description=row.get("vulnerability_description", ""),
731
+ http_request=row.get("http_request", ""),
732
+ http_response=row.get("http_response", ""),
733
+ expected_flag=expected,
734
+ )
735
 
736
 
737
  async def evaluate_web_exploit(req, background_tasks: BackgroundTasks) -> dict: