anvisinghh Spacexpedition commited on
Commit
359c2ff
·
1 Parent(s): 0240330

Upload 8 files (#3)

Browse files

- Upload 8 files (e15ba9b54d5b7fade3af4eaf628b4e89d2355415)


Co-authored-by: Anmol Kesarwani <Spacexpedition@users.noreply.huggingface.co>

Files changed (5) hide show
  1. README.md +82 -0
  2. env.py +57 -54
  3. gitattributes +35 -0
  4. inference.py +37 -27
  5. openenv.yaml +3 -1
README.md CHANGED
@@ -8,3 +8,85 @@ pinned: false
8
  license: mit
9
  app_port: 8000
10
  ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8
  license: mit
9
  app_port: 8000
10
  ---
11
+ Mail Triage Agent v4 (Security Evaluation)
12
+
13
+ This repository contains a high-fidelity environment and an autonomous agent designed for Email Security Triage. The project focuses on detecting sophisticated threats like "Digital Seduction" (Phishing), typo-squatted domains, and malicious URL redirections.
14
+
15
+ 🚀 Overview
16
+
17
+ The system consists of two primary components:
18
+
19
+ Environment (env.py): A FastAPI-based server implementing the OpenEnv specification. it serves a dataset of 15 email scenarios categorized by difficulty (1 to 3).
20
+
21
+ Agent Logic (inference.py): An LLM-powered agent (using gemini-2.0-flash) that analyzes email metadata, headers, and URLs to make triage decisions.
22
+
23
+ 🛠 Project Structure
24
+
25
+ env.py: The core environment logic. Includes the dataset and scoring metrics.
26
+
27
+ inference.py: The agent's decision-making loop.
28
+
29
+ models.py: Pydantic models defining the Observation and Action spaces.
30
+
31
+ openenv.yaml: Metadata for the OpenEnv benchmark framework.
32
+
33
+ Dockerfile: Containerization setup for deployment.
34
+
35
+ requirements.txt: Python dependencies.
36
+
37
+ 🧪 Scoring Logic
38
+
39
+ The environment uses a sophisticated reward system:
40
+
41
+ Perfect Classification: 1.0 + (difficulty * 0.1)
42
+
43
+ Partial Credit: 0.4 (e.g., classifying Phishing as Spam).
44
+
45
+ Dangerous Failure: -1.5 (e.g., letting Phishing into the INBOX).
46
+
47
+ False Positive: -0.5 (e.g., blocking legitimate mail).
48
+
49
+ Reasoning Bonus: +0.05 for providing detailed justifications.
50
+
51
+ ⚙️ Setup & Installation
52
+
53
+ Prerequisites
54
+
55
+ Docker (optional)
56
+
57
+ Python 3.10+
58
+
59
+ A Google Gemini API Key
60
+
61
+ Local Execution
62
+
63
+ Install dependencies:
64
+
65
+ pip install -r requirements.txt
66
+
67
+
68
+ Set your environment variables:
69
+
70
+ export GEMINI_API_KEY="your_api_key_here"
71
+
72
+
73
+ Run the environment server:
74
+
75
+ uvicorn env:app --host 0.0.0.0 --port 8000
76
+
77
+
78
+ In a separate terminal, run the agent:
79
+
80
+ python inference.py
81
+
82
+
83
+ 🛡 Security Scenarios Covered
84
+
85
+ Clean: Official Manipal or Amazon communications with valid SPF/DKIM.
86
+
87
+ Spam: Marketing mail from Swiggy or Internshala.
88
+
89
+ Phishing: Typo-squatted domains (e.g., manipal-edu.in vs manipal.edu) and shortened URLs.
90
+
91
+ Credential Theft: Fake security alerts from bank/Google look-alikes.
92
+
env.py CHANGED
@@ -1,8 +1,8 @@
1
  import asyncio
2
  import random
3
- from typing import Optional, List, Dict
4
  from types import SimpleNamespace
5
- from fastapi import FastAPI
6
  from openenv.core.env_server import Environment
7
  from models import MyEnvV4Observation, MyEnvV4Action, URLInfo
8
 
@@ -11,16 +11,14 @@ class MyEnvV4Env(Environment):
11
  def __init__(self):
12
  super().__init__()
13
  self.dataset = self._generate_sophisticated_dataset()
14
- # For evaluation reproducibility, we could shuffle, but for benchmark stability, we keep order
15
  self.current_step = 0
16
 
17
  def _generate_sophisticated_dataset(self):
18
  """
19
  Expanded dataset with 15 samples across 3 difficulty levels.
20
- Covers Clean, Spam, and Phishing (Digital Seduction).
21
  """
22
  base_data = [
23
- # LEVEL 1: CLEAR CASES (5)
24
  {
25
  "sender": "registrar@manipal.edu",
26
  "subject": "Semester Registration Open",
@@ -37,7 +35,9 @@ class MyEnvV4Env(Environment):
37
  "body": "Track your package delivery status in your Amazon account.",
38
  "raw_headers": "Received: from a9-12.smtp-out.amazonses.com... SPF: pass; DKIM: pass;",
39
  "auth_results": {"SPF": "pass", "DKIM": "pass", "DMARC": "pass"},
40
- "urls": [{"url": "https://amazon.com/track", "is_shortened": False, "domain_age_days": 8000, "has_ssl": True, "reputation_score": 1.0}],
 
 
41
  "label": "INBOX",
42
  "difficulty": 1
43
  },
@@ -47,7 +47,8 @@ class MyEnvV4Env(Environment):
47
  "body": "Congratulations! You have been selected as our winner. CLAIM YOUR $1M NOW!",
48
  "raw_headers": "Received: from unknown-relay.co (103.22.1.5)... SPF: none; DKIM: fail;",
49
  "auth_results": {"SPF": "none", "DKIM": "fail", "DMARC": "none"},
50
- "urls": [{"url": "http://get-cash-free.net/claim", "is_shortened": False, "domain_age_days": 2, "has_ssl": False, "reputation_score": 0.1}],
 
51
  "label": "SPAM",
52
  "difficulty": 1
53
  },
@@ -57,7 +58,8 @@ class MyEnvV4Env(Environment):
57
  "body": "Buy now and save 90% on all prescription drugs. No prescription needed!",
58
  "raw_headers": "Received: from botnet-node.ru... SPF: softfail;",
59
  "auth_results": {"SPF": "softfail", "DKIM": "none", "DMARC": "none"},
60
- "urls": [{"url": "http://cheap-rx.biz", "is_shortened": False, "domain_age_days": 15, "has_ssl": False, "reputation_score": 0.05}],
 
61
  "label": "SPAM",
62
  "difficulty": 1
63
  },
@@ -67,20 +69,21 @@ class MyEnvV4Env(Environment):
67
  "body": "Your Netflix subscription has expired. Click here to login and update billing.",
68
  "raw_headers": "Received: from suspicious-vps.com... SPF: fail; DMARC: fail;",
69
  "auth_results": {"SPF": "fail", "DKIM": "none", "DMARC": "fail"},
70
- "urls": [{"url": "https://bit.ly/fake-netflix-login", "is_shortened": True, "domain_age_days": 3, "has_ssl": True, "reputation_score": 0.02}],
 
71
  "label": "QUARANTINE",
72
  "difficulty": 1
73
  },
74
-
75
- # LEVEL 2: NUANCED / MARKETING / LEGIT BUT ANNOYING (5)
76
  {
77
  "sender": "news@internshala-mail.com",
78
  "subject": "New Internships in Manipal",
79
  "body": "Check out these new opportunities for CSE students. Apply today!",
80
  "raw_headers": "Received: from mktg.server.com... SPF: pass; DKIM: pass;",
81
  "auth_results": {"SPF": "pass", "DKIM": "pass", "DMARC": "pass"},
82
- "urls": [{"url": "https://internshala.com/n/123", "is_shortened": False, "domain_age_days": 2500, "has_ssl": True, "reputation_score": 0.95}],
83
- "label": "SPAM", # Triaged as Promotions/Spam
 
84
  "difficulty": 2
85
  },
86
  {
@@ -93,14 +96,15 @@ class MyEnvV4Env(Environment):
93
  "label": "SPAM",
94
  "difficulty": 2
95
  },
96
- {
97
  "sender": "hr@startup-hiring.co",
98
  "subject": "Interview Invitation",
99
  "body": "We saw your profile on LinkedIn and want to chat about a role.",
100
  "raw_headers": "Received: from linkedin-referral.com... SPF: neutral;",
101
  "auth_results": {"SPF": "neutral", "DKIM": "none", "DMARC": "none"},
102
- "urls": [{"url": "https://startup-hiring.co/apply", "is_shortened": False, "domain_age_days": 45, "has_ssl": True, "reputation_score": 0.6}],
103
- "label": "INBOX", # Legitimate cold outreach
 
104
  "difficulty": 2
105
  },
106
  {
@@ -109,7 +113,8 @@ class MyEnvV4Env(Environment):
109
  "body": "We detected an unusual login to your account from Russia. Please verify.",
110
  "raw_headers": "Received: from spoofed-host.com... SPF: softfail; DMARC: none;",
111
  "auth_results": {"SPF": "softfail", "DKIM": "none", "DMARC": "none"},
112
- "urls": [{"url": "https://t.co/secure-bank-login", "is_shortened": True, "domain_age_days": 10, "has_ssl": True, "reputation_score": 0.3}],
 
113
  "label": "QUARANTINE",
114
  "difficulty": 2
115
  },
@@ -119,61 +124,66 @@ class MyEnvV4Env(Environment):
119
  "body": "A new personal access token was added to your account. If this wasn't you, click here.",
120
  "raw_headers": "Received: from out-21.smtp.github.com... SPF: pass; DKIM: pass;",
121
  "auth_results": {"SPF": "pass", "DKIM": "pass", "DMARC": "pass"},
122
- "urls": [{"url": "https://github.com/settings/tokens", "is_shortened": False, "domain_age_days": 6000, "has_ssl": True, "reputation_score": 1.0}],
 
123
  "label": "INBOX",
124
  "difficulty": 2
125
  },
126
-
127
- # LEVEL 3: DIGITAL SEDUCTION (PHISHING / SPEAR PHISHING) (5)
128
  {
129
- "sender": "dean.office@manipal-edu.in", # Typo-squatted (.in instead of .edu)
130
  "subject": "Urgent: Faculty Grievance Report",
131
  "body": "A report has been filed against your department. Review the grievances here immediately.",
132
- "raw_headers": "Received: from rogue-vps.xyz (45.33.22.11) by mx.google.com; SPF: fail; DMARC: fail;",
133
  "auth_results": {"SPF": "fail", "DKIM": "none", "DMARC": "fail"},
134
- "urls": [{"url": "https://bit.ly/3xYz1-grievance", "is_shortened": True, "domain_age_days": 5, "has_ssl": True, "reputation_score": 0.05}],
 
135
  "label": "QUARANTINE",
136
  "difficulty": 3
137
  },
138
  {
139
- "sender": "it-support@manipal-helpdesk.com", # Look-alike domain
140
  "subject": "Mandatory Password Reset",
141
  "body": "As per the new MIT security policy, all students must reset their password today.",
142
- "raw_headers": "Received: from mail-delivery.online... SPF: pass; DKIM: pass;", # Attacker set up SPF/DKIM correctly!
143
  "auth_results": {"SPF": "pass", "DKIM": "pass", "DMARC": "none"},
144
- "urls": [{"url": "http://manipal-helpdesk.com/reset", "is_shortened": False, "domain_age_days": 1, "has_ssl": False, "reputation_score": 0.1}],
 
145
  "label": "QUARANTINE",
146
  "difficulty": 3
147
  },
148
  {
149
- "sender": "prof.sharma@mit-manipal.org", # Wrong TLD
150
  "subject": "Final Exam Paper Leak?",
151
- "body": "I suspect the paper has leaked. Look at this screenshot and confirm if these are your questions.",
152
  "raw_headers": "Received: from sendgrid.net... SPF: pass;",
153
  "auth_results": {"SPF": "pass", "DKIM": "none", "DMARC": "none"},
154
- "urls": [{"url": "https://dropbox-files.com/s/xyz", "is_shortened": False, "domain_age_days": 4, "has_ssl": True, "reputation_score": 0.2}],
 
155
  "label": "QUARANTINE",
156
  "difficulty": 3
157
  },
158
  {
159
  "sender": "accounts@google-security.info",
160
  "subject": "Critical Security Alert",
161
- "body": "Someone just used your password to try to sign in to your account. Go to your Google account now.",
162
  "raw_headers": "Received: from host-12.xyz... SPF: fail;",
163
  "auth_results": {"SPF": "fail", "DKIM": "none", "DMARC": "fail"},
164
- "urls": [{"url": "https://google-secure-login.info", "is_shortened": False, "domain_age_days": 2, "has_ssl": True, "reputation_score": 0.01}],
 
165
  "label": "QUARANTINE",
166
  "difficulty": 3
167
  },
168
  {
169
  "sender": "library@manipal.edu",
170
  "subject": "Overdue Book Notice",
171
- "body": "Your copy of 'Computer Networks' is overdue. Click to pay the fine of ₹50.",
172
  "raw_headers": "Received: from mail.manipal.edu... SPF: pass; DKIM: pass;",
173
  "auth_results": {"SPF": "pass", "DKIM": "pass", "DMARC": "pass"},
174
- "urls": [{"url": "https://portal.manipal.edu/pay", "is_shortened": False, "domain_age_days": 4000, "has_ssl": True, "reputation_score": 1.0}],
 
175
  "label": "INBOX",
176
- "difficulty": 3 # Difficult because it looks like a phishing lure but is legit.
177
  }
178
  ]
179
  return base_data
@@ -189,7 +199,7 @@ class MyEnvV4Env(Environment):
189
  hop_count=0, auth_results={}, urls=[], echoed_message="End of Session"
190
  )
191
  return SimpleNamespace(observation=obs, reward=reward, done=True)
192
-
193
  data = self.dataset[self.current_step]
194
  obs = MyEnvV4Observation(
195
  sender=data["sender"],
@@ -210,54 +220,47 @@ class MyEnvV4Env(Environment):
210
  target = self.dataset[self.current_step]
211
  correct = target["label"]
212
  prediction = action.message.strip().upper()
213
-
214
- # SOPHISTICATED REWARD LOGIC
215
  reward = 0.0
216
-
217
  if prediction == correct:
218
- # Perfect Match: reward scales with difficulty
219
- reward = 1.0 + (target["difficulty"] * 0.1)
220
  elif correct in ["SPAM", "QUARANTINE"] and prediction in ["SPAM", "QUARANTINE"]:
221
- # Partial Credit: Recognized threat but misclassified type
222
  reward = 0.4
223
  elif correct == "QUARANTINE" and prediction == "INBOX":
224
- # Dangerous Failure: Penalty for letting a threat into the Inbox
225
- reward = -1.5
226
  elif correct == "INBOX" and prediction == "QUARANTINE":
227
- # False Positive: Penalty for blocking legitimate mail
228
  reward = -0.5
229
-
230
- # Add Reasoning Bonus (Explainability)
231
  if hasattr(action, 'reasoning') and action.reasoning and len(action.reasoning) > 30:
232
- # Small bonus if agent provides a justification
233
  reward += 0.05
234
 
235
- # Normalize reward to [0, 1] range as per OpenEnv specs (clipping/rescaling)
236
- # However, many environments allow negative for penalties; we clip to [0,1] for final score
237
  final_reward = max(0.0, min(1.0, reward))
238
-
239
  self.current_step += 1
240
  done = self.current_step >= len(self.dataset)
241
-
242
  return self._get_result(reward=final_reward, done=done)
243
 
244
  async def state(self):
245
  return {"current_step": self.current_step, "total_tasks": len(self.dataset)}
246
 
247
- # Global instance for the server
 
248
  my_env = MyEnvV4Env()
249
  app = FastAPI()
250
 
251
- @app.get("/reset")
252
- async def reset():
 
253
  res = await my_env.reset()
254
  return {"observation": res.observation, "reward": res.reward, "done": res.done}
255
 
 
256
  @app.post("/step")
257
  async def step(action: MyEnvV4Action):
258
  res = await my_env.step(action)
259
  return {"observation": res.observation, "reward": res.reward, "done": res.done}
260
 
 
261
  @app.get("/state")
262
  async def state():
263
  return await my_env.state()
 
1
  import asyncio
2
  import random
3
+ from typing import Optional, List, Dict, Any
4
  from types import SimpleNamespace
5
+ from fastapi import FastAPI, Body
6
  from openenv.core.env_server import Environment
7
  from models import MyEnvV4Observation, MyEnvV4Action, URLInfo
8
 
 
11
  def __init__(self):
12
  super().__init__()
13
  self.dataset = self._generate_sophisticated_dataset()
 
14
  self.current_step = 0
15
 
16
  def _generate_sophisticated_dataset(self):
17
  """
18
  Expanded dataset with 15 samples across 3 difficulty levels.
 
19
  """
20
  base_data = [
21
+ # LEVEL 1: CLEAR CASES
22
  {
23
  "sender": "registrar@manipal.edu",
24
  "subject": "Semester Registration Open",
 
35
  "body": "Track your package delivery status in your Amazon account.",
36
  "raw_headers": "Received: from a9-12.smtp-out.amazonses.com... SPF: pass; DKIM: pass;",
37
  "auth_results": {"SPF": "pass", "DKIM": "pass", "DMARC": "pass"},
38
+ "urls": [
39
+ {"url": "https://amazon.com/track", "is_shortened": False, "domain_age_days": 8000, "has_ssl": True,
40
+ "reputation_score": 1.0}],
41
  "label": "INBOX",
42
  "difficulty": 1
43
  },
 
47
  "body": "Congratulations! You have been selected as our winner. CLAIM YOUR $1M NOW!",
48
  "raw_headers": "Received: from unknown-relay.co (103.22.1.5)... SPF: none; DKIM: fail;",
49
  "auth_results": {"SPF": "none", "DKIM": "fail", "DMARC": "none"},
50
+ "urls": [{"url": "http://get-cash-free.net/claim", "is_shortened": False, "domain_age_days": 2,
51
+ "has_ssl": False, "reputation_score": 0.1}],
52
  "label": "SPAM",
53
  "difficulty": 1
54
  },
 
58
  "body": "Buy now and save 90% on all prescription drugs. No prescription needed!",
59
  "raw_headers": "Received: from botnet-node.ru... SPF: softfail;",
60
  "auth_results": {"SPF": "softfail", "DKIM": "none", "DMARC": "none"},
61
+ "urls": [{"url": "http://cheap-rx.biz", "is_shortened": False, "domain_age_days": 15, "has_ssl": False,
62
+ "reputation_score": 0.05}],
63
  "label": "SPAM",
64
  "difficulty": 1
65
  },
 
69
  "body": "Your Netflix subscription has expired. Click here to login and update billing.",
70
  "raw_headers": "Received: from suspicious-vps.com... SPF: fail; DMARC: fail;",
71
  "auth_results": {"SPF": "fail", "DKIM": "none", "DMARC": "fail"},
72
+ "urls": [{"url": "https://bit.ly/fake-netflix-login", "is_shortened": True, "domain_age_days": 3,
73
+ "has_ssl": True, "reputation_score": 0.02}],
74
  "label": "QUARANTINE",
75
  "difficulty": 1
76
  },
77
+ # LEVEL 2: NUANCED
 
78
  {
79
  "sender": "news@internshala-mail.com",
80
  "subject": "New Internships in Manipal",
81
  "body": "Check out these new opportunities for CSE students. Apply today!",
82
  "raw_headers": "Received: from mktg.server.com... SPF: pass; DKIM: pass;",
83
  "auth_results": {"SPF": "pass", "DKIM": "pass", "DMARC": "pass"},
84
+ "urls": [{"url": "https://internshala.com/n/123", "is_shortened": False, "domain_age_days": 2500,
85
+ "has_ssl": True, "reputation_score": 0.95}],
86
+ "label": "SPAM",
87
  "difficulty": 2
88
  },
89
  {
 
96
  "label": "SPAM",
97
  "difficulty": 2
98
  },
99
+ {
100
  "sender": "hr@startup-hiring.co",
101
  "subject": "Interview Invitation",
102
  "body": "We saw your profile on LinkedIn and want to chat about a role.",
103
  "raw_headers": "Received: from linkedin-referral.com... SPF: neutral;",
104
  "auth_results": {"SPF": "neutral", "DKIM": "none", "DMARC": "none"},
105
+ "urls": [{"url": "https://startup-hiring.co/apply", "is_shortened": False, "domain_age_days": 45,
106
+ "has_ssl": True, "reputation_score": 0.6}],
107
+ "label": "INBOX",
108
  "difficulty": 2
109
  },
110
  {
 
113
  "body": "We detected an unusual login to your account from Russia. Please verify.",
114
  "raw_headers": "Received: from spoofed-host.com... SPF: softfail; DMARC: none;",
115
  "auth_results": {"SPF": "softfail", "DKIM": "none", "DMARC": "none"},
116
+ "urls": [{"url": "https://t.co/secure-bank-login", "is_shortened": True, "domain_age_days": 10,
117
+ "has_ssl": True, "reputation_score": 0.3}],
118
  "label": "QUARANTINE",
119
  "difficulty": 2
120
  },
 
124
  "body": "A new personal access token was added to your account. If this wasn't you, click here.",
125
  "raw_headers": "Received: from out-21.smtp.github.com... SPF: pass; DKIM: pass;",
126
  "auth_results": {"SPF": "pass", "DKIM": "pass", "DMARC": "pass"},
127
+ "urls": [{"url": "https://github.com/settings/tokens", "is_shortened": False, "domain_age_days": 6000,
128
+ "has_ssl": True, "reputation_score": 1.0}],
129
  "label": "INBOX",
130
  "difficulty": 2
131
  },
132
+ # LEVEL 3: SPEAR PHISHING
 
133
  {
134
+ "sender": "dean.office@manipal-edu.in",
135
  "subject": "Urgent: Faculty Grievance Report",
136
  "body": "A report has been filed against your department. Review the grievances here immediately.",
137
+ "raw_headers": "Received: from rogue-vps.xyz... SPF: fail; DMARC: fail;",
138
  "auth_results": {"SPF": "fail", "DKIM": "none", "DMARC": "fail"},
139
+ "urls": [{"url": "https://bit.ly/3xYz1-grievance", "is_shortened": True, "domain_age_days": 5,
140
+ "has_ssl": True, "reputation_score": 0.05}],
141
  "label": "QUARANTINE",
142
  "difficulty": 3
143
  },
144
  {
145
+ "sender": "it-support@manipal-helpdesk.com",
146
  "subject": "Mandatory Password Reset",
147
  "body": "As per the new MIT security policy, all students must reset their password today.",
148
+ "raw_headers": "Received: from mail-delivery.online... SPF: pass; DKIM: pass;",
149
  "auth_results": {"SPF": "pass", "DKIM": "pass", "DMARC": "none"},
150
+ "urls": [{"url": "http://manipal-helpdesk.com/reset", "is_shortened": False, "domain_age_days": 1,
151
+ "has_ssl": False, "reputation_score": 0.1}],
152
  "label": "QUARANTINE",
153
  "difficulty": 3
154
  },
155
  {
156
+ "sender": "prof.sharma@mit-manipal.org",
157
  "subject": "Final Exam Paper Leak?",
158
+ "body": "I suspect the paper has leaked. Look at this screenshot.",
159
  "raw_headers": "Received: from sendgrid.net... SPF: pass;",
160
  "auth_results": {"SPF": "pass", "DKIM": "none", "DMARC": "none"},
161
+ "urls": [{"url": "https://dropbox-files.com/s/xyz", "is_shortened": False, "domain_age_days": 4,
162
+ "has_ssl": True, "reputation_score": 0.2}],
163
  "label": "QUARANTINE",
164
  "difficulty": 3
165
  },
166
  {
167
  "sender": "accounts@google-security.info",
168
  "subject": "Critical Security Alert",
169
+ "body": "Someone just used your password to try to sign in.",
170
  "raw_headers": "Received: from host-12.xyz... SPF: fail;",
171
  "auth_results": {"SPF": "fail", "DKIM": "none", "DMARC": "fail"},
172
+ "urls": [{"url": "https://google-secure-login.info", "is_shortened": False, "domain_age_days": 2,
173
+ "has_ssl": True, "reputation_score": 0.01}],
174
  "label": "QUARANTINE",
175
  "difficulty": 3
176
  },
177
  {
178
  "sender": "library@manipal.edu",
179
  "subject": "Overdue Book Notice",
180
+ "body": "Your copy of 'Computer Networks' is overdue.",
181
  "raw_headers": "Received: from mail.manipal.edu... SPF: pass; DKIM: pass;",
182
  "auth_results": {"SPF": "pass", "DKIM": "pass", "DMARC": "pass"},
183
+ "urls": [{"url": "https://portal.manipal.edu/pay", "is_shortened": False, "domain_age_days": 4000,
184
+ "has_ssl": True, "reputation_score": 1.0}],
185
  "label": "INBOX",
186
+ "difficulty": 3
187
  }
188
  ]
189
  return base_data
 
199
  hop_count=0, auth_results={}, urls=[], echoed_message="End of Session"
200
  )
201
  return SimpleNamespace(observation=obs, reward=reward, done=True)
202
+
203
  data = self.dataset[self.current_step]
204
  obs = MyEnvV4Observation(
205
  sender=data["sender"],
 
220
  target = self.dataset[self.current_step]
221
  correct = target["label"]
222
  prediction = action.message.strip().upper()
223
+
 
224
  reward = 0.0
 
225
  if prediction == correct:
226
+ reward = 1.0 + (target["difficulty"] * 0.1)
 
227
  elif correct in ["SPAM", "QUARANTINE"] and prediction in ["SPAM", "QUARANTINE"]:
 
228
  reward = 0.4
229
  elif correct == "QUARANTINE" and prediction == "INBOX":
230
+ reward = -1.5
 
231
  elif correct == "INBOX" and prediction == "QUARANTINE":
 
232
  reward = -0.5
233
+
 
234
  if hasattr(action, 'reasoning') and action.reasoning and len(action.reasoning) > 30:
 
235
  reward += 0.05
236
 
 
 
237
  final_reward = max(0.0, min(1.0, reward))
 
238
  self.current_step += 1
239
  done = self.current_step >= len(self.dataset)
240
+
241
  return self._get_result(reward=final_reward, done=done)
242
 
243
  async def state(self):
244
  return {"current_step": self.current_step, "total_tasks": len(self.dataset)}
245
 
246
+
247
+ # Global instance
248
  my_env = MyEnvV4Env()
249
  app = FastAPI()
250
 
251
+
252
+ @app.post("/reset") # FIXED: Must be POST for OpenEnv validators
253
+ async def reset(payload: Dict[Any, Any] = Body(default={})):
254
  res = await my_env.reset()
255
  return {"observation": res.observation, "reward": res.reward, "done": res.done}
256
 
257
+
258
  @app.post("/step")
259
  async def step(action: MyEnvV4Action):
260
  res = await my_env.step(action)
261
  return {"observation": res.observation, "reward": res.reward, "done": res.done}
262
 
263
+
264
  @app.get("/state")
265
  async def state():
266
  return await my_env.state()
gitattributes ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tar filter=lfs diff=lfs merge=lfs -text
29
+ *.tflite filter=lfs diff=lfs merge=lfs -text
30
+ *.tgz filter=lfs diff=lfs merge=lfs -text
31
+ *.wasm filter=lfs diff=lfs merge=lfs -text
32
+ *.xz filter=lfs diff=lfs merge=lfs -text
33
+ *.zip filter=lfs diff=lfs merge=lfs -text
34
+ *.zst filter=lfs diff=lfs merge=lfs -text
35
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
inference.py CHANGED
@@ -1,78 +1,88 @@
1
  import asyncio
2
  import os
3
- from typing import List
4
  from openai import OpenAI
5
  from env import MyEnvV4Env
6
  from models import MyEnvV4Action
7
 
8
  # Environment Configuration
9
- # FIX: Point directly to Google's OpenAI-compatible endpoint for Gemini models
10
  API_BASE_URL = os.getenv("API_BASE_URL") or "https://generativelanguage.googleapis.com/v1beta/openai/"
11
- # FIX: Use Gemini API Key instead of Hugging Face token
12
- API_KEY = os.getenv("GEMINI_API_KEY") or ""
13
  MODEL_NAME = "gemini-2.0-flash"
14
- TASK_NAME = "security-mail-triage"
15
 
16
  SYSTEM_PROMPT = """
17
- You are an Advanced Email Security Agent. Analyze the metadata (headers, SPF/DKIM), URLs, and content.
18
  Categories:
19
- - INBOX: Trusted academic/official domains, passed auth, clean history.
20
- - SPAM: Mass marketing, generic lottery/sales, usually safe but unwanted.
21
- - QUARANTINE: Phishing, spear-phishing, credential theft, high-urgency threats, typo-squatted domains.
22
 
23
- Rules:
24
- 1. Examine 'raw_headers' and 'auth_results'.
25
- 2. Inspect 'urls' for low reputation or high age.
26
- 3. Provide reasoning first, then your decision.
27
-
28
- Respond in JSON format:
29
  {
30
- "reasoning": "Explain your logic here...",
31
  "message": "INBOX|SPAM|QUARANTINE"
32
  }
33
  """
34
 
35
 
36
  async def main():
 
 
 
 
37
  client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
38
  env = MyEnvV4Env()
39
 
40
  rewards = []
41
- print(f"[START] Testing Security Triage Environment...")
42
 
 
43
  result = await env.reset()
44
  step_idx = 1
45
 
46
  while not result.done:
47
  obs = result.observation
48
- prompt = f"Sender: {obs.sender}\nSubject: {obs.subject}\nBody: {obs.body}\nHeaders: {obs.raw_headers}\nURLs: {obs.urls}"
 
 
 
 
 
 
 
 
49
 
50
  try:
51
  response = client.chat.completions.create(
52
  model=MODEL_NAME,
53
- messages=[{"role": "system", "content": SYSTEM_PROMPT}, {"role": "user", "content": prompt}],
 
 
 
54
  response_format={"type": "json_object"},
55
  temperature=0.0
56
  )
57
- import json
58
- data = json.loads(response.choices[0].message.content)
59
 
 
 
 
 
60
  action = MyEnvV4Action(message=data["message"], reasoning=data["reasoning"])
61
  result = await env.step(action)
62
  rewards.append(result.reward)
63
 
64
- print(f"[STEP {step_idx}] Action: {action.message} | Reward: {result.reward:.2f}")
65
  step_idx += 1
66
 
67
- # Prevent hitting Gemini Free Tier rate limits (15 requests per minute & token limits)
68
- # Increased to 10 seconds to ensure we do not hit the burst quotas.
69
- await asyncio.sleep(10)
70
  except Exception as e:
71
  print(f"[ERROR] Step {step_idx}: {e}")
72
  break
73
 
74
- score = sum(rewards) / len(rewards) if rewards else 0
75
- print(f"[END] Final Score: {score:.3f}")
76
 
77
 
78
  if __name__ == "__main__":
 
1
  import asyncio
2
  import os
3
+ import json
4
  from openai import OpenAI
5
  from env import MyEnvV4Env
6
  from models import MyEnvV4Action
7
 
8
  # Environment Configuration
9
+ # Standard OpenEnv evaluation environments inject these env vars
10
  API_BASE_URL = os.getenv("API_BASE_URL") or "https://generativelanguage.googleapis.com/v1beta/openai/"
11
+ API_KEY = os.getenv("GEMINI_API_KEY") or os.getenv("OPENAI_API_KEY") or ""
 
12
  MODEL_NAME = "gemini-2.0-flash"
 
13
 
14
  SYSTEM_PROMPT = """
15
+ You are an Advanced Email Security Agent. Analyze the metadata, URLs, and content.
16
  Categories:
17
+ - INBOX: Trusted academic/official domains, passed auth.
18
+ - SPAM: Unwanted marketing or sales.
19
+ - QUARANTINE: Phishing, typo-squatting, or high-risk threats.
20
 
21
+ Respond in strict JSON:
 
 
 
 
 
22
  {
23
+ "reasoning": "Explain your logic...",
24
  "message": "INBOX|SPAM|QUARANTINE"
25
  }
26
  """
27
 
28
 
29
  async def main():
30
+ if not API_KEY:
31
+ print("[ERROR] No API key found. Please set GEMINI_API_KEY.")
32
+ return
33
+
34
  client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
35
  env = MyEnvV4Env()
36
 
37
  rewards = []
38
+ print(f"[START] Running Security Triage Evaluation...")
39
 
40
+ # OpenEnv Reset
41
  result = await env.reset()
42
  step_idx = 1
43
 
44
  while not result.done:
45
  obs = result.observation
46
+ # Prepare the prompt by dumping complex URL objects to dictionaries
47
+ prompt = (
48
+ f"Sender: {obs.sender}\n"
49
+ f"Subject: {obs.subject}\n"
50
+ f"Body: {obs.body}\n"
51
+ f"Headers: {obs.raw_headers}\n"
52
+ f"Auth: {obs.auth_results}\n"
53
+ f"URLs: {[u.model_dump() for u in obs.urls]}"
54
+ )
55
 
56
  try:
57
  response = client.chat.completions.create(
58
  model=MODEL_NAME,
59
+ messages=[
60
+ {"role": "system", "content": SYSTEM_PROMPT},
61
+ {"role": "user", "content": prompt}
62
+ ],
63
  response_format={"type": "json_object"},
64
  temperature=0.0
65
  )
 
 
66
 
67
+ content = response.choices[0].message.content
68
+ data = json.loads(content)
69
+
70
+ # Create action and step the environment
71
  action = MyEnvV4Action(message=data["message"], reasoning=data["reasoning"])
72
  result = await env.step(action)
73
  rewards.append(result.reward)
74
 
75
+ print(f"[STEP {step_idx}] Result: {action.message} | Reward: {result.reward:.2f}")
76
  step_idx += 1
77
 
78
+ # Sleep to respect rate limits (Gemini 2.0 Flash)
79
+ await asyncio.sleep(2)
 
80
  except Exception as e:
81
  print(f"[ERROR] Step {step_idx}: {e}")
82
  break
83
 
84
+ final_score = sum(rewards) / len(rewards) if rewards else 0
85
+ print(f"[END] Evaluation Complete. Final Score: {final_score:.3f}")
86
 
87
 
88
  if __name__ == "__main__":
openenv.yaml CHANGED
@@ -4,7 +4,7 @@ version: "2.0.0"
4
  description: "A high-fidelity security evaluation environment for email triage, featuring difficulty scaling, technical header analysis, and URL reputation modeling."
5
 
6
  # Environment Specification
7
- repo_url: "https://huggingface.co/spaces/spacexpedition/mail_triage"
8
  task_type: "classification"
9
 
10
  # Compliance Metrics & Scoring
@@ -14,6 +14,8 @@ tags:
14
  - security
15
  - phishing-detection
16
  - metadata-analysis
 
 
17
 
18
  # Typed Model References
19
  # These map to the classes defined in models.py
 
4
  description: "A high-fidelity security evaluation environment for email triage, featuring difficulty scaling, technical header analysis, and URL reputation modeling."
5
 
6
  # Environment Specification
7
+ repo_url: "https://huggingface.co/spaces/YOUR_USERNAME/YOUR_SPACE_NAME"
8
  task_type: "classification"
9
 
10
  # Compliance Metrics & Scoring
 
14
  - security
15
  - phishing-detection
16
  - metadata-analysis
17
+ - mit-manipal-hackathon
18
+ - digital-seduction
19
 
20
  # Typed Model References
21
  # These map to the classes defined in models.py