Che237 commited on
Commit
fc40afd
·
verified ·
1 Parent(s): c823c6a

Update the CyberForge ML service

Browse files
Files changed (3) hide show
  1. Dockerfile +3 -5
  2. app.py +116 -393
  3. requirements.txt +1 -12
Dockerfile CHANGED
@@ -1,4 +1,4 @@
1
- # Use Python 3.11 to avoid the gradio_client Python 3.13 bug
2
  FROM python:3.11-slim
3
 
4
  WORKDIR /app
@@ -25,9 +25,7 @@ WORKDIR $HOME/app
25
  # Copy requirements and install dependencies
26
  COPY --chown=user requirements.txt .
27
  RUN pip install --no-cache-dir --upgrade pip && \
28
- pip install --no-cache-dir -r requirements.txt && \
29
- pip install --no-cache-dir ipykernel && \
30
- python -m ipykernel install --user --name python3 --display-name "Python 3"
31
 
32
  # Copy application code
33
  COPY --chown=user . .
@@ -35,5 +33,5 @@ COPY --chown=user . .
35
  # Expose port
36
  EXPOSE 7860
37
 
38
- # Run the application via uvicorn (FastAPI + Gradio mounted)
39
  CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
 
1
+ # Python 3.11: the version this image has been built and run with
2
  FROM python:3.11-slim
3
 
4
  WORKDIR /app
 
25
  # Copy requirements and install dependencies
26
  COPY --chown=user requirements.txt .
27
  RUN pip install --no-cache-dir --upgrade pip && \
28
+ pip install --no-cache-dir -r requirements.txt
 
 
29
 
30
  # Copy application code
31
  COPY --chown=user . .
 
33
  # Expose port
34
  EXPOSE 7860
35
 
36
+ # Run the API via uvicorn (FastAPI only, no web interface)
37
  CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
app.py CHANGED
@@ -1,12 +1,13 @@
1
  """
2
- CyberForge AI - ML Training & Inference Platform
3
- Hugging Face Spaces deployment with:
4
- 1) Gradio UI for notebook execution, training, and inference
5
- 2) FastAPI REST endpoints for the Heroku backend (mlService.js)
 
 
6
  """
7
 
8
- import gradio as gr
9
- import pandas as pd
10
  import numpy as np
11
  import json
12
  import os
@@ -26,12 +27,8 @@ from urllib.parse import urlparse
26
  from url_features import (extract_url_features, popular_domains_loaded, rescale_score,
27
  set_popular_domains, split_url)
28
 
29
- # ML Libraries
30
- from sklearn.model_selection import train_test_split, cross_val_score
31
- from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, IsolationForest
32
- from sklearn.linear_model import LogisticRegression
33
- from sklearn.preprocessing import StandardScaler, LabelEncoder
34
- from sklearn.metrics import classification_report, confusion_matrix, accuracy_score, f1_score
35
  import joblib
36
 
37
  # Hugging Face Hub
@@ -40,7 +37,7 @@ from huggingface_hub import HfApi, hf_hub_download, upload_file
40
  # FastAPI for REST endpoints
41
  from fastapi import FastAPI, Request
42
  from fastapi.middleware.cors import CORSMiddleware
43
- from fastapi.responses import JSONResponse
44
  from starlette.concurrency import run_in_threadpool
45
 
46
  logging.basicConfig(level=logging.INFO)
@@ -55,7 +52,6 @@ MODELS_DIR = APP_DIR / "trained_models"
55
  MODELS_DIR.mkdir(exist_ok=True)
56
  DATASETS_DIR = APP_DIR / "datasets"
57
  DATASETS_DIR.mkdir(exist_ok=True)
58
- NOTEBOOKS_DIR = APP_DIR / "notebooks"
59
  KNOWLEDGE_BASE_DIR = APP_DIR / "knowledge_base"
60
  KNOWLEDGE_BASE_DIR.mkdir(exist_ok=True)
61
  TRAINING_DATA_DIR = APP_DIR / "training_data"
@@ -80,23 +76,16 @@ LOCAL_LLM_TIME_BUDGET_S = float(os.environ.get("LOCAL_LLM_TIME_BUDGET_S", "17"))
80
  LOCAL_LLM_QUEUE_TIMEOUT_S = float(os.environ.get("LOCAL_LLM_QUEUE_TIMEOUT_S", "5"))
81
 
82
  logger.info(f"APP_DIR: {APP_DIR}")
83
- logger.info(f"NOTEBOOKS_DIR: {NOTEBOOKS_DIR}")
84
- logger.info(f"NOTEBOOKS_DIR exists: {NOTEBOOKS_DIR.exists()}")
 
 
 
 
 
85
 
86
- # Model types available for training
87
- MODEL_TYPES = {
88
- "Random Forest": RandomForestClassifier,
89
- "Gradient Boosting": GradientBoostingClassifier,
90
- "Logistic Regression": LogisticRegression,
91
- "Isolation Forest (Anomaly)": IsolationForest,
92
- }
93
-
94
- SECURITY_TASKS = [
95
- "Malware Detection", "Phishing Detection", "Network Intrusion Detection",
96
- "Anomaly Detection", "Botnet Detection", "Web Attack Detection",
97
- "Spam Detection", "Vulnerability Assessment", "DNS Tunneling Detection",
98
- "Cryptomining Detection",
99
- ]
100
 
101
 
102
  # ============================================================================
@@ -114,6 +103,9 @@ async def json_object(request) -> Dict:
114
  """The request body as a JSON object."""
115
  try:
116
  body = await request.json()
 
 
 
117
  except Exception:
118
  raise BadRequest("Request body must be valid JSON")
119
  if not isinstance(body, dict):
@@ -1108,247 +1100,96 @@ transformer_loader = TransformerModelLoader()
1108
 
1109
 
1110
  # ============================================================================
1111
- # NOTEBOOK EXECUTION (existing Gradio functionality)
1112
  # ============================================================================
1113
 
1114
- def get_available_notebooks() -> List[str]:
1115
- if not NOTEBOOKS_DIR.exists():
1116
- return []
1117
- return sorted([f.name for f in NOTEBOOKS_DIR.glob("*.ipynb")])
1118
 
 
 
 
 
 
 
 
1119
 
1120
- def read_notebook_content(notebook_name: str) -> str:
1121
- """Read and display notebook content as markdown"""
1122
- notebook_path = NOTEBOOKS_DIR / notebook_name
1123
- if not notebook_path.exists():
1124
- return f"Notebook not found: {notebook_name}"
1125
- try:
1126
- with open(notebook_path, "r") as f:
1127
- nb = json.load(f)
1128
- output = f"# {notebook_name}\n\n"
1129
- for i, cell in enumerate(nb.get("cells", []), 1):
1130
- cell_type = cell.get("cell_type", "code")
1131
- source = "".join(cell.get("source", []))
1132
- if cell_type == "markdown":
1133
- output += f"{source}\n\n"
1134
- else:
1135
- output += f"### Cell {i} (Python)\n```python\n{source}\n```\n\n"
1136
- return output
1137
- except Exception as e:
1138
- return f"Error reading notebook: {str(e)}"
1139
-
1140
-
1141
- def execute_notebook(notebook_name: str, progress=gr.Progress()) -> Tuple[str, str]:
1142
- """Execute a notebook and return output"""
1143
- notebook_path = NOTEBOOKS_DIR / notebook_name
1144
- output_path = NOTEBOOKS_DIR / f"output_{notebook_name}"
1145
- if not notebook_path.exists():
1146
- available = list(NOTEBOOKS_DIR.glob("*.ipynb")) if NOTEBOOKS_DIR.exists() else []
1147
- return f"Error: Notebook not found: {notebook_path}\nAvailable: {available}", ""
1148
- progress(0.1, desc="Starting notebook execution...")
1149
- try:
1150
- cmd = [
1151
- sys.executable, "-m", "nbconvert",
1152
- "--to", "notebook", "--execute",
1153
- "--output", str(output_path.absolute()),
1154
- "--ExecutePreprocessor.timeout=600",
1155
- "--ExecutePreprocessor.kernel_name=python3",
1156
- str(notebook_path.absolute()),
1157
- ]
1158
- progress(0.3, desc="Executing cells...")
1159
- result = subprocess.run(cmd, capture_output=True, text=True, cwd=str(NOTEBOOKS_DIR), timeout=900)
1160
- progress(0.8, desc="Processing output...")
1161
- if result.returncode == 0:
1162
- if output_path.exists():
1163
- with open(output_path, "r") as f:
1164
- executed_nb = json.load(f)
1165
- outputs = []
1166
- for i, cell in enumerate(executed_nb.get("cells", []), 1):
1167
- if cell.get("cell_type") == "code":
1168
- for out in cell.get("outputs", []):
1169
- if "text" in out:
1170
- outputs.append(f"Cell {i}:\n{''.join(out['text'])}")
1171
- elif "data" in out and "text/plain" in out["data"]:
1172
- outputs.append(f"Cell {i}:\n{''.join(out['data']['text/plain'])}")
1173
- progress(1.0, desc="Complete!")
1174
- return "Notebook executed successfully!", "\n\n".join(outputs)
1175
- else:
1176
- return "Notebook executed but output file not found", result.stdout
1177
- else:
1178
- return f"Execution failed:\n{result.stderr}", result.stdout
1179
- except subprocess.TimeoutExpired:
1180
- return "Error: Notebook execution timed out (15 min limit)", ""
1181
- except Exception as e:
1182
- return f"Error executing notebook: {str(e)}", ""
1183
-
1184
-
1185
- def run_notebook_cell(notebook_name: str, cell_number: int) -> str:
1186
- """Execute a single cell from a notebook"""
1187
- notebook_path = NOTEBOOKS_DIR / notebook_name
1188
- if not notebook_path.exists():
1189
- return f"Error: Notebook not found at {notebook_path}"
1190
- try:
1191
- original_cwd = os.getcwd()
1192
- os.chdir(NOTEBOOKS_DIR)
1193
- with open(notebook_path, "r") as f:
1194
- nb = json.load(f)
1195
- cells = [c for c in nb.get("cells", []) if c.get("cell_type") == "code"]
1196
- if cell_number < 1 or cell_number > len(cells):
1197
- os.chdir(original_cwd)
1198
- return f"Error: Cell {cell_number} not found. Available: 1-{len(cells)}"
1199
- cell = cells[int(cell_number) - 1]
1200
- source = "".join(cell.get("source", []))
1201
- import io
1202
- from contextlib import redirect_stdout, redirect_stderr
1203
-
1204
- namespace = {"__name__": "__main__", "__file__": str(notebook_path)}
1205
- stdout_capture = io.StringIO()
1206
- stderr_capture = io.StringIO()
1207
- with redirect_stdout(stdout_capture), redirect_stderr(stderr_capture):
1208
- try:
1209
- exec(source, namespace)
1210
- except Exception as e:
1211
- os.chdir(original_cwd)
1212
- return f"Error: {str(e)}"
1213
- os.chdir(original_cwd)
1214
- output = stdout_capture.getvalue()
1215
- errors = stderr_capture.getvalue()
1216
- result_text = f"### Cell {int(cell_number)} Output:\n"
1217
- if output:
1218
- result_text += f"```\n{output}\n```\n"
1219
- if errors:
1220
- result_text += f"\n**Warnings/Errors:**\n```\n{errors}\n```"
1221
- if not output and not errors:
1222
- result_text += "*(No output)*"
1223
- return result_text
1224
- except Exception as e:
1225
- try:
1226
- os.chdir(original_cwd)
1227
- except Exception:
1228
- pass
1229
- return f"Error: {str(e)}"
1230
 
 
 
 
1231
 
1232
- # ============================================================================
1233
- # MODEL TRAINING (existing Gradio functionality)
1234
- # ============================================================================
1235
 
1236
- class SecurityModelTrainer:
1237
- def __init__(self):
1238
- self.scaler = StandardScaler()
1239
- self.label_encoder = LabelEncoder()
1240
-
1241
- def prepare_data(self, df: pd.DataFrame, target_col: str = "label") -> Tuple:
1242
- if target_col not in df.columns:
1243
- raise ValueError(f"Target column '{target_col}' not found")
1244
- X = df.drop(columns=[target_col])
1245
- y = df[target_col]
1246
- X = X.select_dtypes(include=[np.number]).fillna(0)
1247
- if y.dtype == "object":
1248
- y = self.label_encoder.fit_transform(y)
1249
- X_scaled = self.scaler.fit_transform(X)
1250
- return train_test_split(X_scaled, y, test_size=0.2, random_state=42)
1251
-
1252
- def train_model(self, model_type: str, X_train, y_train):
1253
- if model_type not in MODEL_TYPES:
1254
- raise ValueError(f"Unknown model type: {model_type}")
1255
- model_class = MODEL_TYPES[model_type]
1256
- if model_type == "Isolation Forest (Anomaly)":
1257
- model = model_class(contamination=0.1, random_state=42)
1258
- else:
1259
- model = model_class(random_state=42)
1260
- model.fit(X_train, y_train)
1261
- return model
1262
 
1263
- def evaluate_model(self, model, X_test, y_test) -> Dict:
1264
- y_pred = model.predict(X_test)
1265
- return {
1266
- "accuracy": accuracy_score(y_test, y_pred),
1267
- "f1_score": f1_score(y_test, y_pred, average="weighted", zero_division=0),
1268
- }
1269
 
 
 
 
 
1270
 
1271
- trainer = SecurityModelTrainer()
1272
 
 
 
 
 
 
1273
 
1274
- def train_model_from_data(data_file, model_type: str, task: str, progress=gr.Progress()):
1275
- """Train model from uploaded data"""
1276
- if data_file is None:
1277
- return "Please upload a CSV file", None, None
1278
- progress(0.1, desc="Loading data...")
1279
- try:
1280
- df = pd.read_csv(data_file.name)
1281
- progress(0.3, desc="Preparing data...")
1282
- X_train, X_test, y_train, y_test = trainer.prepare_data(df)
1283
- progress(0.5, desc=f"Training {model_type}...")
1284
- model = trainer.train_model(model_type, X_train, y_train)
1285
- progress(0.8, desc="Evaluating model...")
1286
- metrics = trainer.evaluate_model(model, X_test, y_test)
1287
- model_name = f"{task.lower().replace(' ', '_')}_{model_type.lower().replace(' ', '_')}"
1288
- model_path = MODELS_DIR / f"{model_name}.pkl"
1289
- joblib.dump(model, model_path)
1290
- progress(1.0, desc="Complete!")
1291
- result = f"""
1292
- ## Training Complete!
1293
-
1294
- **Task:** {task}
1295
- **Model:** {model_type}
1296
- **Samples:** {len(df)}
1297
-
1298
- ### Metrics
1299
- - Accuracy: {metrics['accuracy']:.4f}
1300
- - F1 Score: {metrics['f1_score']:.4f}
1301
-
1302
- **Model saved to:** {model_path}
1303
- """
1304
- return result, str(model_path), json.dumps(metrics, indent=2)
1305
- except Exception as e:
1306
- return f"Error: {str(e)}", None, None
1307
 
 
 
 
 
1308
 
1309
- def run_inference(model_file, features_text: str):
1310
- """Run inference with a trained model"""
1311
- if model_file is None:
1312
- return "Please upload a model file"
1313
- try:
1314
- model = joblib.load(model_file.name)
1315
- features = json.loads(features_text)
1316
- X = np.array([list(features.values())])
1317
- prediction = model.predict(X)[0]
1318
- result = {"prediction": int(prediction), "features_used": len(features)}
1319
- if hasattr(model, "predict_proba"):
1320
- proba = model.predict_proba(X)[0]
1321
- result["confidence"] = float(max(proba))
1322
- result["probabilities"] = {str(i): float(p) for i, p in enumerate(proba)}
1323
- return json.dumps(result, indent=2)
1324
- except Exception as e:
1325
- return f"Error: {str(e)}"
1326
-
1327
-
1328
- def list_trained_models():
1329
- models = list(MODELS_DIR.glob("*.pkl"))
1330
- if not models:
1331
- return "No trained models found"
1332
- output = "## Trained Models\n\n"
1333
- for model_path in models:
1334
- size_kb = model_path.stat().st_size / 1024
1335
- output += f"- **{model_path.name}** ({size_kb:.1f} KB)\n"
1336
- return output
1337
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1338
 
1339
- # ============================================================================
1340
- # FASTAPI APP (REST endpoints for Heroku backend mlService.js)
1341
- # ============================================================================
 
 
 
1342
 
1343
- api = FastAPI(title="CyberForge AI API", version="1.0.0")
1344
 
1345
- api.add_middleware(
1346
- CORSMiddleware,
1347
- allow_origins=["*"],
1348
- allow_credentials=True,
1349
- allow_methods=["*"],
1350
- allow_headers=["*"],
1351
- )
1352
 
1353
 
1354
  @api.exception_handler(BadRequest)
@@ -1450,7 +1291,8 @@ async def api_health():
1450
  "popular_domains_loaded": popular_domains_loaded(),
1451
  "transformers": transformer_loader.transformers_available,
1452
  "transformer_models_loaded": list(transformer_loader.pipelines.keys()),
1453
- "gradio_ui": True,
 
1454
  },
1455
  "version": "2.0.0",
1456
  "new_endpoints": [
@@ -2197,147 +2039,30 @@ async def api_process_dataset(request: Request):
2197
 
2198
 
2199
  # ============================================================================
2200
- # GRADIO INTERFACE (UI tabs – notebooks, training, inference, models, API)
2201
  # ============================================================================
2202
 
2203
- def create_interface():
2204
- with gr.Blocks(title="CyberForge AI") as demo:
2205
- gr.Markdown("""
2206
- # 🔐 CyberForge AI - ML Training Platform
2207
-
2208
- Train cybersecurity ML models and run Jupyter notebooks on Hugging Face.
2209
- """)
2210
-
2211
- with gr.Tabs():
2212
- # ============ NOTEBOOKS TAB ============
2213
- with gr.TabItem("📓 Notebooks"):
2214
- gr.Markdown("### Run ML Pipeline Notebooks\nExecute the CyberForge ML notebooks directly in the cloud.")
2215
- with gr.Row():
2216
- with gr.Column(scale=1):
2217
- notebook_dropdown = gr.Dropdown(
2218
- choices=get_available_notebooks(),
2219
- label="Select Notebook",
2220
- value=get_available_notebooks()[0] if get_available_notebooks() else None,
2221
- )
2222
- refresh_btn = gr.Button("🔄 Refresh List")
2223
- view_btn = gr.Button("👁 View Content", variant="secondary")
2224
- execute_btn = gr.Button("▶ Execute Notebook", variant="primary")
2225
- gr.Markdown("### Run Single Cell")
2226
- cell_number = gr.Number(label="Cell Number", value=1, minimum=1)
2227
- run_cell_btn = gr.Button("Run Cell")
2228
- with gr.Column(scale=2):
2229
- notebook_status = gr.Markdown("Select a notebook to view or execute.")
2230
- notebook_output = gr.Markdown("", label="Output")
2231
-
2232
- def refresh_notebooks():
2233
- notebooks = get_available_notebooks()
2234
- return gr.update(choices=notebooks, value=notebooks[0] if notebooks else None)
2235
-
2236
- refresh_btn.click(refresh_notebooks, outputs=notebook_dropdown)
2237
- view_btn.click(read_notebook_content, inputs=notebook_dropdown, outputs=notebook_output)
2238
- execute_btn.click(execute_notebook, inputs=notebook_dropdown, outputs=[notebook_status, notebook_output])
2239
- run_cell_btn.click(run_notebook_cell, inputs=[notebook_dropdown, cell_number], outputs=notebook_output)
2240
-
2241
- # ============ TRAIN MODEL TAB ============
2242
- with gr.TabItem("🎯 Train Model"):
2243
- gr.Markdown("### Train a Security ML Model\nUpload your dataset and train a model for threat detection.")
2244
- with gr.Row():
2245
- with gr.Column():
2246
- task_dropdown = gr.Dropdown(choices=SECURITY_TASKS, label="Security Task", value="Phishing Detection")
2247
- model_dropdown = gr.Dropdown(choices=list(MODEL_TYPES.keys()), label="Model Type", value="Random Forest")
2248
- data_upload = gr.File(label="Upload Training Data (CSV)", file_types=[".csv"])
2249
- train_btn = gr.Button("🚀 Train Model", variant="primary")
2250
- with gr.Column():
2251
- train_output = gr.Markdown("Upload data and click Train to begin.")
2252
- model_path_output = gr.Textbox(label="Model Path", visible=False)
2253
- metrics_output = gr.Textbox(label="Metrics JSON", visible=False)
2254
- train_btn.click(train_model_from_data, inputs=[data_upload, model_dropdown, task_dropdown], outputs=[train_output, model_path_output, metrics_output])
2255
-
2256
- # ============ INFERENCE TAB ============
2257
- with gr.TabItem("🔍 Inference"):
2258
- gr.Markdown("### Run Model Inference\nLoad a trained model and make predictions.")
2259
- with gr.Row():
2260
- with gr.Column():
2261
- model_upload = gr.File(label="Upload Model (.pkl)")
2262
- features_input = gr.Textbox(label="Features (JSON)", value='{"url_length": 50, "has_https": 1, "digit_count": 5}', lines=5)
2263
- predict_btn = gr.Button("🎯 Predict", variant="primary")
2264
- with gr.Column():
2265
- prediction_output = gr.Textbox(label="Prediction Result", lines=10)
2266
- predict_btn.click(run_inference, inputs=[model_upload, features_input], outputs=prediction_output)
2267
-
2268
- # ============ MODELS TAB ============
2269
- with gr.TabItem("📦 Models"):
2270
- gr.Markdown("### Trained Models")
2271
- models_list = gr.Markdown(list_trained_models())
2272
- refresh_models_btn = gr.Button("🔄 Refresh")
2273
- refresh_models_btn.click(list_trained_models, outputs=models_list)
2274
-
2275
- # ============ API TAB ============
2276
- with gr.TabItem("🔌 API"):
2277
- gr.Markdown("""
2278
- ## API Integration — v2.0
2279
-
2280
- ### Core Endpoints (stable, used by Heroku backend)
2281
-
2282
- | Method | Endpoint | Description |
2283
- |--------|----------|-------------|
2284
- | GET | `/health` | Health check — version, models, services |
2285
- | POST | `/analyze` | AI chat / general analysis (LLM or ML fallback) |
2286
- | POST | `/analyze-url` | URL threat analysis — 4 ML models + risk factors |
2287
- | POST | `/scan-threats` | Deep threat scan (LLM or ML fallback) |
2288
- | POST | `/api/insights/generate` | AI-generated security insights |
2289
- | POST | `/api/models/predict` | Direct ML model prediction |
2290
- | GET | `/models` | List loaded models with source info |
2291
-
2292
- ### v2 Endpoints (new — for distributed agents + AI Deep Scan)
2293
-
2294
- | Method | Endpoint | Description |
2295
- |--------|----------|-------------|
2296
- | POST | `/api/v2/batch-analyze` | Batch analyze up to 20 URLs (for multi-agent) |
2297
- | GET | `/api/v2/explain/{model_name}` | Feature importances / explainability |
2298
- | POST | `/api/v2/url-classify` | BERT-based URL phishing classifier |
2299
- | POST | `/api/v2/dga-detect` | DGA domain detection (entropy heuristic) |
2300
- | POST | `/api/v2/ioc-scan` | Multi-IOC scanner: URL + domain + DGA + BERT |
2301
- | POST | `/api/v2/url-enrich` | Enriched analysis: ML + BERT + DGA + explainability |
2302
- | POST | `/api/v2/chat` | Security chatbot — context-grounded, RAG-lite, machine→human translation |
2303
- | POST | `/api/v2/security-chat` | Security LLM Q&A (open-source model running in this Space) |
2304
- | GET | `/api/v2/status` | Transformer model load status |
2305
-
2306
- ### Quick Examples
2307
-
2308
- ```bash
2309
- # Batch analyze URLs (for distributed agents)
2310
- curl -X POST https://che237-cyberforge.hf.space/api/v2/batch-analyze \\
2311
- -H "Content-Type: application/json" \\
2312
- -d '{"urls": ["https://example.com", "http://evil.xyz/login?id=1"]}'
2313
-
2314
- # Enriched URL analysis (Deep Scan mode)
2315
- curl -X POST https://che237-cyberforge.hf.space/api/v2/url-enrich \\
2316
- -H "Content-Type: application/json" \\
2317
- -d '{"url": "https://example.com"}'
2318
-
2319
- # Feature explainability
2320
- curl https://che237-cyberforge.hf.space/api/v2/explain/phishing_detection
2321
-
2322
- # IOC multi-field scan
2323
- curl -X POST https://che237-cyberforge.hf.space/api/v2/ioc-scan \\
2324
- -H "Content-Type: application/json" \\
2325
- -d '{"url": "http://192.168.1.1/login", "domain": "192.168.1.1"}'
2326
-
2327
- # Chatbot (Wave 2 extension point)
2328
- curl -X POST https://che237-cyberforge.hf.space/api/v2/chat \\
2329
- -H "Content-Type: application/json" \\
2330
- -d '{"message": "Is this URL suspicious: http://login.g00gle.com.xyz/verify"}'
2331
- ```
2332
- """)
2333
-
2334
- gr.Markdown("---\n**CyberForge AI** | [GitHub](https://github.com/Che237/cyberforge) | [Datasets](https://huggingface.co/datasets/Che237/cyberforge-datasets)")
2335
-
2336
- return demo
2337
 
2338
 
2339
  # ============================================================================
2340
- # MAIN – Mount Gradio on FastAPI
2341
  # ============================================================================
2342
 
2343
  # Initialize services at startup
@@ -2347,9 +2072,7 @@ ml_loader.initialize()
2347
  threading.Thread(target=local_llm.load, name="local-llm-loader", daemon=True).start()
2348
  logger.info("✅ Services initialized")
2349
 
2350
- # Create Gradio app and mount it on FastAPI
2351
- demo = create_interface()
2352
- app = gr.mount_gradio_app(api, demo, path="/")
2353
 
2354
  if __name__ == "__main__":
2355
  import uvicorn
 
1
  """
2
+ CyberForge AI - ML inference service
3
+ Hugging Face Spaces deployment: FastAPI REST endpoints for the CyberForge
4
+ backend. Every route except / and /health needs the service key
5
+ (ML_SERVICE_KEY), which only the backend holds; the two training routes need
6
+ the Space owner's Hugging Face token instead. Nothing here runs uploaded code,
7
+ notebooks or uploaded models.
8
  """
9
 
10
+ import hmac
 
11
  import numpy as np
12
  import json
13
  import os
 
27
  from url_features import (extract_url_features, popular_domains_loaded, rescale_score,
28
  set_popular_domains, split_url)
29
 
30
+ # Models are loaded only from the model repository (ml_loader below), never
31
+ # from a caller: loading a joblib/pickle file can run code.
 
 
 
 
32
  import joblib
33
 
34
  # Hugging Face Hub
 
37
  # FastAPI for REST endpoints
38
  from fastapi import FastAPI, Request
39
  from fastapi.middleware.cors import CORSMiddleware
40
+ from fastapi.responses import HTMLResponse, JSONResponse
41
  from starlette.concurrency import run_in_threadpool
42
 
43
  logging.basicConfig(level=logging.INFO)
 
52
  MODELS_DIR.mkdir(exist_ok=True)
53
  DATASETS_DIR = APP_DIR / "datasets"
54
  DATASETS_DIR.mkdir(exist_ok=True)
 
55
  KNOWLEDGE_BASE_DIR = APP_DIR / "knowledge_base"
56
  KNOWLEDGE_BASE_DIR.mkdir(exist_ok=True)
57
  TRAINING_DATA_DIR = APP_DIR / "training_data"
 
76
  LOCAL_LLM_QUEUE_TIMEOUT_S = float(os.environ.get("LOCAL_LLM_QUEUE_TIMEOUT_S", "5"))
77
 
78
  logger.info(f"APP_DIR: {APP_DIR}")
79
+ # Calls to every route except / and /health must carry this key in the
80
+ # X-CyberForge-Service-Key header. It is a Space secret; the backend holds the
81
+ # same value. Without it set, those routes answer 503 rather than run open.
82
+ ML_SERVICE_KEY = os.environ.get("ML_SERVICE_KEY", "").strip()
83
+ SERVICE_KEY_HEADER = "x-cyberforge-service-key"
84
+ # The largest request body accepted, in bytes. Larger ones get HTTP 413.
85
+ ML_MAX_BODY_BYTES = int(os.environ.get("ML_MAX_BODY_BYTES", str(256 * 1024)))
86
 
87
+ logger.info(f"APP_DIR: {APP_DIR}")
88
+ logger.info(f"Service key {'set' if ML_SERVICE_KEY else 'NOT set: inference routes will answer 503'}")
 
 
 
 
 
 
 
 
 
 
 
 
89
 
90
 
91
  # ============================================================================
 
103
  """The request body as a JSON object."""
104
  try:
105
  body = await request.json()
106
+ except BodyTooLarge:
107
+ # Raised by ServiceGate while the body streams in; it answers 413.
108
+ raise
109
  except Exception:
110
  raise BadRequest("Request body must be valid JSON")
111
  if not isinstance(body, dict):
 
1100
 
1101
 
1102
  # ============================================================================
1103
+ # FASTAPI APP (REST endpoints for Heroku backend mlService.js)
1104
  # ============================================================================
1105
 
1106
+ api = FastAPI(title="CyberForge AI API", version="1.0.0")
 
 
 
1107
 
1108
+ api.add_middleware(
1109
+ CORSMiddleware,
1110
+ allow_origins=["*"],
1111
+ allow_credentials=True,
1112
+ allow_methods=["*"],
1113
+ allow_headers=["*"],
1114
+ )
1115
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1116
 
1117
+ PUBLIC_PATHS = {"/", "/health"}
1118
+ # These check the owner's Hugging Face token themselves (owner_token below).
1119
+ OWNER_PATHS = {"/api/admin/train", "/api/admin/train/status"}
1120
 
 
 
 
1121
 
1122
+ class BodyTooLarge(Exception):
1123
+ pass
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1124
 
 
 
 
 
 
 
1125
 
1126
+ def service_key_valid(presented: Optional[bytes]) -> bool:
1127
+ if not ML_SERVICE_KEY or not presented:
1128
+ return False
1129
+ return hmac.compare_digest(presented, ML_SERVICE_KEY.encode("utf-8"))
1130
 
 
1131
 
1132
+ async def _reply(send, status: int, detail: str, code: str):
1133
+ body = json.dumps({"detail": detail, "code": code}).encode("utf-8")
1134
+ await send({"type": "http.response.start", "status": status,
1135
+ "headers": [(b"content-type", b"application/json"), (b"content-length", str(len(body)).encode())]})
1136
+ await send({"type": "http.response.body", "body": body})
1137
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1138
 
1139
+ class ServiceGate:
1140
+ """Outermost middleware. It refuses a call without the service key before
1141
+ any route runs, and a body over ML_MAX_BODY_BYTES whether or not it states
1142
+ its length."""
1143
 
1144
+ def __init__(self, app):
1145
+ self.app = app
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1146
 
1147
+ async def __call__(self, scope, receive, send):
1148
+ if scope["type"] != "http":
1149
+ return await self.app(scope, receive, send)
1150
+ path = scope.get("path", "")
1151
+ headers = dict(scope.get("headers") or [])
1152
+ # A CORS preflight carries no credentials; the CORS middleware answers it.
1153
+ if scope.get("method") != "OPTIONS" and path not in PUBLIC_PATHS and path not in OWNER_PATHS:
1154
+ if not ML_SERVICE_KEY:
1155
+ return await _reply(send, 503, "This service has no service key configured", "SERVICE_KEY_NOT_CONFIGURED")
1156
+ if not service_key_valid(headers.get(SERVICE_KEY_HEADER.encode())):
1157
+ return await _reply(send, 401, "A valid service key is required", "SERVICE_KEY_REQUIRED")
1158
+ stated = headers.get(b"content-length")
1159
+ if stated is not None:
1160
+ try:
1161
+ if int(stated) > ML_MAX_BODY_BYTES:
1162
+ return await _reply(send, 413, f"Request body is over {ML_MAX_BODY_BYTES} bytes", "BODY_TOO_LARGE")
1163
+ except ValueError:
1164
+ return await _reply(send, 400, "Content-Length must be a number", "BAD_CONTENT_LENGTH")
1165
+ received = 0
1166
+ started = False
1167
+
1168
+ async def bounded_receive():
1169
+ nonlocal received
1170
+ message = await receive()
1171
+ if message["type"] == "http.request":
1172
+ received += len(message.get("body", b""))
1173
+ if received > ML_MAX_BODY_BYTES:
1174
+ raise BodyTooLarge()
1175
+ return message
1176
+
1177
+ async def tracked_send(message):
1178
+ nonlocal started
1179
+ if message["type"] == "http.response.start":
1180
+ started = True
1181
+ await send(message)
1182
 
1183
+ try:
1184
+ await self.app(scope, bounded_receive, tracked_send)
1185
+ except BodyTooLarge:
1186
+ if started:
1187
+ raise
1188
+ await _reply(send, 413, f"Request body is over {ML_MAX_BODY_BYTES} bytes", "BODY_TOO_LARGE")
1189
 
 
1190
 
1191
+ # Added last, so it runs first: before CORS and every route.
1192
+ api.add_middleware(ServiceGate)
 
 
 
 
 
1193
 
1194
 
1195
  @api.exception_handler(BadRequest)
 
1291
  "popular_domains_loaded": popular_domains_loaded(),
1292
  "transformers": transformer_loader.transformers_available,
1293
  "transformer_models_loaded": list(transformer_loader.pipelines.keys()),
1294
+ "service_key": "required" if ML_SERVICE_KEY else "not_configured",
1295
+ "max_body_bytes": ML_MAX_BODY_BYTES,
1296
  },
1297
  "version": "2.0.0",
1298
  "new_endpoints": [
 
2039
 
2040
 
2041
  # ============================================================================
2042
+ # LANDING PAGE (static; the service has no interactive UI)
2043
  # ============================================================================
2044
 
2045
+ LANDING_HTML = """<!doctype html>
2046
+ <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
2047
+ <title>CyberForge ML service</title>
2048
+ <style>body{font-family:system-ui,sans-serif;max-width:46rem;margin:2rem auto;padding:0 1rem;line-height:1.5;color:#1b1f24;background:#fff}
2049
+ code{background:#f1f3f5;padding:0 .25rem;border-radius:3px}</style></head>
2050
+ <body><h1>CyberForge ML service</h1>
2051
+ <p>This Space serves the detection models and the security chat model to the CyberForge backend.
2052
+ It has no interactive interface: it does not run notebooks, train on uploaded data, or load uploaded models.</p>
2053
+ <p><code>GET /health</code> is public. Every other route needs the service key held by the backend,
2054
+ and model training needs the Space owner's token.</p>
2055
+ <p>Source: <a href="https://github.com/Che237/cyberforge">github.com/Che237/cyberforge</a></p>
2056
+ </body></html>"""
2057
+
2058
+
2059
+ @api.get("/", response_class=HTMLResponse)
2060
+ async def landing():
2061
+ return LANDING_HTML
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2062
 
2063
 
2064
  # ============================================================================
2065
+ # MAIN
2066
  # ============================================================================
2067
 
2068
  # Initialize services at startup
 
2072
  threading.Thread(target=local_llm.load, name="local-llm-loader", daemon=True).start()
2073
  logger.info("✅ Services initialized")
2074
 
2075
+ app = api
 
 
2076
 
2077
  if __name__ == "__main__":
2078
  import uvicorn
requirements.txt CHANGED
@@ -1,5 +1,5 @@
1
  # CyberForge AI - HuggingFace Space
2
- # Combined Gradio UI + FastAPI REST API
3
 
4
  # Core ML
5
  # Pinned: the .pkl files in Che237/cyberforge-models were saved with 1.8.0 and the
@@ -14,9 +14,6 @@ torch>=2.2.0
14
  transformers>=4.40.0
15
  tokenizers>=0.19.0
16
 
17
- # Gradio UI
18
- gradio>=5.20.0
19
-
20
  # FastAPI REST endpoints (for Heroku backend)
21
  fastapi>=0.100.0
22
  uvicorn[standard]>=0.22.0
@@ -35,14 +32,6 @@ tldextract>=5.1.0
35
  --only-binary llama-cpp-python
36
  llama-cpp-python==0.3.36
37
 
38
- # Notebook execution
39
- nbformat>=5.10.0
40
- nbconvert>=7.16.0
41
- nest_asyncio>=1.6.0
42
-
43
- # HTTP client for notebooks
44
- httpx>=0.27.0
45
-
46
  # Data handling
47
  datasets>=2.20.0
48
  pyarrow>=17.0.0
 
1
  # CyberForge AI - HuggingFace Space
2
+ # FastAPI REST API only. There is no Gradio UI and no notebook runner (V2-ML01).
3
 
4
  # Core ML
5
  # Pinned: the .pkl files in Che237/cyberforge-models were saved with 1.8.0 and the
 
14
  transformers>=4.40.0
15
  tokenizers>=0.19.0
16
 
 
 
 
17
  # FastAPI REST endpoints (for Heroku backend)
18
  fastapi>=0.100.0
19
  uvicorn[standard]>=0.22.0
 
32
  --only-binary llama-cpp-python
33
  llama-cpp-python==0.3.36
34
 
 
 
 
 
 
 
 
 
35
  # Data handling
36
  datasets>=2.20.0
37
  pyarrow>=17.0.0