Spaces:
Running
Running
Update the CyberForge ML service
Browse files- Dockerfile +3 -5
- app.py +116 -393
- requirements.txt +1 -12
Dockerfile
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
#
|
| 2 |
FROM python:3.11-slim
|
| 3 |
|
| 4 |
WORKDIR /app
|
|
@@ -25,9 +25,7 @@ WORKDIR $HOME/app
|
|
| 25 |
# Copy requirements and install dependencies
|
| 26 |
COPY --chown=user requirements.txt .
|
| 27 |
RUN pip install --no-cache-dir --upgrade pip && \
|
| 28 |
-
pip install --no-cache-dir -r requirements.txt
|
| 29 |
-
pip install --no-cache-dir ipykernel && \
|
| 30 |
-
python -m ipykernel install --user --name python3 --display-name "Python 3"
|
| 31 |
|
| 32 |
# Copy application code
|
| 33 |
COPY --chown=user . .
|
|
@@ -35,5 +33,5 @@ COPY --chown=user . .
|
|
| 35 |
# Expose port
|
| 36 |
EXPOSE 7860
|
| 37 |
|
| 38 |
-
# Run the
|
| 39 |
CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
|
|
|
|
| 1 |
+
# Python 3.11: the version this image has been built and run with
|
| 2 |
FROM python:3.11-slim
|
| 3 |
|
| 4 |
WORKDIR /app
|
|
|
|
| 25 |
# Copy requirements and install dependencies
|
| 26 |
COPY --chown=user requirements.txt .
|
| 27 |
RUN pip install --no-cache-dir --upgrade pip && \
|
| 28 |
+
pip install --no-cache-dir -r requirements.txt
|
|
|
|
|
|
|
| 29 |
|
| 30 |
# Copy application code
|
| 31 |
COPY --chown=user . .
|
|
|
|
| 33 |
# Expose port
|
| 34 |
EXPOSE 7860
|
| 35 |
|
| 36 |
+
# Run the API via uvicorn (FastAPI only, no web interface)
|
| 37 |
CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
|
app.py
CHANGED
|
@@ -1,12 +1,13 @@
|
|
| 1 |
"""
|
| 2 |
-
CyberForge AI - ML
|
| 3 |
-
Hugging Face Spaces deployment
|
| 4 |
-
|
| 5 |
-
|
|
|
|
|
|
|
| 6 |
"""
|
| 7 |
|
| 8 |
-
import
|
| 9 |
-
import pandas as pd
|
| 10 |
import numpy as np
|
| 11 |
import json
|
| 12 |
import os
|
|
@@ -26,12 +27,8 @@ from urllib.parse import urlparse
|
|
| 26 |
from url_features import (extract_url_features, popular_domains_loaded, rescale_score,
|
| 27 |
set_popular_domains, split_url)
|
| 28 |
|
| 29 |
-
#
|
| 30 |
-
from
|
| 31 |
-
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, IsolationForest
|
| 32 |
-
from sklearn.linear_model import LogisticRegression
|
| 33 |
-
from sklearn.preprocessing import StandardScaler, LabelEncoder
|
| 34 |
-
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score, f1_score
|
| 35 |
import joblib
|
| 36 |
|
| 37 |
# Hugging Face Hub
|
|
@@ -40,7 +37,7 @@ from huggingface_hub import HfApi, hf_hub_download, upload_file
|
|
| 40 |
# FastAPI for REST endpoints
|
| 41 |
from fastapi import FastAPI, Request
|
| 42 |
from fastapi.middleware.cors import CORSMiddleware
|
| 43 |
-
from fastapi.responses import JSONResponse
|
| 44 |
from starlette.concurrency import run_in_threadpool
|
| 45 |
|
| 46 |
logging.basicConfig(level=logging.INFO)
|
|
@@ -55,7 +52,6 @@ MODELS_DIR = APP_DIR / "trained_models"
|
|
| 55 |
MODELS_DIR.mkdir(exist_ok=True)
|
| 56 |
DATASETS_DIR = APP_DIR / "datasets"
|
| 57 |
DATASETS_DIR.mkdir(exist_ok=True)
|
| 58 |
-
NOTEBOOKS_DIR = APP_DIR / "notebooks"
|
| 59 |
KNOWLEDGE_BASE_DIR = APP_DIR / "knowledge_base"
|
| 60 |
KNOWLEDGE_BASE_DIR.mkdir(exist_ok=True)
|
| 61 |
TRAINING_DATA_DIR = APP_DIR / "training_data"
|
|
@@ -80,23 +76,16 @@ LOCAL_LLM_TIME_BUDGET_S = float(os.environ.get("LOCAL_LLM_TIME_BUDGET_S", "17"))
|
|
| 80 |
LOCAL_LLM_QUEUE_TIMEOUT_S = float(os.environ.get("LOCAL_LLM_QUEUE_TIMEOUT_S", "5"))
|
| 81 |
|
| 82 |
logger.info(f"APP_DIR: {APP_DIR}")
|
| 83 |
-
|
| 84 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 85 |
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
"Random Forest": RandomForestClassifier,
|
| 89 |
-
"Gradient Boosting": GradientBoostingClassifier,
|
| 90 |
-
"Logistic Regression": LogisticRegression,
|
| 91 |
-
"Isolation Forest (Anomaly)": IsolationForest,
|
| 92 |
-
}
|
| 93 |
-
|
| 94 |
-
SECURITY_TASKS = [
|
| 95 |
-
"Malware Detection", "Phishing Detection", "Network Intrusion Detection",
|
| 96 |
-
"Anomaly Detection", "Botnet Detection", "Web Attack Detection",
|
| 97 |
-
"Spam Detection", "Vulnerability Assessment", "DNS Tunneling Detection",
|
| 98 |
-
"Cryptomining Detection",
|
| 99 |
-
]
|
| 100 |
|
| 101 |
|
| 102 |
# ============================================================================
|
|
@@ -114,6 +103,9 @@ async def json_object(request) -> Dict:
|
|
| 114 |
"""The request body as a JSON object."""
|
| 115 |
try:
|
| 116 |
body = await request.json()
|
|
|
|
|
|
|
|
|
|
| 117 |
except Exception:
|
| 118 |
raise BadRequest("Request body must be valid JSON")
|
| 119 |
if not isinstance(body, dict):
|
|
@@ -1108,247 +1100,96 @@ transformer_loader = TransformerModelLoader()
|
|
| 1108 |
|
| 1109 |
|
| 1110 |
# ============================================================================
|
| 1111 |
-
#
|
| 1112 |
# ============================================================================
|
| 1113 |
|
| 1114 |
-
|
| 1115 |
-
if not NOTEBOOKS_DIR.exists():
|
| 1116 |
-
return []
|
| 1117 |
-
return sorted([f.name for f in NOTEBOOKS_DIR.glob("*.ipynb")])
|
| 1118 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1119 |
|
| 1120 |
-
def read_notebook_content(notebook_name: str) -> str:
|
| 1121 |
-
"""Read and display notebook content as markdown"""
|
| 1122 |
-
notebook_path = NOTEBOOKS_DIR / notebook_name
|
| 1123 |
-
if not notebook_path.exists():
|
| 1124 |
-
return f"Notebook not found: {notebook_name}"
|
| 1125 |
-
try:
|
| 1126 |
-
with open(notebook_path, "r") as f:
|
| 1127 |
-
nb = json.load(f)
|
| 1128 |
-
output = f"# {notebook_name}\n\n"
|
| 1129 |
-
for i, cell in enumerate(nb.get("cells", []), 1):
|
| 1130 |
-
cell_type = cell.get("cell_type", "code")
|
| 1131 |
-
source = "".join(cell.get("source", []))
|
| 1132 |
-
if cell_type == "markdown":
|
| 1133 |
-
output += f"{source}\n\n"
|
| 1134 |
-
else:
|
| 1135 |
-
output += f"### Cell {i} (Python)\n```python\n{source}\n```\n\n"
|
| 1136 |
-
return output
|
| 1137 |
-
except Exception as e:
|
| 1138 |
-
return f"Error reading notebook: {str(e)}"
|
| 1139 |
-
|
| 1140 |
-
|
| 1141 |
-
def execute_notebook(notebook_name: str, progress=gr.Progress()) -> Tuple[str, str]:
|
| 1142 |
-
"""Execute a notebook and return output"""
|
| 1143 |
-
notebook_path = NOTEBOOKS_DIR / notebook_name
|
| 1144 |
-
output_path = NOTEBOOKS_DIR / f"output_{notebook_name}"
|
| 1145 |
-
if not notebook_path.exists():
|
| 1146 |
-
available = list(NOTEBOOKS_DIR.glob("*.ipynb")) if NOTEBOOKS_DIR.exists() else []
|
| 1147 |
-
return f"Error: Notebook not found: {notebook_path}\nAvailable: {available}", ""
|
| 1148 |
-
progress(0.1, desc="Starting notebook execution...")
|
| 1149 |
-
try:
|
| 1150 |
-
cmd = [
|
| 1151 |
-
sys.executable, "-m", "nbconvert",
|
| 1152 |
-
"--to", "notebook", "--execute",
|
| 1153 |
-
"--output", str(output_path.absolute()),
|
| 1154 |
-
"--ExecutePreprocessor.timeout=600",
|
| 1155 |
-
"--ExecutePreprocessor.kernel_name=python3",
|
| 1156 |
-
str(notebook_path.absolute()),
|
| 1157 |
-
]
|
| 1158 |
-
progress(0.3, desc="Executing cells...")
|
| 1159 |
-
result = subprocess.run(cmd, capture_output=True, text=True, cwd=str(NOTEBOOKS_DIR), timeout=900)
|
| 1160 |
-
progress(0.8, desc="Processing output...")
|
| 1161 |
-
if result.returncode == 0:
|
| 1162 |
-
if output_path.exists():
|
| 1163 |
-
with open(output_path, "r") as f:
|
| 1164 |
-
executed_nb = json.load(f)
|
| 1165 |
-
outputs = []
|
| 1166 |
-
for i, cell in enumerate(executed_nb.get("cells", []), 1):
|
| 1167 |
-
if cell.get("cell_type") == "code":
|
| 1168 |
-
for out in cell.get("outputs", []):
|
| 1169 |
-
if "text" in out:
|
| 1170 |
-
outputs.append(f"Cell {i}:\n{''.join(out['text'])}")
|
| 1171 |
-
elif "data" in out and "text/plain" in out["data"]:
|
| 1172 |
-
outputs.append(f"Cell {i}:\n{''.join(out['data']['text/plain'])}")
|
| 1173 |
-
progress(1.0, desc="Complete!")
|
| 1174 |
-
return "Notebook executed successfully!", "\n\n".join(outputs)
|
| 1175 |
-
else:
|
| 1176 |
-
return "Notebook executed but output file not found", result.stdout
|
| 1177 |
-
else:
|
| 1178 |
-
return f"Execution failed:\n{result.stderr}", result.stdout
|
| 1179 |
-
except subprocess.TimeoutExpired:
|
| 1180 |
-
return "Error: Notebook execution timed out (15 min limit)", ""
|
| 1181 |
-
except Exception as e:
|
| 1182 |
-
return f"Error executing notebook: {str(e)}", ""
|
| 1183 |
-
|
| 1184 |
-
|
| 1185 |
-
def run_notebook_cell(notebook_name: str, cell_number: int) -> str:
|
| 1186 |
-
"""Execute a single cell from a notebook"""
|
| 1187 |
-
notebook_path = NOTEBOOKS_DIR / notebook_name
|
| 1188 |
-
if not notebook_path.exists():
|
| 1189 |
-
return f"Error: Notebook not found at {notebook_path}"
|
| 1190 |
-
try:
|
| 1191 |
-
original_cwd = os.getcwd()
|
| 1192 |
-
os.chdir(NOTEBOOKS_DIR)
|
| 1193 |
-
with open(notebook_path, "r") as f:
|
| 1194 |
-
nb = json.load(f)
|
| 1195 |
-
cells = [c for c in nb.get("cells", []) if c.get("cell_type") == "code"]
|
| 1196 |
-
if cell_number < 1 or cell_number > len(cells):
|
| 1197 |
-
os.chdir(original_cwd)
|
| 1198 |
-
return f"Error: Cell {cell_number} not found. Available: 1-{len(cells)}"
|
| 1199 |
-
cell = cells[int(cell_number) - 1]
|
| 1200 |
-
source = "".join(cell.get("source", []))
|
| 1201 |
-
import io
|
| 1202 |
-
from contextlib import redirect_stdout, redirect_stderr
|
| 1203 |
-
|
| 1204 |
-
namespace = {"__name__": "__main__", "__file__": str(notebook_path)}
|
| 1205 |
-
stdout_capture = io.StringIO()
|
| 1206 |
-
stderr_capture = io.StringIO()
|
| 1207 |
-
with redirect_stdout(stdout_capture), redirect_stderr(stderr_capture):
|
| 1208 |
-
try:
|
| 1209 |
-
exec(source, namespace)
|
| 1210 |
-
except Exception as e:
|
| 1211 |
-
os.chdir(original_cwd)
|
| 1212 |
-
return f"Error: {str(e)}"
|
| 1213 |
-
os.chdir(original_cwd)
|
| 1214 |
-
output = stdout_capture.getvalue()
|
| 1215 |
-
errors = stderr_capture.getvalue()
|
| 1216 |
-
result_text = f"### Cell {int(cell_number)} Output:\n"
|
| 1217 |
-
if output:
|
| 1218 |
-
result_text += f"```\n{output}\n```\n"
|
| 1219 |
-
if errors:
|
| 1220 |
-
result_text += f"\n**Warnings/Errors:**\n```\n{errors}\n```"
|
| 1221 |
-
if not output and not errors:
|
| 1222 |
-
result_text += "*(No output)*"
|
| 1223 |
-
return result_text
|
| 1224 |
-
except Exception as e:
|
| 1225 |
-
try:
|
| 1226 |
-
os.chdir(original_cwd)
|
| 1227 |
-
except Exception:
|
| 1228 |
-
pass
|
| 1229 |
-
return f"Error: {str(e)}"
|
| 1230 |
|
|
|
|
|
|
|
|
|
|
| 1231 |
|
| 1232 |
-
# ============================================================================
|
| 1233 |
-
# MODEL TRAINING (existing Gradio functionality)
|
| 1234 |
-
# ============================================================================
|
| 1235 |
|
| 1236 |
-
class
|
| 1237 |
-
|
| 1238 |
-
self.scaler = StandardScaler()
|
| 1239 |
-
self.label_encoder = LabelEncoder()
|
| 1240 |
-
|
| 1241 |
-
def prepare_data(self, df: pd.DataFrame, target_col: str = "label") -> Tuple:
|
| 1242 |
-
if target_col not in df.columns:
|
| 1243 |
-
raise ValueError(f"Target column '{target_col}' not found")
|
| 1244 |
-
X = df.drop(columns=[target_col])
|
| 1245 |
-
y = df[target_col]
|
| 1246 |
-
X = X.select_dtypes(include=[np.number]).fillna(0)
|
| 1247 |
-
if y.dtype == "object":
|
| 1248 |
-
y = self.label_encoder.fit_transform(y)
|
| 1249 |
-
X_scaled = self.scaler.fit_transform(X)
|
| 1250 |
-
return train_test_split(X_scaled, y, test_size=0.2, random_state=42)
|
| 1251 |
-
|
| 1252 |
-
def train_model(self, model_type: str, X_train, y_train):
|
| 1253 |
-
if model_type not in MODEL_TYPES:
|
| 1254 |
-
raise ValueError(f"Unknown model type: {model_type}")
|
| 1255 |
-
model_class = MODEL_TYPES[model_type]
|
| 1256 |
-
if model_type == "Isolation Forest (Anomaly)":
|
| 1257 |
-
model = model_class(contamination=0.1, random_state=42)
|
| 1258 |
-
else:
|
| 1259 |
-
model = model_class(random_state=42)
|
| 1260 |
-
model.fit(X_train, y_train)
|
| 1261 |
-
return model
|
| 1262 |
|
| 1263 |
-
def evaluate_model(self, model, X_test, y_test) -> Dict:
|
| 1264 |
-
y_pred = model.predict(X_test)
|
| 1265 |
-
return {
|
| 1266 |
-
"accuracy": accuracy_score(y_test, y_pred),
|
| 1267 |
-
"f1_score": f1_score(y_test, y_pred, average="weighted", zero_division=0),
|
| 1268 |
-
}
|
| 1269 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1270 |
|
| 1271 |
-
trainer = SecurityModelTrainer()
|
| 1272 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1273 |
|
| 1274 |
-
def train_model_from_data(data_file, model_type: str, task: str, progress=gr.Progress()):
|
| 1275 |
-
"""Train model from uploaded data"""
|
| 1276 |
-
if data_file is None:
|
| 1277 |
-
return "Please upload a CSV file", None, None
|
| 1278 |
-
progress(0.1, desc="Loading data...")
|
| 1279 |
-
try:
|
| 1280 |
-
df = pd.read_csv(data_file.name)
|
| 1281 |
-
progress(0.3, desc="Preparing data...")
|
| 1282 |
-
X_train, X_test, y_train, y_test = trainer.prepare_data(df)
|
| 1283 |
-
progress(0.5, desc=f"Training {model_type}...")
|
| 1284 |
-
model = trainer.train_model(model_type, X_train, y_train)
|
| 1285 |
-
progress(0.8, desc="Evaluating model...")
|
| 1286 |
-
metrics = trainer.evaluate_model(model, X_test, y_test)
|
| 1287 |
-
model_name = f"{task.lower().replace(' ', '_')}_{model_type.lower().replace(' ', '_')}"
|
| 1288 |
-
model_path = MODELS_DIR / f"{model_name}.pkl"
|
| 1289 |
-
joblib.dump(model, model_path)
|
| 1290 |
-
progress(1.0, desc="Complete!")
|
| 1291 |
-
result = f"""
|
| 1292 |
-
## Training Complete!
|
| 1293 |
-
|
| 1294 |
-
**Task:** {task}
|
| 1295 |
-
**Model:** {model_type}
|
| 1296 |
-
**Samples:** {len(df)}
|
| 1297 |
-
|
| 1298 |
-
### Metrics
|
| 1299 |
-
- Accuracy: {metrics['accuracy']:.4f}
|
| 1300 |
-
- F1 Score: {metrics['f1_score']:.4f}
|
| 1301 |
-
|
| 1302 |
-
**Model saved to:** {model_path}
|
| 1303 |
-
"""
|
| 1304 |
-
return result, str(model_path), json.dumps(metrics, indent=2)
|
| 1305 |
-
except Exception as e:
|
| 1306 |
-
return f"Error: {str(e)}", None, None
|
| 1307 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1308 |
|
| 1309 |
-
def
|
| 1310 |
-
|
| 1311 |
-
if model_file is None:
|
| 1312 |
-
return "Please upload a model file"
|
| 1313 |
-
try:
|
| 1314 |
-
model = joblib.load(model_file.name)
|
| 1315 |
-
features = json.loads(features_text)
|
| 1316 |
-
X = np.array([list(features.values())])
|
| 1317 |
-
prediction = model.predict(X)[0]
|
| 1318 |
-
result = {"prediction": int(prediction), "features_used": len(features)}
|
| 1319 |
-
if hasattr(model, "predict_proba"):
|
| 1320 |
-
proba = model.predict_proba(X)[0]
|
| 1321 |
-
result["confidence"] = float(max(proba))
|
| 1322 |
-
result["probabilities"] = {str(i): float(p) for i, p in enumerate(proba)}
|
| 1323 |
-
return json.dumps(result, indent=2)
|
| 1324 |
-
except Exception as e:
|
| 1325 |
-
return f"Error: {str(e)}"
|
| 1326 |
-
|
| 1327 |
-
|
| 1328 |
-
def list_trained_models():
|
| 1329 |
-
models = list(MODELS_DIR.glob("*.pkl"))
|
| 1330 |
-
if not models:
|
| 1331 |
-
return "No trained models found"
|
| 1332 |
-
output = "## Trained Models\n\n"
|
| 1333 |
-
for model_path in models:
|
| 1334 |
-
size_kb = model_path.stat().st_size / 1024
|
| 1335 |
-
output += f"- **{model_path.name}** ({size_kb:.1f} KB)\n"
|
| 1336 |
-
return output
|
| 1337 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1338 |
|
| 1339 |
-
|
| 1340 |
-
|
| 1341 |
-
|
|
|
|
|
|
|
|
|
|
| 1342 |
|
| 1343 |
-
api = FastAPI(title="CyberForge AI API", version="1.0.0")
|
| 1344 |
|
| 1345 |
-
|
| 1346 |
-
|
| 1347 |
-
allow_origins=["*"],
|
| 1348 |
-
allow_credentials=True,
|
| 1349 |
-
allow_methods=["*"],
|
| 1350 |
-
allow_headers=["*"],
|
| 1351 |
-
)
|
| 1352 |
|
| 1353 |
|
| 1354 |
@api.exception_handler(BadRequest)
|
|
@@ -1450,7 +1291,8 @@ async def api_health():
|
|
| 1450 |
"popular_domains_loaded": popular_domains_loaded(),
|
| 1451 |
"transformers": transformer_loader.transformers_available,
|
| 1452 |
"transformer_models_loaded": list(transformer_loader.pipelines.keys()),
|
| 1453 |
-
"
|
|
|
|
| 1454 |
},
|
| 1455 |
"version": "2.0.0",
|
| 1456 |
"new_endpoints": [
|
|
@@ -2197,147 +2039,30 @@ async def api_process_dataset(request: Request):
|
|
| 2197 |
|
| 2198 |
|
| 2199 |
# ============================================================================
|
| 2200 |
-
#
|
| 2201 |
# ============================================================================
|
| 2202 |
|
| 2203 |
-
|
| 2204 |
-
|
| 2205 |
-
|
| 2206 |
-
|
| 2207 |
-
|
| 2208 |
-
|
| 2209 |
-
|
| 2210 |
-
|
| 2211 |
-
|
| 2212 |
-
|
| 2213 |
-
|
| 2214 |
-
|
| 2215 |
-
|
| 2216 |
-
|
| 2217 |
-
|
| 2218 |
-
|
| 2219 |
-
|
| 2220 |
-
value=get_available_notebooks()[0] if get_available_notebooks() else None,
|
| 2221 |
-
)
|
| 2222 |
-
refresh_btn = gr.Button("🔄 Refresh List")
|
| 2223 |
-
view_btn = gr.Button("👁 View Content", variant="secondary")
|
| 2224 |
-
execute_btn = gr.Button("▶ Execute Notebook", variant="primary")
|
| 2225 |
-
gr.Markdown("### Run Single Cell")
|
| 2226 |
-
cell_number = gr.Number(label="Cell Number", value=1, minimum=1)
|
| 2227 |
-
run_cell_btn = gr.Button("Run Cell")
|
| 2228 |
-
with gr.Column(scale=2):
|
| 2229 |
-
notebook_status = gr.Markdown("Select a notebook to view or execute.")
|
| 2230 |
-
notebook_output = gr.Markdown("", label="Output")
|
| 2231 |
-
|
| 2232 |
-
def refresh_notebooks():
|
| 2233 |
-
notebooks = get_available_notebooks()
|
| 2234 |
-
return gr.update(choices=notebooks, value=notebooks[0] if notebooks else None)
|
| 2235 |
-
|
| 2236 |
-
refresh_btn.click(refresh_notebooks, outputs=notebook_dropdown)
|
| 2237 |
-
view_btn.click(read_notebook_content, inputs=notebook_dropdown, outputs=notebook_output)
|
| 2238 |
-
execute_btn.click(execute_notebook, inputs=notebook_dropdown, outputs=[notebook_status, notebook_output])
|
| 2239 |
-
run_cell_btn.click(run_notebook_cell, inputs=[notebook_dropdown, cell_number], outputs=notebook_output)
|
| 2240 |
-
|
| 2241 |
-
# ============ TRAIN MODEL TAB ============
|
| 2242 |
-
with gr.TabItem("🎯 Train Model"):
|
| 2243 |
-
gr.Markdown("### Train a Security ML Model\nUpload your dataset and train a model for threat detection.")
|
| 2244 |
-
with gr.Row():
|
| 2245 |
-
with gr.Column():
|
| 2246 |
-
task_dropdown = gr.Dropdown(choices=SECURITY_TASKS, label="Security Task", value="Phishing Detection")
|
| 2247 |
-
model_dropdown = gr.Dropdown(choices=list(MODEL_TYPES.keys()), label="Model Type", value="Random Forest")
|
| 2248 |
-
data_upload = gr.File(label="Upload Training Data (CSV)", file_types=[".csv"])
|
| 2249 |
-
train_btn = gr.Button("🚀 Train Model", variant="primary")
|
| 2250 |
-
with gr.Column():
|
| 2251 |
-
train_output = gr.Markdown("Upload data and click Train to begin.")
|
| 2252 |
-
model_path_output = gr.Textbox(label="Model Path", visible=False)
|
| 2253 |
-
metrics_output = gr.Textbox(label="Metrics JSON", visible=False)
|
| 2254 |
-
train_btn.click(train_model_from_data, inputs=[data_upload, model_dropdown, task_dropdown], outputs=[train_output, model_path_output, metrics_output])
|
| 2255 |
-
|
| 2256 |
-
# ============ INFERENCE TAB ============
|
| 2257 |
-
with gr.TabItem("🔍 Inference"):
|
| 2258 |
-
gr.Markdown("### Run Model Inference\nLoad a trained model and make predictions.")
|
| 2259 |
-
with gr.Row():
|
| 2260 |
-
with gr.Column():
|
| 2261 |
-
model_upload = gr.File(label="Upload Model (.pkl)")
|
| 2262 |
-
features_input = gr.Textbox(label="Features (JSON)", value='{"url_length": 50, "has_https": 1, "digit_count": 5}', lines=5)
|
| 2263 |
-
predict_btn = gr.Button("🎯 Predict", variant="primary")
|
| 2264 |
-
with gr.Column():
|
| 2265 |
-
prediction_output = gr.Textbox(label="Prediction Result", lines=10)
|
| 2266 |
-
predict_btn.click(run_inference, inputs=[model_upload, features_input], outputs=prediction_output)
|
| 2267 |
-
|
| 2268 |
-
# ============ MODELS TAB ============
|
| 2269 |
-
with gr.TabItem("📦 Models"):
|
| 2270 |
-
gr.Markdown("### Trained Models")
|
| 2271 |
-
models_list = gr.Markdown(list_trained_models())
|
| 2272 |
-
refresh_models_btn = gr.Button("🔄 Refresh")
|
| 2273 |
-
refresh_models_btn.click(list_trained_models, outputs=models_list)
|
| 2274 |
-
|
| 2275 |
-
# ============ API TAB ============
|
| 2276 |
-
with gr.TabItem("🔌 API"):
|
| 2277 |
-
gr.Markdown("""
|
| 2278 |
-
## API Integration — v2.0
|
| 2279 |
-
|
| 2280 |
-
### Core Endpoints (stable, used by Heroku backend)
|
| 2281 |
-
|
| 2282 |
-
| Method | Endpoint | Description |
|
| 2283 |
-
|--------|----------|-------------|
|
| 2284 |
-
| GET | `/health` | Health check — version, models, services |
|
| 2285 |
-
| POST | `/analyze` | AI chat / general analysis (LLM or ML fallback) |
|
| 2286 |
-
| POST | `/analyze-url` | URL threat analysis — 4 ML models + risk factors |
|
| 2287 |
-
| POST | `/scan-threats` | Deep threat scan (LLM or ML fallback) |
|
| 2288 |
-
| POST | `/api/insights/generate` | AI-generated security insights |
|
| 2289 |
-
| POST | `/api/models/predict` | Direct ML model prediction |
|
| 2290 |
-
| GET | `/models` | List loaded models with source info |
|
| 2291 |
-
|
| 2292 |
-
### v2 Endpoints (new — for distributed agents + AI Deep Scan)
|
| 2293 |
-
|
| 2294 |
-
| Method | Endpoint | Description |
|
| 2295 |
-
|--------|----------|-------------|
|
| 2296 |
-
| POST | `/api/v2/batch-analyze` | Batch analyze up to 20 URLs (for multi-agent) |
|
| 2297 |
-
| GET | `/api/v2/explain/{model_name}` | Feature importances / explainability |
|
| 2298 |
-
| POST | `/api/v2/url-classify` | BERT-based URL phishing classifier |
|
| 2299 |
-
| POST | `/api/v2/dga-detect` | DGA domain detection (entropy heuristic) |
|
| 2300 |
-
| POST | `/api/v2/ioc-scan` | Multi-IOC scanner: URL + domain + DGA + BERT |
|
| 2301 |
-
| POST | `/api/v2/url-enrich` | Enriched analysis: ML + BERT + DGA + explainability |
|
| 2302 |
-
| POST | `/api/v2/chat` | Security chatbot — context-grounded, RAG-lite, machine→human translation |
|
| 2303 |
-
| POST | `/api/v2/security-chat` | Security LLM Q&A (open-source model running in this Space) |
|
| 2304 |
-
| GET | `/api/v2/status` | Transformer model load status |
|
| 2305 |
-
|
| 2306 |
-
### Quick Examples
|
| 2307 |
-
|
| 2308 |
-
```bash
|
| 2309 |
-
# Batch analyze URLs (for distributed agents)
|
| 2310 |
-
curl -X POST https://che237-cyberforge.hf.space/api/v2/batch-analyze \\
|
| 2311 |
-
-H "Content-Type: application/json" \\
|
| 2312 |
-
-d '{"urls": ["https://example.com", "http://evil.xyz/login?id=1"]}'
|
| 2313 |
-
|
| 2314 |
-
# Enriched URL analysis (Deep Scan mode)
|
| 2315 |
-
curl -X POST https://che237-cyberforge.hf.space/api/v2/url-enrich \\
|
| 2316 |
-
-H "Content-Type: application/json" \\
|
| 2317 |
-
-d '{"url": "https://example.com"}'
|
| 2318 |
-
|
| 2319 |
-
# Feature explainability
|
| 2320 |
-
curl https://che237-cyberforge.hf.space/api/v2/explain/phishing_detection
|
| 2321 |
-
|
| 2322 |
-
# IOC multi-field scan
|
| 2323 |
-
curl -X POST https://che237-cyberforge.hf.space/api/v2/ioc-scan \\
|
| 2324 |
-
-H "Content-Type: application/json" \\
|
| 2325 |
-
-d '{"url": "http://192.168.1.1/login", "domain": "192.168.1.1"}'
|
| 2326 |
-
|
| 2327 |
-
# Chatbot (Wave 2 extension point)
|
| 2328 |
-
curl -X POST https://che237-cyberforge.hf.space/api/v2/chat \\
|
| 2329 |
-
-H "Content-Type: application/json" \\
|
| 2330 |
-
-d '{"message": "Is this URL suspicious: http://login.g00gle.com.xyz/verify"}'
|
| 2331 |
-
```
|
| 2332 |
-
""")
|
| 2333 |
-
|
| 2334 |
-
gr.Markdown("---\n**CyberForge AI** | [GitHub](https://github.com/Che237/cyberforge) | [Datasets](https://huggingface.co/datasets/Che237/cyberforge-datasets)")
|
| 2335 |
-
|
| 2336 |
-
return demo
|
| 2337 |
|
| 2338 |
|
| 2339 |
# ============================================================================
|
| 2340 |
-
# MAIN
|
| 2341 |
# ============================================================================
|
| 2342 |
|
| 2343 |
# Initialize services at startup
|
|
@@ -2347,9 +2072,7 @@ ml_loader.initialize()
|
|
| 2347 |
threading.Thread(target=local_llm.load, name="local-llm-loader", daemon=True).start()
|
| 2348 |
logger.info("✅ Services initialized")
|
| 2349 |
|
| 2350 |
-
|
| 2351 |
-
demo = create_interface()
|
| 2352 |
-
app = gr.mount_gradio_app(api, demo, path="/")
|
| 2353 |
|
| 2354 |
if __name__ == "__main__":
|
| 2355 |
import uvicorn
|
|
|
|
| 1 |
"""
|
| 2 |
+
CyberForge AI - ML inference service
|
| 3 |
+
Hugging Face Spaces deployment: FastAPI REST endpoints for the CyberForge
|
| 4 |
+
backend. Every route except / and /health needs the service key
|
| 5 |
+
(ML_SERVICE_KEY), which only the backend holds; the two training routes need
|
| 6 |
+
the Space owner's Hugging Face token instead. Nothing here runs uploaded code,
|
| 7 |
+
notebooks or uploaded models.
|
| 8 |
"""
|
| 9 |
|
| 10 |
+
import hmac
|
|
|
|
| 11 |
import numpy as np
|
| 12 |
import json
|
| 13 |
import os
|
|
|
|
| 27 |
from url_features import (extract_url_features, popular_domains_loaded, rescale_score,
|
| 28 |
set_popular_domains, split_url)
|
| 29 |
|
| 30 |
+
# Models are loaded only from the model repository (ml_loader below), never
|
| 31 |
+
# from a caller: loading a joblib/pickle file can run code.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
import joblib
|
| 33 |
|
| 34 |
# Hugging Face Hub
|
|
|
|
| 37 |
# FastAPI for REST endpoints
|
| 38 |
from fastapi import FastAPI, Request
|
| 39 |
from fastapi.middleware.cors import CORSMiddleware
|
| 40 |
+
from fastapi.responses import HTMLResponse, JSONResponse
|
| 41 |
from starlette.concurrency import run_in_threadpool
|
| 42 |
|
| 43 |
logging.basicConfig(level=logging.INFO)
|
|
|
|
| 52 |
MODELS_DIR.mkdir(exist_ok=True)
|
| 53 |
DATASETS_DIR = APP_DIR / "datasets"
|
| 54 |
DATASETS_DIR.mkdir(exist_ok=True)
|
|
|
|
| 55 |
KNOWLEDGE_BASE_DIR = APP_DIR / "knowledge_base"
|
| 56 |
KNOWLEDGE_BASE_DIR.mkdir(exist_ok=True)
|
| 57 |
TRAINING_DATA_DIR = APP_DIR / "training_data"
|
|
|
|
| 76 |
LOCAL_LLM_QUEUE_TIMEOUT_S = float(os.environ.get("LOCAL_LLM_QUEUE_TIMEOUT_S", "5"))
|
| 77 |
|
| 78 |
logger.info(f"APP_DIR: {APP_DIR}")
|
| 79 |
+
# Calls to every route except / and /health must carry this key in the
|
| 80 |
+
# X-CyberForge-Service-Key header. It is a Space secret; the backend holds the
|
| 81 |
+
# same value. Without it set, those routes answer 503 rather than run open.
|
| 82 |
+
ML_SERVICE_KEY = os.environ.get("ML_SERVICE_KEY", "").strip()
|
| 83 |
+
SERVICE_KEY_HEADER = "x-cyberforge-service-key"
|
| 84 |
+
# The largest request body accepted, in bytes. Larger ones get HTTP 413.
|
| 85 |
+
ML_MAX_BODY_BYTES = int(os.environ.get("ML_MAX_BODY_BYTES", str(256 * 1024)))
|
| 86 |
|
| 87 |
+
logger.info(f"APP_DIR: {APP_DIR}")
|
| 88 |
+
logger.info(f"Service key {'set' if ML_SERVICE_KEY else 'NOT set: inference routes will answer 503'}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 89 |
|
| 90 |
|
| 91 |
# ============================================================================
|
|
|
|
| 103 |
"""The request body as a JSON object."""
|
| 104 |
try:
|
| 105 |
body = await request.json()
|
| 106 |
+
except BodyTooLarge:
|
| 107 |
+
# Raised by ServiceGate while the body streams in; it answers 413.
|
| 108 |
+
raise
|
| 109 |
except Exception:
|
| 110 |
raise BadRequest("Request body must be valid JSON")
|
| 111 |
if not isinstance(body, dict):
|
|
|
|
| 1100 |
|
| 1101 |
|
| 1102 |
# ============================================================================
|
| 1103 |
+
# FASTAPI APP (REST endpoints for Heroku backend mlService.js)
|
| 1104 |
# ============================================================================
|
| 1105 |
|
| 1106 |
+
api = FastAPI(title="CyberForge AI API", version="1.0.0")
|
|
|
|
|
|
|
|
|
|
| 1107 |
|
| 1108 |
+
api.add_middleware(
|
| 1109 |
+
CORSMiddleware,
|
| 1110 |
+
allow_origins=["*"],
|
| 1111 |
+
allow_credentials=True,
|
| 1112 |
+
allow_methods=["*"],
|
| 1113 |
+
allow_headers=["*"],
|
| 1114 |
+
)
|
| 1115 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1116 |
|
| 1117 |
+
PUBLIC_PATHS = {"/", "/health"}
|
| 1118 |
+
# These check the owner's Hugging Face token themselves (owner_token below).
|
| 1119 |
+
OWNER_PATHS = {"/api/admin/train", "/api/admin/train/status"}
|
| 1120 |
|
|
|
|
|
|
|
|
|
|
| 1121 |
|
| 1122 |
+
class BodyTooLarge(Exception):
|
| 1123 |
+
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1124 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1125 |
|
| 1126 |
+
def service_key_valid(presented: Optional[bytes]) -> bool:
|
| 1127 |
+
if not ML_SERVICE_KEY or not presented:
|
| 1128 |
+
return False
|
| 1129 |
+
return hmac.compare_digest(presented, ML_SERVICE_KEY.encode("utf-8"))
|
| 1130 |
|
|
|
|
| 1131 |
|
| 1132 |
+
async def _reply(send, status: int, detail: str, code: str):
|
| 1133 |
+
body = json.dumps({"detail": detail, "code": code}).encode("utf-8")
|
| 1134 |
+
await send({"type": "http.response.start", "status": status,
|
| 1135 |
+
"headers": [(b"content-type", b"application/json"), (b"content-length", str(len(body)).encode())]})
|
| 1136 |
+
await send({"type": "http.response.body", "body": body})
|
| 1137 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1138 |
|
| 1139 |
+
class ServiceGate:
|
| 1140 |
+
"""Outermost middleware. It refuses a call without the service key before
|
| 1141 |
+
any route runs, and a body over ML_MAX_BODY_BYTES whether or not it states
|
| 1142 |
+
its length."""
|
| 1143 |
|
| 1144 |
+
def __init__(self, app):
|
| 1145 |
+
self.app = app
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1146 |
|
| 1147 |
+
async def __call__(self, scope, receive, send):
|
| 1148 |
+
if scope["type"] != "http":
|
| 1149 |
+
return await self.app(scope, receive, send)
|
| 1150 |
+
path = scope.get("path", "")
|
| 1151 |
+
headers = dict(scope.get("headers") or [])
|
| 1152 |
+
# A CORS preflight carries no credentials; the CORS middleware answers it.
|
| 1153 |
+
if scope.get("method") != "OPTIONS" and path not in PUBLIC_PATHS and path not in OWNER_PATHS:
|
| 1154 |
+
if not ML_SERVICE_KEY:
|
| 1155 |
+
return await _reply(send, 503, "This service has no service key configured", "SERVICE_KEY_NOT_CONFIGURED")
|
| 1156 |
+
if not service_key_valid(headers.get(SERVICE_KEY_HEADER.encode())):
|
| 1157 |
+
return await _reply(send, 401, "A valid service key is required", "SERVICE_KEY_REQUIRED")
|
| 1158 |
+
stated = headers.get(b"content-length")
|
| 1159 |
+
if stated is not None:
|
| 1160 |
+
try:
|
| 1161 |
+
if int(stated) > ML_MAX_BODY_BYTES:
|
| 1162 |
+
return await _reply(send, 413, f"Request body is over {ML_MAX_BODY_BYTES} bytes", "BODY_TOO_LARGE")
|
| 1163 |
+
except ValueError:
|
| 1164 |
+
return await _reply(send, 400, "Content-Length must be a number", "BAD_CONTENT_LENGTH")
|
| 1165 |
+
received = 0
|
| 1166 |
+
started = False
|
| 1167 |
+
|
| 1168 |
+
async def bounded_receive():
|
| 1169 |
+
nonlocal received
|
| 1170 |
+
message = await receive()
|
| 1171 |
+
if message["type"] == "http.request":
|
| 1172 |
+
received += len(message.get("body", b""))
|
| 1173 |
+
if received > ML_MAX_BODY_BYTES:
|
| 1174 |
+
raise BodyTooLarge()
|
| 1175 |
+
return message
|
| 1176 |
+
|
| 1177 |
+
async def tracked_send(message):
|
| 1178 |
+
nonlocal started
|
| 1179 |
+
if message["type"] == "http.response.start":
|
| 1180 |
+
started = True
|
| 1181 |
+
await send(message)
|
| 1182 |
|
| 1183 |
+
try:
|
| 1184 |
+
await self.app(scope, bounded_receive, tracked_send)
|
| 1185 |
+
except BodyTooLarge:
|
| 1186 |
+
if started:
|
| 1187 |
+
raise
|
| 1188 |
+
await _reply(send, 413, f"Request body is over {ML_MAX_BODY_BYTES} bytes", "BODY_TOO_LARGE")
|
| 1189 |
|
|
|
|
| 1190 |
|
| 1191 |
+
# Added last, so it runs first: before CORS and every route.
|
| 1192 |
+
api.add_middleware(ServiceGate)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1193 |
|
| 1194 |
|
| 1195 |
@api.exception_handler(BadRequest)
|
|
|
|
| 1291 |
"popular_domains_loaded": popular_domains_loaded(),
|
| 1292 |
"transformers": transformer_loader.transformers_available,
|
| 1293 |
"transformer_models_loaded": list(transformer_loader.pipelines.keys()),
|
| 1294 |
+
"service_key": "required" if ML_SERVICE_KEY else "not_configured",
|
| 1295 |
+
"max_body_bytes": ML_MAX_BODY_BYTES,
|
| 1296 |
},
|
| 1297 |
"version": "2.0.0",
|
| 1298 |
"new_endpoints": [
|
|
|
|
| 2039 |
|
| 2040 |
|
| 2041 |
# ============================================================================
|
| 2042 |
+
# LANDING PAGE (static; the service has no interactive UI)
|
| 2043 |
# ============================================================================
|
| 2044 |
|
| 2045 |
+
LANDING_HTML = """<!doctype html>
|
| 2046 |
+
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
|
| 2047 |
+
<title>CyberForge ML service</title>
|
| 2048 |
+
<style>body{font-family:system-ui,sans-serif;max-width:46rem;margin:2rem auto;padding:0 1rem;line-height:1.5;color:#1b1f24;background:#fff}
|
| 2049 |
+
code{background:#f1f3f5;padding:0 .25rem;border-radius:3px}</style></head>
|
| 2050 |
+
<body><h1>CyberForge ML service</h1>
|
| 2051 |
+
<p>This Space serves the detection models and the security chat model to the CyberForge backend.
|
| 2052 |
+
It has no interactive interface: it does not run notebooks, train on uploaded data, or load uploaded models.</p>
|
| 2053 |
+
<p><code>GET /health</code> is public. Every other route needs the service key held by the backend,
|
| 2054 |
+
and model training needs the Space owner's token.</p>
|
| 2055 |
+
<p>Source: <a href="https://github.com/Che237/cyberforge">github.com/Che237/cyberforge</a></p>
|
| 2056 |
+
</body></html>"""
|
| 2057 |
+
|
| 2058 |
+
|
| 2059 |
+
@api.get("/", response_class=HTMLResponse)
|
| 2060 |
+
async def landing():
|
| 2061 |
+
return LANDING_HTML
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2062 |
|
| 2063 |
|
| 2064 |
# ============================================================================
|
| 2065 |
+
# MAIN
|
| 2066 |
# ============================================================================
|
| 2067 |
|
| 2068 |
# Initialize services at startup
|
|
|
|
| 2072 |
threading.Thread(target=local_llm.load, name="local-llm-loader", daemon=True).start()
|
| 2073 |
logger.info("✅ Services initialized")
|
| 2074 |
|
| 2075 |
+
app = api
|
|
|
|
|
|
|
| 2076 |
|
| 2077 |
if __name__ == "__main__":
|
| 2078 |
import uvicorn
|
requirements.txt
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
# CyberForge AI - HuggingFace Space
|
| 2 |
-
#
|
| 3 |
|
| 4 |
# Core ML
|
| 5 |
# Pinned: the .pkl files in Che237/cyberforge-models were saved with 1.8.0 and the
|
|
@@ -14,9 +14,6 @@ torch>=2.2.0
|
|
| 14 |
transformers>=4.40.0
|
| 15 |
tokenizers>=0.19.0
|
| 16 |
|
| 17 |
-
# Gradio UI
|
| 18 |
-
gradio>=5.20.0
|
| 19 |
-
|
| 20 |
# FastAPI REST endpoints (for Heroku backend)
|
| 21 |
fastapi>=0.100.0
|
| 22 |
uvicorn[standard]>=0.22.0
|
|
@@ -35,14 +32,6 @@ tldextract>=5.1.0
|
|
| 35 |
--only-binary llama-cpp-python
|
| 36 |
llama-cpp-python==0.3.36
|
| 37 |
|
| 38 |
-
# Notebook execution
|
| 39 |
-
nbformat>=5.10.0
|
| 40 |
-
nbconvert>=7.16.0
|
| 41 |
-
nest_asyncio>=1.6.0
|
| 42 |
-
|
| 43 |
-
# HTTP client for notebooks
|
| 44 |
-
httpx>=0.27.0
|
| 45 |
-
|
| 46 |
# Data handling
|
| 47 |
datasets>=2.20.0
|
| 48 |
pyarrow>=17.0.0
|
|
|
|
| 1 |
# CyberForge AI - HuggingFace Space
|
| 2 |
+
# FastAPI REST API only. There is no Gradio UI and no notebook runner (V2-ML01).
|
| 3 |
|
| 4 |
# Core ML
|
| 5 |
# Pinned: the .pkl files in Che237/cyberforge-models were saved with 1.8.0 and the
|
|
|
|
| 14 |
transformers>=4.40.0
|
| 15 |
tokenizers>=0.19.0
|
| 16 |
|
|
|
|
|
|
|
|
|
|
| 17 |
# FastAPI REST endpoints (for Heroku backend)
|
| 18 |
fastapi>=0.100.0
|
| 19 |
uvicorn[standard]>=0.22.0
|
|
|
|
| 32 |
--only-binary llama-cpp-python
|
| 33 |
llama-cpp-python==0.3.36
|
| 34 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
# Data handling
|
| 36 |
datasets>=2.20.0
|
| 37 |
pyarrow>=17.0.0
|