Spaces:
Configuration error
Configuration error
nayab zahoor commited on
Commit ·
9f26583
1
Parent(s): d7a6f3b
Deploy: Full-Stack App without large binary images
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- BACKEND +0 -1
- BACKEND/.gitignore +16 -0
- BACKEND/app.zip +3 -0
- BACKEND/app/__init__.py +0 -0
- BACKEND/app/auth.py +111 -0
- BACKEND/app/constants.py +3 -0
- BACKEND/app/database.py +17 -0
- BACKEND/app/db_models.py +38 -0
- BACKEND/app/inference/compare_support.py +154 -0
- BACKEND/app/inference/decide_result.py +88 -0
- BACKEND/app/inference/evaluate_seen_vs_unseen.py +200 -0
- BACKEND/app/inference/make_embedding.py +48 -0
- BACKEND/app/inference/scan_user_apk.py +70 -0
- BACKEND/app/main.py +179 -0
- BACKEND/app/models/resnet34_siamese.py +73 -0
- BACKEND/app/preprocessing/apk_pipeline.py +38 -0
- BACKEND/app/preprocessing/apk_to_grayscale.py +74 -0
- BACKEND/app/preprocessing/dex_utils.py +48 -0
- BACKEND/app/preprocessing/resize_utils.py +26 -0
- BACKEND/app/routers/auth.py +88 -0
- BACKEND/app/routers/history.py +41 -0
- BACKEND/app/routers/local_report.py +134 -0
- BACKEND/app/routers/sandbox.py +298 -0
- BACKEND/app/routers/scan.py +0 -0
- BACKEND/app/schemas.py +41 -0
- BACKEND/app/settings.py +26 -0
- BACKEND/app/static_analysis/__init__.py +0 -0
- BACKEND/app/static_analysis/apk_analyzer.py +11 -0
- BACKEND/app/support/build_support_set.py +1 -0
- BACKEND/app/support/save_support_embeddings.py +138 -0
- BACKEND/app/training/dataset_loader.py +1 -0
- BACKEND/app/training/loss_functions.py +22 -0
- BACKEND/app/training/pair_dataset.py +79 -0
- BACKEND/app/training/train_resnet34.py +112 -0
- BACKEND/app/utils/file_utils.py +5 -0
- BACKEND/app/utils/image_utils.py +5 -0
- BACKEND/data_split.py +270 -0
- BACKEND/dockerfile +28 -0
- BACKEND/list_models.py +10 -0
- BACKEND/requirements.txt +37 -0
- BACKEND/run_backend.py +27 -0
- BACKEND/setup_dataset.py +142 -0
- BACKEND/streamlit_app.py +0 -0
- BACKEND/support_embeddings.zip +3 -0
- BACKEND/train_and_prepare.py +13 -0
- BACKEND/trained_models/evaluation_metrics.json +59 -0
- BACKEND/trained_models/train_history.json +122 -0
- malware-detection-frontend +0 -1
- malware-detection-frontend/.gitignore +31 -0
- malware-detection-frontend/README.md +70 -0
BACKEND
DELETED
|
@@ -1 +0,0 @@
|
|
| 1 |
-
Subproject commit 05bc4c478ac699cb0b5a362dc3cb711738c28a7a
|
|
|
|
|
|
BACKEND/.gitignore
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
@"
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.pyc
|
| 4 |
+
.venv/
|
| 5 |
+
venv/
|
| 6 |
+
data/
|
| 7 |
+
uploads/
|
| 8 |
+
outputs/
|
| 9 |
+
temp/
|
| 10 |
+
trained_models/*.pth
|
| 11 |
+
support_embeddings/*.pkl
|
| 12 |
+
*.log
|
| 13 |
+
.DS_Store
|
| 14 |
+
Thumbs.db
|
| 15 |
+
"@ | Out-File -FilePath .gitignore -Encoding utf8
|
| 16 |
+
malware_users.db
|
BACKEND/app.zip
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e0632397952ed746853ec8bad79d403b9f02eb92ec856bcfb04fba6ef48b68c2
|
| 3 |
+
size 54075
|
BACKEND/app/__init__.py
ADDED
|
File without changes
|
BACKEND/app/auth.py
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from datetime import datetime, timedelta
|
| 3 |
+
from jose import JWTError, jwt
|
| 4 |
+
from dotenv import load_dotenv
|
| 5 |
+
from fastapi import Depends, HTTPException, status
|
| 6 |
+
from fastapi.security import OAuth2PasswordBearer
|
| 7 |
+
from sqlalchemy.orm import Session
|
| 8 |
+
from app.database import get_db
|
| 9 |
+
from app.db_models import User
|
| 10 |
+
|
| 11 |
+
load_dotenv()
|
| 12 |
+
|
| 13 |
+
SECRET_KEY = os.getenv("SECRET_KEY", "your-secret-key-change-in-production")
|
| 14 |
+
ALGORITHM = "HS256"
|
| 15 |
+
ACCESS_TOKEN_EXPIRE_MINUTES = 30
|
| 16 |
+
|
| 17 |
+
oauth2_scheme = OAuth2PasswordBearer(tokenUrl="auth/login")
|
| 18 |
+
|
| 19 |
+
# Plain text (no hashing) – ONLY for development
|
| 20 |
+
def verify_password(plain_password, stored_password):
|
| 21 |
+
return plain_password == stored_password
|
| 22 |
+
|
| 23 |
+
def get_password_hash(password):
|
| 24 |
+
return password # store as plain text
|
| 25 |
+
|
| 26 |
+
def create_access_token(data: dict, expires_delta: timedelta = None):
|
| 27 |
+
to_encode = data.copy()
|
| 28 |
+
if expires_delta:
|
| 29 |
+
expire = datetime.utcnow() + expires_delta
|
| 30 |
+
else:
|
| 31 |
+
expire = datetime.utcnow() + timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES)
|
| 32 |
+
to_encode.update({"exp": expire})
|
| 33 |
+
encoded_jwt = jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM)
|
| 34 |
+
return encoded_jwt
|
| 35 |
+
|
| 36 |
+
def send_reset_email(email, token):
|
| 37 |
+
reset_link = f"http://localhost:3000/reset-password?token={token}"
|
| 38 |
+
print(f"Password reset link for {email}: {reset_link}")
|
| 39 |
+
|
| 40 |
+
def get_current_user(token: str = Depends(oauth2_scheme), db: Session = Depends(get_db)):
|
| 41 |
+
credentials_exception = HTTPException(
|
| 42 |
+
status_code=status.HTTP_401_UNAUTHORIZED,
|
| 43 |
+
detail="Could not validate credentials",
|
| 44 |
+
headers={"WWW-Authenticate": "Bearer"},
|
| 45 |
+
)
|
| 46 |
+
try:
|
| 47 |
+
payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
|
| 48 |
+
email: str = payload.get("sub")
|
| 49 |
+
if email is None:
|
| 50 |
+
raise credentials_exception
|
| 51 |
+
except JWTError:
|
| 52 |
+
raise credentials_exception
|
| 53 |
+
user = db.query(User).filter(User.email == email).first()
|
| 54 |
+
if user is None:
|
| 55 |
+
raise credentials_exception
|
| 56 |
+
return user
|
| 57 |
+
import smtplib
|
| 58 |
+
from email.message import EmailMessage
|
| 59 |
+
import os
|
| 60 |
+
|
| 61 |
+
def send_reset_email(email, token):
|
| 62 |
+
# Construct reset link
|
| 63 |
+
reset_link = f"http://localhost:3000/reset-password?token={token}"
|
| 64 |
+
|
| 65 |
+
# Email content
|
| 66 |
+
subject = "Password Reset Request"
|
| 67 |
+
body = f"""
|
| 68 |
+
Hello,
|
| 69 |
+
|
| 70 |
+
You requested a password reset for your Android Malware Detection account.
|
| 71 |
+
|
| 72 |
+
Click the link below to reset your password:
|
| 73 |
+
{reset_link}
|
| 74 |
+
|
| 75 |
+
This link will expire in 1 hour.
|
| 76 |
+
|
| 77 |
+
If you did not request this, please ignore this email.
|
| 78 |
+
|
| 79 |
+
Regards,
|
| 80 |
+
Android Malware Detection Team
|
| 81 |
+
"""
|
| 82 |
+
|
| 83 |
+
# Get email credentials from environment
|
| 84 |
+
smtp_host = os.getenv("EMAIL_HOST", "smtp.gmail.com")
|
| 85 |
+
smtp_port = int(os.getenv("EMAIL_PORT", 587))
|
| 86 |
+
sender_email = os.getenv("EMAIL_HOST_USER")
|
| 87 |
+
sender_password = os.getenv("EMAIL_HOST_PASSWORD")
|
| 88 |
+
|
| 89 |
+
if not sender_email or not sender_password:
|
| 90 |
+
print("Email credentials not set. Reset link would have been sent to:", email)
|
| 91 |
+
print("Reset link:", reset_link)
|
| 92 |
+
return
|
| 93 |
+
|
| 94 |
+
# Create email message
|
| 95 |
+
msg = EmailMessage()
|
| 96 |
+
msg.set_content(body)
|
| 97 |
+
msg["Subject"] = subject
|
| 98 |
+
msg["From"] = sender_email
|
| 99 |
+
msg["To"] = email
|
| 100 |
+
|
| 101 |
+
# Send email
|
| 102 |
+
try:
|
| 103 |
+
with smtplib.SMTP(smtp_host, smtp_port) as server:
|
| 104 |
+
server.starttls()
|
| 105 |
+
server.login(sender_email, sender_password)
|
| 106 |
+
server.send_message(msg)
|
| 107 |
+
print(f"Password reset email sent to {email}")
|
| 108 |
+
except Exception as e:
|
| 109 |
+
print(f"Failed to send email: {e}")
|
| 110 |
+
# Fallback: print to console for debugging
|
| 111 |
+
print(f"Reset link for {email}: {reset_link}")
|
BACKEND/app/constants.py
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
TRAIN_FAMILIES = ["benign", "banking", "smsware"]
|
| 2 |
+
ALL_FAMILIES = ["benign", "banking", "smsware", "adware", "riskware"]
|
| 3 |
+
MALWARE_FAMILIES = ["banking", "smsware", "adware", "riskware"]
|
BACKEND/app/database.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from sqlalchemy import create_engine
|
| 2 |
+
from sqlalchemy.ext.declarative import declarative_base
|
| 3 |
+
from sqlalchemy.orm import sessionmaker
|
| 4 |
+
|
| 5 |
+
SQLALCHEMY_DATABASE_URL = "sqlite:///./malware_users.db"
|
| 6 |
+
|
| 7 |
+
engine = create_engine(SQLALCHEMY_DATABASE_URL, connect_args={"check_same_thread": False})
|
| 8 |
+
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
| 9 |
+
|
| 10 |
+
Base = declarative_base()
|
| 11 |
+
|
| 12 |
+
def get_db():
|
| 13 |
+
db = SessionLocal()
|
| 14 |
+
try:
|
| 15 |
+
yield db
|
| 16 |
+
finally:
|
| 17 |
+
db.close()
|
BACKEND/app/db_models.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from sqlalchemy import Column, Integer, String, DateTime, ForeignKey, Float, JSON
|
| 2 |
+
from sqlalchemy.sql import func
|
| 3 |
+
from app.database import Base
|
| 4 |
+
|
| 5 |
+
class User(Base):
|
| 6 |
+
__tablename__ = "users"
|
| 7 |
+
id = Column(Integer, primary_key=True, index=True)
|
| 8 |
+
email = Column(String(100), unique=True, index=True, nullable=False)
|
| 9 |
+
hashed_password = Column(String(255), nullable=False)
|
| 10 |
+
full_name = Column(String(100), nullable=True) # NEW
|
| 11 |
+
role = Column(String(20), nullable=False)
|
| 12 |
+
university = Column(String(200), nullable=True)
|
| 13 |
+
organization = Column(String(200), nullable=True)
|
| 14 |
+
org_details = Column(String(500), nullable=True)
|
| 15 |
+
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
| 16 |
+
|
| 17 |
+
class PasswordResetToken(Base):
|
| 18 |
+
__tablename__ = "password_reset_tokens"
|
| 19 |
+
|
| 20 |
+
id = Column(Integer, primary_key=True, index=True)
|
| 21 |
+
user_id = Column(Integer, ForeignKey("users.id"), nullable=False)
|
| 22 |
+
token = Column(String(255), unique=True, index=True, nullable=False)
|
| 23 |
+
expires_at = Column(DateTime(timezone=True), nullable=False)
|
| 24 |
+
|
| 25 |
+
class ScanHistory(Base):
|
| 26 |
+
__tablename__ = "scan_history"
|
| 27 |
+
|
| 28 |
+
id = Column(Integer, primary_key=True, index=True)
|
| 29 |
+
user_id = Column(Integer, ForeignKey("users.id"), nullable=False)
|
| 30 |
+
file_name = Column(String(255), nullable=False)
|
| 31 |
+
predicted_family = Column(String(50))
|
| 32 |
+
predicted_label = Column(String(20))
|
| 33 |
+
confidence = Column(Float)
|
| 34 |
+
danger_score = Column(Float)
|
| 35 |
+
permissions = Column(JSON) # store list
|
| 36 |
+
api_calls = Column(JSON) # store list
|
| 37 |
+
full_response = Column(JSON) # store the whole scan result
|
| 38 |
+
scanned_at = Column(DateTime(timezone=True), server_default=func.now())
|
BACKEND/app/inference/compare_support.py
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import pickle
|
| 3 |
+
from typing import Dict, List, Tuple
|
| 4 |
+
|
| 5 |
+
import numpy as np
|
| 6 |
+
|
| 7 |
+
from app.settings import SUPPORT_EMBEDDINGS_PATH
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def load_support_embeddings() -> Dict[str, List[dict]]:
|
| 11 |
+
if not os.path.isfile(SUPPORT_EMBEDDINGS_PATH):
|
| 12 |
+
raise FileNotFoundError(
|
| 13 |
+
f"Support embeddings file not found: {SUPPORT_EMBEDDINGS_PATH}"
|
| 14 |
+
)
|
| 15 |
+
|
| 16 |
+
with open(SUPPORT_EMBEDDINGS_PATH, "rb") as f:
|
| 17 |
+
return pickle.load(f)
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def l2_normalize(x: np.ndarray) -> np.ndarray:
|
| 21 |
+
x = np.asarray(x, dtype=np.float32).flatten()
|
| 22 |
+
norm = np.linalg.norm(x)
|
| 23 |
+
if norm < 1e-8:
|
| 24 |
+
return x
|
| 25 |
+
return x / norm
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def cosine_similarity(a: np.ndarray, b: np.ndarray) -> float:
|
| 29 |
+
a = l2_normalize(a)
|
| 30 |
+
b = l2_normalize(b)
|
| 31 |
+
return float(np.dot(a, b))
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def build_prototype(embeddings: List[np.ndarray]) -> np.ndarray:
|
| 35 |
+
arr = np.stack([l2_normalize(e) for e in embeddings], axis=0)
|
| 36 |
+
proto = np.mean(arr, axis=0)
|
| 37 |
+
return l2_normalize(proto)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def get_weighted_family_items(items: List[dict]) -> Tuple[List[np.ndarray], List[float]]:
|
| 41 |
+
"""
|
| 42 |
+
support images ko زیادہ importance
|
| 43 |
+
train images ko کم importance
|
| 44 |
+
"""
|
| 45 |
+
embeddings = []
|
| 46 |
+
weights = []
|
| 47 |
+
|
| 48 |
+
for item in items:
|
| 49 |
+
emb = np.array(item["embedding"], dtype=np.float32).flatten()
|
| 50 |
+
emb = l2_normalize(emb)
|
| 51 |
+
|
| 52 |
+
source = item.get("source", "support")
|
| 53 |
+
|
| 54 |
+
if source == "support":
|
| 55 |
+
weight = 1.0
|
| 56 |
+
elif source == "train":
|
| 57 |
+
weight = 0.65
|
| 58 |
+
else:
|
| 59 |
+
weight = 0.8
|
| 60 |
+
|
| 61 |
+
embeddings.append(emb)
|
| 62 |
+
weights.append(weight)
|
| 63 |
+
|
| 64 |
+
return embeddings, weights
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def weighted_topk_score(
|
| 68 |
+
query_embedding: np.ndarray,
|
| 69 |
+
embeddings: List[np.ndarray],
|
| 70 |
+
weights: List[float],
|
| 71 |
+
top_k: int = 5
|
| 72 |
+
) -> float:
|
| 73 |
+
if not embeddings:
|
| 74 |
+
return 0.0
|
| 75 |
+
|
| 76 |
+
sims = []
|
| 77 |
+
for emb, w in zip(embeddings, weights):
|
| 78 |
+
sim = cosine_similarity(query_embedding, emb)
|
| 79 |
+
sims.append(sim * w)
|
| 80 |
+
|
| 81 |
+
sims = sorted(sims, reverse=True)
|
| 82 |
+
k = min(top_k, len(sims))
|
| 83 |
+
return float(np.mean(sims[:k]))
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def weighted_best_score(
|
| 87 |
+
query_embedding: np.ndarray,
|
| 88 |
+
embeddings: List[np.ndarray],
|
| 89 |
+
weights: List[float]
|
| 90 |
+
) -> float:
|
| 91 |
+
if not embeddings:
|
| 92 |
+
return 0.0
|
| 93 |
+
|
| 94 |
+
best = -1.0
|
| 95 |
+
for emb, w in zip(embeddings, weights):
|
| 96 |
+
sim = cosine_similarity(query_embedding, emb) * w
|
| 97 |
+
if sim > best:
|
| 98 |
+
best = sim
|
| 99 |
+
return float(best)
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def compare_with_support(query_embedding: np.ndarray) -> Dict[str, float]:
|
| 103 |
+
db = load_support_embeddings()
|
| 104 |
+
family_scores = {}
|
| 105 |
+
|
| 106 |
+
query_embedding = l2_normalize(query_embedding)
|
| 107 |
+
|
| 108 |
+
for family, items in db.items():
|
| 109 |
+
if not items:
|
| 110 |
+
continue
|
| 111 |
+
|
| 112 |
+
support_embeddings, source_weights = get_weighted_family_items(items)
|
| 113 |
+
|
| 114 |
+
if not support_embeddings:
|
| 115 |
+
continue
|
| 116 |
+
|
| 117 |
+
# ----- prototype -----
|
| 118 |
+
prototype = build_prototype(support_embeddings)
|
| 119 |
+
prototype_score = cosine_similarity(query_embedding, prototype)
|
| 120 |
+
|
| 121 |
+
# ----- local nearest evidence -----
|
| 122 |
+
best_score = weighted_best_score(
|
| 123 |
+
query_embedding=query_embedding,
|
| 124 |
+
embeddings=support_embeddings,
|
| 125 |
+
weights=source_weights,
|
| 126 |
+
)
|
| 127 |
+
|
| 128 |
+
top3_score = weighted_topk_score(
|
| 129 |
+
query_embedding=query_embedding,
|
| 130 |
+
embeddings=support_embeddings,
|
| 131 |
+
weights=source_weights,
|
| 132 |
+
top_k=3,
|
| 133 |
+
)
|
| 134 |
+
|
| 135 |
+
top5_score = weighted_topk_score(
|
| 136 |
+
query_embedding=query_embedding,
|
| 137 |
+
embeddings=support_embeddings,
|
| 138 |
+
weights=source_weights,
|
| 139 |
+
top_k=5,
|
| 140 |
+
)
|
| 141 |
+
|
| 142 |
+
# ----- fusion -----
|
| 143 |
+
# prototype = class center
|
| 144 |
+
# best/topk = local evidence
|
| 145 |
+
final_score = (
|
| 146 |
+
0.40 * prototype_score +
|
| 147 |
+
0.25 * best_score +
|
| 148 |
+
0.20 * top3_score +
|
| 149 |
+
0.15 * top5_score
|
| 150 |
+
)
|
| 151 |
+
|
| 152 |
+
family_scores[family] = float(final_score)
|
| 153 |
+
|
| 154 |
+
return family_scores
|
BACKEND/app/inference/decide_result.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Dict
|
| 2 |
+
import numpy as np
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
def softmax(x: np.ndarray, temperature: float = 0.30) -> np.ndarray:
|
| 6 |
+
x = np.asarray(x, dtype=np.float32)
|
| 7 |
+
|
| 8 |
+
if x.size == 0:
|
| 9 |
+
return x
|
| 10 |
+
|
| 11 |
+
temperature = max(temperature, 1e-6)
|
| 12 |
+
x = x / temperature
|
| 13 |
+
x = x - np.max(x)
|
| 14 |
+
|
| 15 |
+
exp_x = np.exp(x)
|
| 16 |
+
denom = np.sum(exp_x)
|
| 17 |
+
|
| 18 |
+
if denom < 1e-8:
|
| 19 |
+
return np.ones_like(x) / len(x)
|
| 20 |
+
|
| 21 |
+
return exp_x / denom
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def decide_prediction(similarity_scores: Dict[str, float]) -> Dict[str, float]:
|
| 25 |
+
if not similarity_scores:
|
| 26 |
+
return {
|
| 27 |
+
"predicted_family": "unknown",
|
| 28 |
+
"predicted_label": "unknown",
|
| 29 |
+
"confidence": 0.0,
|
| 30 |
+
"danger_score": 0.0,
|
| 31 |
+
"family_matches": {},
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
families = list(similarity_scores.keys())
|
| 35 |
+
scores = np.array([similarity_scores[f] for f in families], dtype=np.float32)
|
| 36 |
+
|
| 37 |
+
probs = softmax(scores, temperature=0.30)
|
| 38 |
+
|
| 39 |
+
ranked = sorted(
|
| 40 |
+
zip(families, scores, probs),
|
| 41 |
+
key=lambda x: x[1],
|
| 42 |
+
reverse=True
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
best_family, best_score, best_prob = ranked[0]
|
| 46 |
+
|
| 47 |
+
if len(ranked) > 1:
|
| 48 |
+
second_family, second_score, second_prob = ranked[1]
|
| 49 |
+
margin = float(best_score - second_score)
|
| 50 |
+
else:
|
| 51 |
+
second_family, second_score, second_prob = "none", 0.0, 0.0
|
| 52 |
+
margin = float(best_score)
|
| 53 |
+
|
| 54 |
+
margin_conf = max(0.0, min(1.0, (margin + 0.05) / 0.15))
|
| 55 |
+
|
| 56 |
+
confidence = (0.75 * float(best_prob)) + (0.25 * margin_conf)
|
| 57 |
+
confidence = float(max(0.0, min(1.0, confidence))) * 100.0
|
| 58 |
+
|
| 59 |
+
if best_family == "benign":
|
| 60 |
+
predicted_label = "benign"
|
| 61 |
+
danger_score = max(0.0, 100.0 - confidence)
|
| 62 |
+
else:
|
| 63 |
+
predicted_label = "malware"
|
| 64 |
+
danger_score = confidence
|
| 65 |
+
|
| 66 |
+
family_matches = {
|
| 67 |
+
family: round(float(prob) * 100.0, 2)
|
| 68 |
+
for family, prob in zip(families, probs)
|
| 69 |
+
}
|
| 70 |
+
|
| 71 |
+
return {
|
| 72 |
+
"predicted_family": best_family,
|
| 73 |
+
"predicted_label": predicted_label,
|
| 74 |
+
"confidence": round(confidence, 2),
|
| 75 |
+
"danger_score": round(danger_score, 2),
|
| 76 |
+
"family_matches": family_matches,
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
if __name__ == "__main__":
|
| 81 |
+
scores = {
|
| 82 |
+
"trojan": 0.82,
|
| 83 |
+
"ransomware": 0.61,
|
| 84 |
+
"benign": 0.12
|
| 85 |
+
}
|
| 86 |
+
|
| 87 |
+
result = decide_prediction(scores)
|
| 88 |
+
print(result)
|
BACKEND/app/inference/evaluate_seen_vs_unseen.py
ADDED
|
@@ -0,0 +1,200 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import json
|
| 3 |
+
from typing import Dict, List, Tuple
|
| 4 |
+
|
| 5 |
+
import numpy as np
|
| 6 |
+
from sklearn.metrics import confusion_matrix
|
| 7 |
+
|
| 8 |
+
from app.inference.make_embedding import load_model, make_embedding
|
| 9 |
+
from app.inference.compare_support import compare_with_support
|
| 10 |
+
from app.inference.decide_result import decide_prediction
|
| 11 |
+
from app.settings import TEST_IMAGES_DIR
|
| 12 |
+
|
| 13 |
+
SEEN_FAMILIES = ["benign", "banking", "smsware"]
|
| 14 |
+
UNSEEN_FAMILIES = ["adware", "riskware"]
|
| 15 |
+
|
| 16 |
+
def get_family_image_paths(root_dir: str) -> Dict[str, List[str]]:
|
| 17 |
+
family_to_images = {}
|
| 18 |
+
|
| 19 |
+
if not os.path.isdir(root_dir):
|
| 20 |
+
raise FileNotFoundError(f"Test directory not found: {root_dir}")
|
| 21 |
+
|
| 22 |
+
for family in sorted(os.listdir(root_dir)):
|
| 23 |
+
family_dir = os.path.join(root_dir, family)
|
| 24 |
+
|
| 25 |
+
if not os.path.isdir(family_dir):
|
| 26 |
+
continue
|
| 27 |
+
|
| 28 |
+
images = sorted(
|
| 29 |
+
os.path.join(family_dir, f)
|
| 30 |
+
for f in os.listdir(family_dir)
|
| 31 |
+
if f.lower().endswith(".png")
|
| 32 |
+
)
|
| 33 |
+
|
| 34 |
+
if images:
|
| 35 |
+
family_to_images[family] = images
|
| 36 |
+
|
| 37 |
+
return family_to_images
|
| 38 |
+
|
| 39 |
+
def evaluate_family(
|
| 40 |
+
family: str,
|
| 41 |
+
image_paths: List[str],
|
| 42 |
+
model,
|
| 43 |
+
) -> Tuple[int, int, List[dict]]:
|
| 44 |
+
correct = 0
|
| 45 |
+
total = 0
|
| 46 |
+
details = []
|
| 47 |
+
|
| 48 |
+
for image_path in image_paths:
|
| 49 |
+
try:
|
| 50 |
+
query_embedding = make_embedding(image_path, model).numpy().astype(np.float32).flatten()
|
| 51 |
+
similarity_scores = compare_with_support(query_embedding)
|
| 52 |
+
result = decide_prediction(similarity_scores)
|
| 53 |
+
|
| 54 |
+
predicted_family = result["predicted_family"]
|
| 55 |
+
is_correct = predicted_family == family
|
| 56 |
+
|
| 57 |
+
total += 1
|
| 58 |
+
if is_correct:
|
| 59 |
+
correct += 1
|
| 60 |
+
|
| 61 |
+
details.append({
|
| 62 |
+
"file_name": os.path.basename(image_path),
|
| 63 |
+
"actual_family": family,
|
| 64 |
+
"predicted_family": predicted_family,
|
| 65 |
+
"predicted_label": result["predicted_label"],
|
| 66 |
+
"confidence": result["confidence"],
|
| 67 |
+
"danger_score": result["danger_score"],
|
| 68 |
+
"is_correct": is_correct,
|
| 69 |
+
})
|
| 70 |
+
|
| 71 |
+
status = "OK" if is_correct else "WRONG"
|
| 72 |
+
print(
|
| 73 |
+
f"[{status}] {family} | {os.path.basename(image_path)} "
|
| 74 |
+
f"-> predicted: {predicted_family}, confidence: {result['confidence']}%"
|
| 75 |
+
)
|
| 76 |
+
|
| 77 |
+
except Exception as e:
|
| 78 |
+
print(f"[FAILED] {family} | {os.path.basename(image_path)}: {e}")
|
| 79 |
+
|
| 80 |
+
return correct, total, details
|
| 81 |
+
|
| 82 |
+
def safe_accuracy(correct: int, total: int) -> float:
|
| 83 |
+
if total == 0:
|
| 84 |
+
return 0.0
|
| 85 |
+
return round((correct / total) * 100, 2)
|
| 86 |
+
|
| 87 |
+
def evaluate():
|
| 88 |
+
print("Loading model...")
|
| 89 |
+
model = load_model()
|
| 90 |
+
|
| 91 |
+
print(f"Reading test images from: {TEST_IMAGES_DIR}")
|
| 92 |
+
family_to_images = get_family_image_paths(TEST_IMAGES_DIR)
|
| 93 |
+
|
| 94 |
+
if not family_to_images:
|
| 95 |
+
raise ValueError("No test images found.")
|
| 96 |
+
|
| 97 |
+
overall_correct = 0
|
| 98 |
+
overall_total = 0
|
| 99 |
+
|
| 100 |
+
seen_correct = 0
|
| 101 |
+
seen_total = 0
|
| 102 |
+
|
| 103 |
+
unseen_correct = 0
|
| 104 |
+
unseen_total = 0
|
| 105 |
+
|
| 106 |
+
family_results = {}
|
| 107 |
+
all_details = []
|
| 108 |
+
|
| 109 |
+
print("\nStarting evaluation...\n")
|
| 110 |
+
|
| 111 |
+
for family, image_paths in family_to_images.items():
|
| 112 |
+
print(f"Evaluating family: {family}")
|
| 113 |
+
correct, total, details = evaluate_family(family, image_paths, model)
|
| 114 |
+
|
| 115 |
+
acc = safe_accuracy(correct, total)
|
| 116 |
+
family_results[family] = {
|
| 117 |
+
"correct": correct,
|
| 118 |
+
"total": total,
|
| 119 |
+
"accuracy": acc,
|
| 120 |
+
}
|
| 121 |
+
all_details.extend(details)
|
| 122 |
+
|
| 123 |
+
overall_correct += correct
|
| 124 |
+
overall_total += total
|
| 125 |
+
|
| 126 |
+
if family in SEEN_FAMILIES:
|
| 127 |
+
seen_correct += correct
|
| 128 |
+
seen_total += total
|
| 129 |
+
elif family in UNSEEN_FAMILIES:
|
| 130 |
+
unseen_correct += correct
|
| 131 |
+
unseen_total += total
|
| 132 |
+
|
| 133 |
+
print(f"Family accuracy for {family}: {acc}% ({correct}/{total})\n")
|
| 134 |
+
|
| 135 |
+
overall_accuracy = safe_accuracy(overall_correct, overall_total)
|
| 136 |
+
seen_accuracy = safe_accuracy(seen_correct, seen_total)
|
| 137 |
+
unseen_accuracy = safe_accuracy(unseen_correct, unseen_total)
|
| 138 |
+
|
| 139 |
+
print("\n" + "=" * 60)
|
| 140 |
+
print("FINAL EVALUATION RESULTS")
|
| 141 |
+
print("=" * 60)
|
| 142 |
+
print(f"Overall Accuracy : {overall_accuracy}% ({overall_correct}/{overall_total})")
|
| 143 |
+
print(f"Seen Accuracy : {seen_accuracy}% ({seen_correct}/{seen_total})")
|
| 144 |
+
print(f"Unseen Accuracy : {unseen_accuracy}% ({unseen_correct}/{unseen_total})")
|
| 145 |
+
print("\nPer-Family Accuracy:")
|
| 146 |
+
for family, stats in family_results.items():
|
| 147 |
+
print(f" - {family}: {stats['accuracy']}% ({stats['correct']}/{stats['total']})")
|
| 148 |
+
|
| 149 |
+
# --- Build and Print Confusion Matrix ---
|
| 150 |
+
y_true = [item['actual_family'] for item in all_details]
|
| 151 |
+
y_pred = [item['predicted_family'] for item in all_details]
|
| 152 |
+
|
| 153 |
+
labels = sorted(list(set(y_true + y_pred)))
|
| 154 |
+
cm = confusion_matrix(y_true, y_pred, labels=labels)
|
| 155 |
+
|
| 156 |
+
print("\n" + "=" * 60)
|
| 157 |
+
print("CONFUSION MATRIX")
|
| 158 |
+
print("=" * 60)
|
| 159 |
+
|
| 160 |
+
# Print header row
|
| 161 |
+
header = f"{'':>12}" + "".join(f"{lab:>12}" for lab in labels)
|
| 162 |
+
print(header)
|
| 163 |
+
print("-" * (12 + 12 * len(labels)))
|
| 164 |
+
|
| 165 |
+
# Print each row
|
| 166 |
+
for i, label in enumerate(labels):
|
| 167 |
+
row = f"{label:>12}" + "".join(f"{cm[i][j]:>12}" for j in range(len(labels)))
|
| 168 |
+
print(row)
|
| 169 |
+
|
| 170 |
+
print("-" * (12 + 12 * len(labels)))
|
| 171 |
+
print("(Rows = Actual, Columns = Predicted)")
|
| 172 |
+
|
| 173 |
+
cm_data = cm.tolist()
|
| 174 |
+
|
| 175 |
+
# --- Save JSON ---
|
| 176 |
+
from app.settings import TRAINED_MODELS_DIR
|
| 177 |
+
|
| 178 |
+
results = {
|
| 179 |
+
"overall_accuracy": overall_accuracy,
|
| 180 |
+
"seen_accuracy": seen_accuracy,
|
| 181 |
+
"unseen_accuracy": unseen_accuracy,
|
| 182 |
+
"family_accuracy": {fam: stats["accuracy"] for fam, stats in family_results.items()},
|
| 183 |
+
"final_loss": None,
|
| 184 |
+
"confusion_matrix": {
|
| 185 |
+
"labels": labels,
|
| 186 |
+
"data": cm_data
|
| 187 |
+
}
|
| 188 |
+
}
|
| 189 |
+
|
| 190 |
+
os.makedirs(TRAINED_MODELS_DIR, exist_ok=True)
|
| 191 |
+
output_path = os.path.join(TRAINED_MODELS_DIR, "evaluation_metrics.json")
|
| 192 |
+
with open(output_path, "w") as f:
|
| 193 |
+
json.dump(results, f, indent=2)
|
| 194 |
+
|
| 195 |
+
print(f"\nEvaluation metrics saved to: {output_path}")
|
| 196 |
+
|
| 197 |
+
return results
|
| 198 |
+
|
| 199 |
+
if __name__ == "__main__":
|
| 200 |
+
evaluate()
|
BACKEND/app/inference/make_embedding.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from typing import Optional
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
import torch.nn.functional as F
|
| 6 |
+
from PIL import Image
|
| 7 |
+
from torchvision import transforms
|
| 8 |
+
|
| 9 |
+
from app.models.resnet34_siamese import SiameseResNet34
|
| 10 |
+
from app.settings import IMAGE_SIZE, EMBED_DIM, DEVICE, MODEL_PATH
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
_transform = transforms.Compose([
|
| 14 |
+
transforms.Resize((IMAGE_SIZE, IMAGE_SIZE)),
|
| 15 |
+
transforms.ToTensor(),
|
| 16 |
+
])
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def load_model(model_path: Optional[str] = None) -> SiameseResNet34:
|
| 20 |
+
model_path = model_path or MODEL_PATH
|
| 21 |
+
|
| 22 |
+
model = SiameseResNet34(
|
| 23 |
+
embed_dim=EMBED_DIM,
|
| 24 |
+
pretrained=False,
|
| 25 |
+
freeze_backbone=False,
|
| 26 |
+
).to(DEVICE)
|
| 27 |
+
|
| 28 |
+
state = torch.load(model_path, map_location=DEVICE)
|
| 29 |
+
model.load_state_dict(state)
|
| 30 |
+
model.eval()
|
| 31 |
+
return model
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def image_to_tensor(image_path: str) -> torch.Tensor:
|
| 35 |
+
if not os.path.isfile(image_path):
|
| 36 |
+
raise FileNotFoundError(f"Image not found: {image_path}")
|
| 37 |
+
|
| 38 |
+
img = Image.open(image_path).convert("L")
|
| 39 |
+
tensor = _transform(img).unsqueeze(0) # [1,1,H,W]
|
| 40 |
+
return tensor
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
@torch.no_grad()
|
| 44 |
+
def make_embedding(image_path: str, model: SiameseResNet34) -> torch.Tensor:
|
| 45 |
+
x = image_to_tensor(image_path).to(DEVICE)
|
| 46 |
+
emb = model.forward_once(x)
|
| 47 |
+
emb = F.normalize(emb, p=2, dim=1)
|
| 48 |
+
return emb.squeeze(0).cpu()
|
BACKEND/app/inference/scan_user_apk.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import uuid
|
| 3 |
+
import shutil
|
| 4 |
+
import numpy as np
|
| 5 |
+
|
| 6 |
+
from app.static_analysis.apk_analyzer import extract_permissions, extract_api_calls
|
| 7 |
+
from app.preprocessing.apk_pipeline import apk_to_image_pipeline
|
| 8 |
+
from app.inference.make_embedding import load_model, make_embedding
|
| 9 |
+
from app.inference.compare_support import compare_with_support
|
| 10 |
+
from app.inference.decide_result import decide_prediction
|
| 11 |
+
from app.settings import UPLOADS_DIR, OUTPUTS_DIR
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def ensure_dir(path: str):
|
| 15 |
+
os.makedirs(path, exist_ok=True)
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def scan_user_apk(apk_file_path: str):
|
| 19 |
+
ensure_dir(UPLOADS_DIR)
|
| 20 |
+
ensure_dir(OUTPUTS_DIR)
|
| 21 |
+
|
| 22 |
+
base_id = str(uuid.uuid4())
|
| 23 |
+
temp_dir = os.path.join(UPLOADS_DIR, f"temp_{base_id}")
|
| 24 |
+
output_image_path = os.path.join(OUTPUTS_DIR, f"{base_id}.png")
|
| 25 |
+
|
| 26 |
+
processing_steps = [
|
| 27 |
+
"APK uploaded successfully",
|
| 28 |
+
"Extracting classes.dex",
|
| 29 |
+
"Generating grayscale image",
|
| 30 |
+
"Generating embedding",
|
| 31 |
+
"Comparing with support families",
|
| 32 |
+
"Preparing final prediction",
|
| 33 |
+
]
|
| 34 |
+
|
| 35 |
+
try:
|
| 36 |
+
apk_to_image_pipeline(
|
| 37 |
+
apk_path=apk_file_path,
|
| 38 |
+
temp_dir=temp_dir,
|
| 39 |
+
output_image_path=output_image_path,
|
| 40 |
+
final_size=(224, 224),
|
| 41 |
+
)
|
| 42 |
+
|
| 43 |
+
model = load_model()
|
| 44 |
+
query_embedding = make_embedding(output_image_path, model).numpy().astype(np.float32).flatten()
|
| 45 |
+
|
| 46 |
+
similarity_scores = compare_with_support(query_embedding)
|
| 47 |
+
result = decide_prediction(similarity_scores)
|
| 48 |
+
|
| 49 |
+
# Extract static analysis data (permissions and API calls) – inside the function
|
| 50 |
+
permissions = extract_permissions(apk_file_path)
|
| 51 |
+
api_calls = extract_api_calls(apk_file_path)
|
| 52 |
+
|
| 53 |
+
return {
|
| 54 |
+
"success": True,
|
| 55 |
+
"file_name": os.path.basename(apk_file_path),
|
| 56 |
+
"predicted_family": result["predicted_family"],
|
| 57 |
+
"predicted_label": result["predicted_label"],
|
| 58 |
+
"confidence": result["confidence"],
|
| 59 |
+
"danger_score": result["danger_score"],
|
| 60 |
+
"model_name": "Siamese ResNet34",
|
| 61 |
+
"grayscale_image_url": f"/outputs/{os.path.basename(output_image_path)}",
|
| 62 |
+
"family_matches": result["family_matches"],
|
| 63 |
+
"processing_steps": processing_steps,
|
| 64 |
+
"permissions": permissions,
|
| 65 |
+
"api_calls": api_calls
|
| 66 |
+
}
|
| 67 |
+
|
| 68 |
+
finally:
|
| 69 |
+
if os.path.isdir(temp_dir):
|
| 70 |
+
shutil.rmtree(temp_dir, ignore_errors=True)
|
BACKEND/app/main.py
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import shutil
|
| 3 |
+
import uuid
|
| 4 |
+
import json
|
| 5 |
+
import pickle
|
| 6 |
+
from fastapi import FastAPI, File, UploadFile, HTTPException
|
| 7 |
+
from fastapi.staticfiles import StaticFiles
|
| 8 |
+
from fastapi.middleware.cors import CORSMiddleware
|
| 9 |
+
from fastapi.responses import FileResponse
|
| 10 |
+
from app.schemas import ScanResponse
|
| 11 |
+
from app.settings import UPLOADS_DIR, OUTPUTS_DIR, TRAINED_MODELS_DIR, DATA_DIR, SUPPORT_EMBEDDINGS_PATH
|
| 12 |
+
from app.inference.scan_user_apk import scan_user_apk
|
| 13 |
+
from app.database import engine, Base
|
| 14 |
+
from app.routers import auth, history, local_report, sandbox
|
| 15 |
+
|
| 16 |
+
# ==================== 1. APP INITIALIZATION ====================
|
| 17 |
+
app = FastAPI(title="Android Malware FSL API")
|
| 18 |
+
|
| 19 |
+
# ==================== 2. CORS MIDDLEWARE ====================
|
| 20 |
+
app.add_middleware(
|
| 21 |
+
CORSMiddleware,
|
| 22 |
+
allow_origins=["http://localhost:3000", "http://127.0.0.1:3000", "*"],
|
| 23 |
+
allow_credentials=True,
|
| 24 |
+
allow_methods=["*"],
|
| 25 |
+
allow_headers=["*"],
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
# ==================== 3. DIRECTORY SETUP ====================
|
| 29 |
+
os.makedirs(UPLOADS_DIR, exist_ok=True)
|
| 30 |
+
os.makedirs(OUTPUTS_DIR, exist_ok=True)
|
| 31 |
+
|
| 32 |
+
# Static files mount for outputs (PDFs, images)
|
| 33 |
+
app.mount("/outputs", StaticFiles(directory=OUTPUTS_DIR), name="outputs")
|
| 34 |
+
|
| 35 |
+
# ==================== 4. DATABASE SETUP ====================
|
| 36 |
+
Base.metadata.create_all(bind=engine)
|
| 37 |
+
|
| 38 |
+
# ==================== 5. INCLUDE ROUTERS (API ENDPOINTS) ====================
|
| 39 |
+
app.include_router(auth.router)
|
| 40 |
+
app.include_router(history.router)
|
| 41 |
+
app.include_router(local_report.router)
|
| 42 |
+
app.include_router(sandbox.router)
|
| 43 |
+
|
| 44 |
+
# ==================== 6. API ENDPOINTS ====================
|
| 45 |
+
@app.get("/")
|
| 46 |
+
def root():
|
| 47 |
+
return {"message": "Android Malware FSL backend is running"}
|
| 48 |
+
|
| 49 |
+
@app.get("/health")
|
| 50 |
+
def health():
|
| 51 |
+
return {"status": "ok"}
|
| 52 |
+
|
| 53 |
+
@app.post("/scan", response_model=ScanResponse)
|
| 54 |
+
async def scan(apk: UploadFile = File(...)):
|
| 55 |
+
if not apk.filename:
|
| 56 |
+
raise HTTPException(status_code=400, detail="No file uploaded")
|
| 57 |
+
unique_name = f"{uuid.uuid4()}_{apk.filename}"
|
| 58 |
+
apk_path = os.path.join(UPLOADS_DIR, unique_name)
|
| 59 |
+
try:
|
| 60 |
+
with open(apk_path, "wb") as f:
|
| 61 |
+
shutil.copyfileobj(apk.file, f)
|
| 62 |
+
result = scan_user_apk(apk_path)
|
| 63 |
+
return result
|
| 64 |
+
except Exception as e:
|
| 65 |
+
raise HTTPException(status_code=500, detail=str(e))
|
| 66 |
+
|
| 67 |
+
@app.get("/api/train-history")
|
| 68 |
+
async def get_train_history():
|
| 69 |
+
history_path = os.path.join(TRAINED_MODELS_DIR, "train_history.json")
|
| 70 |
+
if not os.path.isfile(history_path):
|
| 71 |
+
return []
|
| 72 |
+
with open(history_path, "r") as f:
|
| 73 |
+
return json.load(f)
|
| 74 |
+
|
| 75 |
+
@app.get("/api/evaluation-metrics")
|
| 76 |
+
async def get_evaluation_metrics():
|
| 77 |
+
metrics_path = os.path.join(TRAINED_MODELS_DIR, "evaluation_metrics.json")
|
| 78 |
+
if not os.path.isfile(metrics_path):
|
| 79 |
+
return {
|
| 80 |
+
"overall_accuracy": 0.0,
|
| 81 |
+
"seen_accuracy": 0.0,
|
| 82 |
+
"unseen_accuracy": 0.0,
|
| 83 |
+
"family_accuracy": {},
|
| 84 |
+
"final_loss": None
|
| 85 |
+
}
|
| 86 |
+
with open(metrics_path, "r") as f:
|
| 87 |
+
return json.load(f)
|
| 88 |
+
|
| 89 |
+
@app.get("/api/dashboard-stats")
|
| 90 |
+
async def dashboard_stats():
|
| 91 |
+
try:
|
| 92 |
+
raw_dir = os.path.join(DATA_DIR, "raw_apks")
|
| 93 |
+
families = ["benign", "banking", "smsware", "adware", "riskware"]
|
| 94 |
+
raw_apks = {}
|
| 95 |
+
for family in families:
|
| 96 |
+
family_dir = os.path.join(raw_dir, family)
|
| 97 |
+
if os.path.isdir(family_dir):
|
| 98 |
+
raw_apks[family] = len([f for f in os.listdir(family_dir) if os.path.isfile(os.path.join(family_dir, f))])
|
| 99 |
+
else:
|
| 100 |
+
raw_apks[family] = 0
|
| 101 |
+
|
| 102 |
+
splits = ["train_images", "test_images", "support_set"]
|
| 103 |
+
image_splits = {}
|
| 104 |
+
split_totals = {}
|
| 105 |
+
for split in splits:
|
| 106 |
+
split_dir = os.path.join(DATA_DIR, split)
|
| 107 |
+
image_splits[split] = {}
|
| 108 |
+
total = 0
|
| 109 |
+
for family in families:
|
| 110 |
+
family_dir = os.path.join(split_dir, family)
|
| 111 |
+
if os.path.isdir(family_dir):
|
| 112 |
+
count = len([f for f in os.listdir(family_dir) if f.endswith(".png")])
|
| 113 |
+
else:
|
| 114 |
+
count = 0
|
| 115 |
+
image_splits[split][family] = count
|
| 116 |
+
total += count
|
| 117 |
+
split_totals[split] = total
|
| 118 |
+
|
| 119 |
+
embedding_count = 0
|
| 120 |
+
if os.path.isfile(SUPPORT_EMBEDDINGS_PATH):
|
| 121 |
+
try:
|
| 122 |
+
with open(SUPPORT_EMBEDDINGS_PATH, "rb") as f:
|
| 123 |
+
data = pickle.load(f)
|
| 124 |
+
embedding_count = sum(len(v) for v in data.values())
|
| 125 |
+
except:
|
| 126 |
+
pass
|
| 127 |
+
|
| 128 |
+
training_history = await get_train_history()
|
| 129 |
+
evaluation = await get_evaluation_metrics()
|
| 130 |
+
|
| 131 |
+
return {
|
| 132 |
+
"success": True,
|
| 133 |
+
"raw_apks": raw_apks,
|
| 134 |
+
"image_splits": image_splits,
|
| 135 |
+
"split_totals": split_totals,
|
| 136 |
+
"embedding_count": embedding_count,
|
| 137 |
+
"training_history": training_history,
|
| 138 |
+
"evaluation": evaluation,
|
| 139 |
+
}
|
| 140 |
+
except Exception as e:
|
| 141 |
+
raise HTTPException(status_code=500, detail=str(e))
|
| 142 |
+
|
| 143 |
+
# ==================== 7. SERVE REACT STATIC FILES (SPA) ====================
|
| 144 |
+
# Yeh path Dockerfile ke mutabiq set kiya gaya hai.
|
| 145 |
+
# Build folder frontend_build/ mein aayega (jo backend root ke level par hai).
|
| 146 |
+
FRONTEND_BUILD_DIR = os.path.abspath(os.path.join(os.path.dirname(os.path.dirname(__file__)), "frontend_build"))
|
| 147 |
+
INDEX_HTML = os.path.join(FRONTEND_BUILD_DIR, "index.html")
|
| 148 |
+
|
| 149 |
+
# Agar frontend build exist karta hai, to static assets mount karein
|
| 150 |
+
if os.path.exists(FRONTEND_BUILD_DIR):
|
| 151 |
+
# React ke static assets (JS, CSS, images) ko mount karein
|
| 152 |
+
app.mount("/assets", StaticFiles(directory=os.path.join(FRONTEND_BUILD_DIR, "assets")), name="react_assets")
|
| 153 |
+
|
| 154 |
+
# YEH CATCH-ALL ROUTE SAB SE AKHRI MEIN AAYEGA.
|
| 155 |
+
# Is liye yeh API routes (/scan, /api/*, /auth/*, /health) ko interfere nahi karega.
|
| 156 |
+
@app.get("/{full_path:path}")
|
| 157 |
+
async def serve_react_app(full_path: str):
|
| 158 |
+
"""
|
| 159 |
+
Agar request kisi API route (jaise /api, /auth, /health) se match nahi karti,
|
| 160 |
+
toh React ka index.html serve karo taake React Router frontend sambhal sake.
|
| 161 |
+
"""
|
| 162 |
+
# Agar frontend build folder hi nahi mila, toh error bhejein
|
| 163 |
+
if not os.path.exists(INDEX_HTML):
|
| 164 |
+
return {"error": "Frontend not built. Please run 'npm run build' and copy to frontend_build/"}
|
| 165 |
+
|
| 166 |
+
# Agar koi static file (jaise .js, .css, .png) request ho rahi hai
|
| 167 |
+
# aur wo build folder mein mojood hai, toh use serve karein
|
| 168 |
+
file_path = os.path.join(FRONTEND_BUILD_DIR, full_path)
|
| 169 |
+
if os.path.isfile(file_path):
|
| 170 |
+
return FileResponse(file_path)
|
| 171 |
+
|
| 172 |
+
# Baqi sab routes (jaise /dashboard, /login, /analysis) ke liye index.html bhejo
|
| 173 |
+
return FileResponse(INDEX_HTML)
|
| 174 |
+
|
| 175 |
+
# ==================== 8. (OPTIONAL) ROOT REDIRECT ====================
|
| 176 |
+
# Agar koi root (/) par aaye aur React build mojood hai, toh index.html serve karein.
|
| 177 |
+
# Lekin upar @app.get("/") already API response de raha hai.
|
| 178 |
+
# Agar aap chahte hain ke root par React chale, toh @app.get("/") ko comment kar dein.
|
| 179 |
+
# Main ne API root rakhna behtar samjha, kyunki backend health check ke liye zaroori hai.
|
BACKEND/app/models/resnet34_siamese.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
from torchvision import models
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class ResNet34Embedding(nn.Module):
|
| 7 |
+
def __init__(
|
| 8 |
+
self,
|
| 9 |
+
embed_dim: int = 128,
|
| 10 |
+
pretrained: bool = False,
|
| 11 |
+
freeze_backbone: bool = False,
|
| 12 |
+
):
|
| 13 |
+
super().__init__()
|
| 14 |
+
|
| 15 |
+
backbone = models.resnet34(
|
| 16 |
+
weights=models.ResNet34_Weights.DEFAULT if pretrained else None
|
| 17 |
+
)
|
| 18 |
+
|
| 19 |
+
old_conv = backbone.conv1
|
| 20 |
+
backbone.conv1 = nn.Conv2d(
|
| 21 |
+
in_channels=1,
|
| 22 |
+
out_channels=old_conv.out_channels,
|
| 23 |
+
kernel_size=old_conv.kernel_size,
|
| 24 |
+
stride=old_conv.stride,
|
| 25 |
+
padding=old_conv.padding,
|
| 26 |
+
bias=False,
|
| 27 |
+
)
|
| 28 |
+
|
| 29 |
+
if pretrained:
|
| 30 |
+
with torch.no_grad():
|
| 31 |
+
backbone.conv1.weight.copy_(old_conv.weight.mean(dim=1, keepdim=True))
|
| 32 |
+
|
| 33 |
+
self.feature_extractor = nn.Sequential(*list(backbone.children())[:-1])
|
| 34 |
+
|
| 35 |
+
self.embedding_head = nn.Sequential(
|
| 36 |
+
nn.Flatten(),
|
| 37 |
+
nn.Linear(512, 256),
|
| 38 |
+
nn.ReLU(inplace=True),
|
| 39 |
+
nn.Dropout(0.2),
|
| 40 |
+
nn.Linear(256, embed_dim),
|
| 41 |
+
)
|
| 42 |
+
|
| 43 |
+
if freeze_backbone:
|
| 44 |
+
for param in self.feature_extractor.parameters():
|
| 45 |
+
param.requires_grad = False
|
| 46 |
+
|
| 47 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 48 |
+
x = self.feature_extractor(x)
|
| 49 |
+
x = self.embedding_head(x)
|
| 50 |
+
return x
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
class SiameseResNet34(nn.Module):
|
| 54 |
+
def __init__(
|
| 55 |
+
self,
|
| 56 |
+
embed_dim: int = 128,
|
| 57 |
+
pretrained: bool = False,
|
| 58 |
+
freeze_backbone: bool = False,
|
| 59 |
+
):
|
| 60 |
+
super().__init__()
|
| 61 |
+
self.encoder = ResNet34Embedding(
|
| 62 |
+
embed_dim=embed_dim,
|
| 63 |
+
pretrained=pretrained,
|
| 64 |
+
freeze_backbone=freeze_backbone,
|
| 65 |
+
)
|
| 66 |
+
|
| 67 |
+
def forward_once(self, x: torch.Tensor) -> torch.Tensor:
|
| 68 |
+
return self.encoder(x)
|
| 69 |
+
|
| 70 |
+
def forward(self, x1: torch.Tensor, x2: torch.Tensor):
|
| 71 |
+
e1 = self.forward_once(x1)
|
| 72 |
+
e2 = self.forward_once(x2)
|
| 73 |
+
return e1, e2
|
BACKEND/app/preprocessing/apk_pipeline.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import shutil
|
| 3 |
+
from typing import Tuple
|
| 4 |
+
|
| 5 |
+
from .dex_utils import extract_primary_dex_from_apk
|
| 6 |
+
from .apk_to_grayscale import dex_to_grayscale_image
|
| 7 |
+
from .resize_utils import resize_image
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def apk_to_image_pipeline(
|
| 11 |
+
apk_path: str,
|
| 12 |
+
temp_dir: str,
|
| 13 |
+
output_image_path: str,
|
| 14 |
+
final_size: Tuple[int, int] = (300, 300),
|
| 15 |
+
) -> str:
|
| 16 |
+
"""
|
| 17 |
+
Full pipeline:
|
| 18 |
+
APK -> extract classes.dex -> raw grayscale image -> resized grayscale image
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
if not os.path.isfile(apk_path):
|
| 22 |
+
raise FileNotFoundError(f"APK not found: {apk_path}")
|
| 23 |
+
|
| 24 |
+
os.makedirs(temp_dir, exist_ok=True)
|
| 25 |
+
os.makedirs(os.path.dirname(output_image_path), exist_ok=True)
|
| 26 |
+
|
| 27 |
+
dex_extract_dir = os.path.join(temp_dir, "unzipped")
|
| 28 |
+
raw_gray_path = os.path.join(temp_dir, "raw_gray.png")
|
| 29 |
+
|
| 30 |
+
# Clean previous temp data
|
| 31 |
+
if os.path.exists(dex_extract_dir):
|
| 32 |
+
shutil.rmtree(dex_extract_dir, ignore_errors=True)
|
| 33 |
+
|
| 34 |
+
dex_path = extract_primary_dex_from_apk(apk_path, dex_extract_dir)
|
| 35 |
+
dex_to_grayscale_image(dex_path, raw_gray_path)
|
| 36 |
+
resize_image(raw_gray_path, output_image_path, size=final_size)
|
| 37 |
+
|
| 38 |
+
return output_image_path
|
BACKEND/app/preprocessing/apk_to_grayscale.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import math
|
| 2 |
+
import os
|
| 3 |
+
from typing import Tuple
|
| 4 |
+
|
| 5 |
+
import numpy as np
|
| 6 |
+
from PIL import Image
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def calculate_image_width(file_size: int) -> int:
|
| 10 |
+
"""
|
| 11 |
+
Common practical width selection based on file size.
|
| 12 |
+
This gives more stable image shapes than pure sqrt for many APKs.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
if file_size < 10_000:
|
| 16 |
+
return 32
|
| 17 |
+
if file_size < 30_000:
|
| 18 |
+
return 64
|
| 19 |
+
if file_size < 60_000:
|
| 20 |
+
return 128
|
| 21 |
+
if file_size < 100_000:
|
| 22 |
+
return 256
|
| 23 |
+
if file_size < 200_000:
|
| 24 |
+
return 384
|
| 25 |
+
if file_size < 500_000:
|
| 26 |
+
return 512
|
| 27 |
+
if file_size < 1_000_000:
|
| 28 |
+
return 768
|
| 29 |
+
return 1024
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def dex_to_grayscale_array(dex_path: str) -> np.ndarray:
|
| 33 |
+
"""
|
| 34 |
+
Read a .dex file as raw bytes and convert to 2D uint8 grayscale array.
|
| 35 |
+
"""
|
| 36 |
+
|
| 37 |
+
if not os.path.isfile(dex_path):
|
| 38 |
+
raise FileNotFoundError(f"DEX file not found: {dex_path}")
|
| 39 |
+
|
| 40 |
+
with open(dex_path, "rb") as f:
|
| 41 |
+
byte_data = f.read()
|
| 42 |
+
|
| 43 |
+
if not byte_data:
|
| 44 |
+
raise Exception(f"DEX file is empty: {dex_path}")
|
| 45 |
+
|
| 46 |
+
byte_array = np.frombuffer(byte_data, dtype=np.uint8)
|
| 47 |
+
|
| 48 |
+
width = calculate_image_width(len(byte_array))
|
| 49 |
+
height = math.ceil(len(byte_array) / width)
|
| 50 |
+
|
| 51 |
+
padded_size = width * height
|
| 52 |
+
padded_array = np.pad(
|
| 53 |
+
byte_array,
|
| 54 |
+
(0, padded_size - len(byte_array)),
|
| 55 |
+
mode="constant",
|
| 56 |
+
constant_values=0,
|
| 57 |
+
)
|
| 58 |
+
|
| 59 |
+
image_array = padded_array.reshape((height, width))
|
| 60 |
+
return image_array
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def dex_to_grayscale_image(dex_path: str, output_image_path: str) -> str:
|
| 64 |
+
"""
|
| 65 |
+
Convert DEX file to grayscale PNG image and save it.
|
| 66 |
+
"""
|
| 67 |
+
|
| 68 |
+
os.makedirs(os.path.dirname(output_image_path), exist_ok=True)
|
| 69 |
+
|
| 70 |
+
image_array = dex_to_grayscale_array(dex_path)
|
| 71 |
+
img = Image.fromarray(image_array, mode="L")
|
| 72 |
+
img.save(output_image_path)
|
| 73 |
+
|
| 74 |
+
return output_image_path
|
BACKEND/app/preprocessing/dex_utils.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import zipfile
|
| 3 |
+
from typing import List
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def extract_dex_files_from_apk(apk_path: str, extract_dir: str) -> List[str]:
|
| 7 |
+
"""
|
| 8 |
+
Extract all .dex files from an APK into extract_dir.
|
| 9 |
+
Returns list of extracted dex file paths.
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
if not os.path.isfile(apk_path):
|
| 13 |
+
raise FileNotFoundError(f"APK file not found: {apk_path}")
|
| 14 |
+
|
| 15 |
+
os.makedirs(extract_dir, exist_ok=True)
|
| 16 |
+
extracted_dex_paths: List[str] = []
|
| 17 |
+
|
| 18 |
+
try:
|
| 19 |
+
with zipfile.ZipFile(apk_path, "r") as apk_zip:
|
| 20 |
+
for member in apk_zip.namelist():
|
| 21 |
+
if member.lower().endswith(".dex"):
|
| 22 |
+
apk_zip.extract(member, extract_dir)
|
| 23 |
+
extracted_path = os.path.join(extract_dir, member)
|
| 24 |
+
extracted_dex_paths.append(extracted_path)
|
| 25 |
+
except zipfile.BadZipFile as e:
|
| 26 |
+
raise Exception(f"Invalid or corrupted APK: {apk_path}") from e
|
| 27 |
+
except Exception as e:
|
| 28 |
+
raise Exception(f"Failed to extract APK: {apk_path}") from e
|
| 29 |
+
|
| 30 |
+
if not extracted_dex_paths:
|
| 31 |
+
raise Exception(f"No .dex file found in APK: {apk_path}")
|
| 32 |
+
|
| 33 |
+
return extracted_dex_paths
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def extract_primary_dex_from_apk(apk_path: str, extract_dir: str) -> str:
|
| 37 |
+
"""
|
| 38 |
+
Extract only primary classes.dex if present.
|
| 39 |
+
Otherwise return the first dex file found.
|
| 40 |
+
"""
|
| 41 |
+
|
| 42 |
+
dex_files = extract_dex_files_from_apk(apk_path, extract_dir)
|
| 43 |
+
|
| 44 |
+
for dex_path in dex_files:
|
| 45 |
+
if os.path.basename(dex_path).lower() == "classes.dex":
|
| 46 |
+
return dex_path
|
| 47 |
+
|
| 48 |
+
return dex_files[0]
|
BACKEND/app/preprocessing/resize_utils.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from typing import Tuple
|
| 3 |
+
|
| 4 |
+
from PIL import Image
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def resize_image(
|
| 8 |
+
input_path: str,
|
| 9 |
+
output_path: str,
|
| 10 |
+
size: Tuple[int, int] = (300, 300),
|
| 11 |
+
) -> str:
|
| 12 |
+
"""
|
| 13 |
+
Resize grayscale image to fixed size for model input.
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
if not os.path.isfile(input_path):
|
| 17 |
+
raise FileNotFoundError(f"Image not found: {input_path}")
|
| 18 |
+
|
| 19 |
+
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
| 20 |
+
|
| 21 |
+
with Image.open(input_path) as img:
|
| 22 |
+
img = img.convert("L")
|
| 23 |
+
img = img.resize(size)
|
| 24 |
+
img.save(output_path)
|
| 25 |
+
|
| 26 |
+
return output_path
|
BACKEND/app/routers/auth.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter, Depends, HTTPException, status
|
| 2 |
+
from sqlalchemy.orm import Session
|
| 3 |
+
from datetime import datetime, timedelta
|
| 4 |
+
import secrets
|
| 5 |
+
|
| 6 |
+
from app.database import get_db
|
| 7 |
+
from app.db_models import User, PasswordResetToken
|
| 8 |
+
from app.schemas import UserCreate, UserLogin, Token, PasswordResetRequest, PasswordResetConfirm
|
| 9 |
+
from app.auth import verify_password, get_password_hash, create_access_token, send_reset_email, get_current_user
|
| 10 |
+
|
| 11 |
+
router = APIRouter(prefix="/auth", tags=["authentication"])
|
| 12 |
+
|
| 13 |
+
@router.post("/signup", response_model=Token)
|
| 14 |
+
def signup(user: UserCreate, db: Session = Depends(get_db)):
|
| 15 |
+
# Check if user exists
|
| 16 |
+
existing = db.query(User).filter(User.email == user.email).first()
|
| 17 |
+
if existing:
|
| 18 |
+
raise HTTPException(status_code=400, detail="Email already registered")
|
| 19 |
+
|
| 20 |
+
hashed = get_password_hash(user.password)
|
| 21 |
+
db_user = User(
|
| 22 |
+
email=user.email,
|
| 23 |
+
hashed_password=hashed,
|
| 24 |
+
role=user.role,
|
| 25 |
+
full_name=user.full_name,
|
| 26 |
+
university=user.university,
|
| 27 |
+
organization=user.organization,
|
| 28 |
+
org_details=user.org_details
|
| 29 |
+
)
|
| 30 |
+
db.add(db_user)
|
| 31 |
+
db.commit()
|
| 32 |
+
db.refresh(db_user)
|
| 33 |
+
|
| 34 |
+
# Create access token
|
| 35 |
+
access_token = create_access_token(data={"sub": db_user.email, "role": db_user.role})
|
| 36 |
+
return {"access_token": access_token, "token_type": "bearer"}
|
| 37 |
+
|
| 38 |
+
@router.post("/login", response_model=Token)
|
| 39 |
+
def login(user: UserLogin, db: Session = Depends(get_db)):
|
| 40 |
+
db_user = db.query(User).filter(User.email == user.email).first()
|
| 41 |
+
if not db_user or not verify_password(user.password, db_user.hashed_password):
|
| 42 |
+
raise HTTPException(status_code=401, detail="Invalid credentials")
|
| 43 |
+
|
| 44 |
+
access_token = create_access_token(data={"sub": db_user.email, "role": db_user.role})
|
| 45 |
+
return {"access_token": access_token, "token_type": "bearer"}
|
| 46 |
+
|
| 47 |
+
# Forgot password (optional – keep as before)
|
| 48 |
+
@router.post("/forgot-password")
|
| 49 |
+
def forgot_password(request: PasswordResetRequest, db: Session = Depends(get_db)):
|
| 50 |
+
user = db.query(User).filter(User.email == request.email).first()
|
| 51 |
+
if not user:
|
| 52 |
+
return {"message": "If that email exists, a reset link has been sent"}
|
| 53 |
+
|
| 54 |
+
token = secrets.token_urlsafe(32)
|
| 55 |
+
expires = datetime.utcnow() + timedelta(hours=1)
|
| 56 |
+
db.query(PasswordResetToken).filter(PasswordResetToken.user_id == user.id).delete()
|
| 57 |
+
db_token = PasswordResetToken(user_id=user.id, token=token, expires_at=expires)
|
| 58 |
+
db.add(db_token)
|
| 59 |
+
db.commit()
|
| 60 |
+
send_reset_email(user.email, token)
|
| 61 |
+
return {"message": "Password reset link sent to your email"}
|
| 62 |
+
|
| 63 |
+
@router.post("/reset-password")
|
| 64 |
+
def reset_password(confirm: PasswordResetConfirm, db: Session = Depends(get_db)):
|
| 65 |
+
token_entry = db.query(PasswordResetToken).filter(PasswordResetToken.token == confirm.token).first()
|
| 66 |
+
if not token_entry or token_entry.expires_at < datetime.utcnow():
|
| 67 |
+
raise HTTPException(status_code=400, detail="Invalid or expired token")
|
| 68 |
+
|
| 69 |
+
user = db.query(User).filter(User.id == token_entry.user_id).first()
|
| 70 |
+
if not user:
|
| 71 |
+
raise HTTPException(status_code=400, detail="User not found")
|
| 72 |
+
|
| 73 |
+
user.hashed_password = get_password_hash(confirm.new_password)
|
| 74 |
+
db.delete(token_entry)
|
| 75 |
+
db.commit()
|
| 76 |
+
return {"message": "Password updated successfully"}
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
@router.get("/me")
|
| 80 |
+
def get_me(current_user: User = Depends(get_current_user)):
|
| 81 |
+
return {
|
| 82 |
+
"id": current_user.id,
|
| 83 |
+
"email": current_user.email,
|
| 84 |
+
"full_name": current_user.full_name,
|
| 85 |
+
"role": current_user.role,
|
| 86 |
+
"university": current_user.university,
|
| 87 |
+
"organization": current_user.organization,
|
| 88 |
+
}
|
BACKEND/app/routers/history.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter, Depends, HTTPException, Query
|
| 2 |
+
from sqlalchemy.orm import Session
|
| 3 |
+
from app.database import get_db
|
| 4 |
+
from app.db_models import ScanHistory, User
|
| 5 |
+
from app.auth import get_current_user
|
| 6 |
+
|
| 7 |
+
router = APIRouter(prefix="/history", tags=["history"])
|
| 8 |
+
|
| 9 |
+
@router.get("/")
|
| 10 |
+
def get_history(
|
| 11 |
+
skip: int = 0,
|
| 12 |
+
limit: int = 50,
|
| 13 |
+
threat_type: str = Query(None, regex="^(malware|benign)$"),
|
| 14 |
+
search: str = None,
|
| 15 |
+
db: Session = Depends(get_db),
|
| 16 |
+
current_user: User = Depends(get_current_user)
|
| 17 |
+
):
|
| 18 |
+
query = db.query(ScanHistory).filter(ScanHistory.user_id == current_user.id)
|
| 19 |
+
if threat_type:
|
| 20 |
+
query = query.filter(ScanHistory.predicted_label == threat_type)
|
| 21 |
+
if search:
|
| 22 |
+
query = query.filter(ScanHistory.file_name.contains(search))
|
| 23 |
+
results = query.order_by(ScanHistory.scanned_at.desc()).offset(skip).limit(limit).all()
|
| 24 |
+
return results
|
| 25 |
+
|
| 26 |
+
@router.post("/save")
|
| 27 |
+
def save_scan(scan_data: dict, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)):
|
| 28 |
+
history = ScanHistory(
|
| 29 |
+
user_id=current_user.id,
|
| 30 |
+
file_name=scan_data.get("file_name"),
|
| 31 |
+
predicted_family=scan_data.get("predicted_family"),
|
| 32 |
+
predicted_label=scan_data.get("predicted_label"),
|
| 33 |
+
confidence=scan_data.get("confidence"),
|
| 34 |
+
danger_score=scan_data.get("danger_score"),
|
| 35 |
+
permissions=scan_data.get("permissions", []),
|
| 36 |
+
api_calls=scan_data.get("api_calls", []),
|
| 37 |
+
full_response=scan_data
|
| 38 |
+
)
|
| 39 |
+
db.add(history)
|
| 40 |
+
db.commit()
|
| 41 |
+
return {"message": "Saved"}
|
BACKEND/app/routers/local_report.py
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
from fastapi import APIRouter
|
| 3 |
+
from pydantic import BaseModel
|
| 4 |
+
from typing import Dict, Any
|
| 5 |
+
|
| 6 |
+
router = APIRouter(prefix="/local-report", tags=["Local Report Generation"])
|
| 7 |
+
|
| 8 |
+
class ScanResultPayload(BaseModel):
|
| 9 |
+
result: Dict[str, Any]
|
| 10 |
+
|
| 11 |
+
def generate_detailed_report(scan: Dict[str, Any]) -> Dict[str, str]:
|
| 12 |
+
"""Generate a rich, human-readable report from the scan data."""
|
| 13 |
+
# Basic info
|
| 14 |
+
label = scan.get("predicted_label", "unknown").upper()
|
| 15 |
+
confidence = scan.get("confidence", 0)
|
| 16 |
+
danger = scan.get("danger_score", 0)
|
| 17 |
+
family = scan.get("predicted_family", "unknown")
|
| 18 |
+
|
| 19 |
+
# Risk assessment paragraph
|
| 20 |
+
if label == "MALWARE":
|
| 21 |
+
risk = (f"The APK is classified as **malware** with {confidence:.2f}% confidence and a danger score of {danger:.2f}%. "
|
| 22 |
+
f"This indicates a high probability of malicious behaviour. The predicted family is '{family}', "
|
| 23 |
+
f"which typically performs harmful actions such as data theft, SMS fraud, or device control. "
|
| 24 |
+
f"Immediate action is recommended.")
|
| 25 |
+
else:
|
| 26 |
+
risk = (f"The APK is classified as **benign** with {confidence:.2f}% confidence. The danger score is {danger:.2f}%, "
|
| 27 |
+
f"indicating low risk. The predicted family is '{family}', which generally does not exhibit malicious behaviour. "
|
| 28 |
+
f"However, always review requested permissions and API calls for any anomalies.")
|
| 29 |
+
|
| 30 |
+
# Malware classification paragraph
|
| 31 |
+
if label == "MALWARE":
|
| 32 |
+
classification = (f"The model predicts this APK belongs to the '{family}' malware family with {confidence:.2f}% confidence. "
|
| 33 |
+
f"Families like '{family}' are known for specific malicious patterns. For instance, banking trojans may steal credentials, "
|
| 34 |
+
f"while SMS malware sends premium messages. The confidence score suggests the model is "
|
| 35 |
+
f"{'highly' if confidence > 80 else 'moderately'} certain about this classification.")
|
| 36 |
+
else:
|
| 37 |
+
classification = (f"The APK is classified as benign. The model predicts it belongs to the '{family}' family, "
|
| 38 |
+
f"which is not associated with known malware behaviour. The confidence score of {confidence:.2f}% indicates "
|
| 39 |
+
f"{'strong' if confidence > 80 else 'reasonable'} certainty that this app is safe.")
|
| 40 |
+
|
| 41 |
+
# Permissions analysis
|
| 42 |
+
perms = scan.get("permissions", [])
|
| 43 |
+
if perms:
|
| 44 |
+
perm_list = "\n".join(perms)
|
| 45 |
+
perm_analysis = f"The APK requests the following permissions:\n{perm_list}\n\n"
|
| 46 |
+
dangerous = [p for p in perms if any(d in p for d in ["SEND_SMS", "READ_SMS", "READ_CONTACTS", "ACCESS_FINE_LOCATION", "CAMERA", "RECORD_AUDIO"])]
|
| 47 |
+
if dangerous:
|
| 48 |
+
perm_analysis += f"Dangerous permissions detected: {', '.join(dangerous)}. These can lead to privacy breaches or financial loss."
|
| 49 |
+
else:
|
| 50 |
+
perm_analysis += "No highly dangerous permissions (SMS, contacts, location, camera, microphone) are requested."
|
| 51 |
+
else:
|
| 52 |
+
perm_analysis = "No permissions were extracted from the APK. This is unusual for a typical Android app; the file may be corrupted or not a standard APK."
|
| 53 |
+
|
| 54 |
+
# API calls analysis
|
| 55 |
+
apis = scan.get("api_calls", [])
|
| 56 |
+
if apis:
|
| 57 |
+
api_list = "\n".join(apis)
|
| 58 |
+
api_analysis = f"The APK contains the following suspicious API calls:\n{api_list}\n\n"
|
| 59 |
+
if any("SmsManager" in a for a in apis):
|
| 60 |
+
api_analysis += "`SmsManager` calls can send SMS messages without user interaction, leading to premium charges or spam."
|
| 61 |
+
if any("HttpURLConnection" in a for a in apis):
|
| 62 |
+
api_analysis += "`HttpURLConnection` indicates network activity – possible data exfiltration or command‑and‑control communication."
|
| 63 |
+
if any("Runtime.exec" in a for a in apis):
|
| 64 |
+
api_analysis += "`Runtime.exec` allows executing arbitrary commands, which is highly suspicious and could lead to system compromise."
|
| 65 |
+
else:
|
| 66 |
+
api_analysis = "No suspicious API calls were detected."
|
| 67 |
+
|
| 68 |
+
# Behaviour patterns (derived from permissions)
|
| 69 |
+
patterns = []
|
| 70 |
+
if any("SEND_SMS" in p for p in perms):
|
| 71 |
+
patterns.append("Sends SMS")
|
| 72 |
+
if any("READ_SMS" in p for p in perms):
|
| 73 |
+
patterns.append("Reads SMS")
|
| 74 |
+
if any("READ_CONTACTS" in p for p in perms):
|
| 75 |
+
patterns.append("Reads Contacts")
|
| 76 |
+
if any("ACCESS_FINE_LOCATION" in p for p in perms):
|
| 77 |
+
patterns.append("Accesses Location")
|
| 78 |
+
if any("CAMERA" in p for p in perms):
|
| 79 |
+
patterns.append("Uses Camera")
|
| 80 |
+
if any("RECORD_AUDIO" in p for p in perms):
|
| 81 |
+
patterns.append("Records Audio")
|
| 82 |
+
if any("INTERNET" in p for p in perms):
|
| 83 |
+
patterns.append("Internet Access")
|
| 84 |
+
behavior = f"Behaviour patterns: {', '.join(patterns) if patterns else 'None'}."
|
| 85 |
+
|
| 86 |
+
# Network activity
|
| 87 |
+
network = "HTTP network calls detected – potential data exfiltration." if "HttpURLConnection" in str(apis) else "No suspicious network indicators."
|
| 88 |
+
|
| 89 |
+
# Permission‑behaviour correlation
|
| 90 |
+
correlation = ("The APK requests permissions that align with its observed behaviour patterns. "
|
| 91 |
+
"For example, if it requests `READ_CONTACTS` and uses `HttpURLConnection`, it could send contact data to a remote server. "
|
| 92 |
+
"Review the combination of permissions and API calls to assess risk.") if perms and apis else "Insufficient data for correlation analysis."
|
| 93 |
+
|
| 94 |
+
# Data leakage
|
| 95 |
+
data_leakage = ("Potential data leakage points include: sending SMS messages, reading contacts, accessing location, and network communication. "
|
| 96 |
+
"If the app transmits this data to external servers, it may violate user privacy.") if any(p in perms for p in ["SEND_SMS", "READ_CONTACTS", "ACCESS_FINE_LOCATION"]) else "No obvious data leakage indicators."
|
| 97 |
+
|
| 98 |
+
# Security vulnerabilities
|
| 99 |
+
vulnerabilities = ("The presence of `Runtime.exec` or dynamic code loading could introduce command injection vulnerabilities. "
|
| 100 |
+
"Additionally, using `HttpURLConnection` without proper certificate validation may expose data to man‑in‑the‑middle attacks.") if any("Runtime.exec" in a for a in apis) else "No high‑risk vulnerabilities detected from the static analysis."
|
| 101 |
+
|
| 102 |
+
# Behaviour timeline
|
| 103 |
+
timeline = ("Upon installation, the app could request sensitive permissions, then in the background use those permissions to collect data and send it over the network. "
|
| 104 |
+
"If SMS permissions are granted, it might silently send premium messages. A timeline would require dynamic analysis to confirm actual behaviour.")
|
| 105 |
+
|
| 106 |
+
# IOCs
|
| 107 |
+
iocs = "Indicators of compromise include the combination of dangerous permissions (e.g., `SEND_SMS`, `READ_CONTACTS`) and network API calls. "
|
| 108 |
+
if apis:
|
| 109 |
+
iocs += f"Suspicious API calls: {', '.join(apis[:3])}."
|
| 110 |
+
else:
|
| 111 |
+
iocs += "No specific API‑based IOCs found."
|
| 112 |
+
|
| 113 |
+
# Final verdict
|
| 114 |
+
verdict = ("⚠️ **Malicious – block installation.**" if label == "MALWARE" else "✅ **Benign – safe to use.**")
|
| 115 |
+
|
| 116 |
+
return {
|
| 117 |
+
"risk_assessment": risk,
|
| 118 |
+
"malware_classification": classification,
|
| 119 |
+
"permissions_analysis": perm_analysis,
|
| 120 |
+
"api_call_analysis": api_analysis,
|
| 121 |
+
"code_behavior_analysis": behavior,
|
| 122 |
+
"network_activity": network,
|
| 123 |
+
"permission_behavior_correlation": correlation,
|
| 124 |
+
"data_leakage_analysis": data_leakage,
|
| 125 |
+
"security_vulnerabilities": vulnerabilities,
|
| 126 |
+
"behavior_timeline": timeline,
|
| 127 |
+
"indicators_of_compromise": iocs,
|
| 128 |
+
"final_verdict": verdict
|
| 129 |
+
}
|
| 130 |
+
|
| 131 |
+
@router.post("/generate")
|
| 132 |
+
async def generate_local_report(payload: ScanResultPayload):
|
| 133 |
+
# Always return the detailed template (fast and reliable)
|
| 134 |
+
return generate_detailed_report(payload.result)
|
BACKEND/app/routers/sandbox.py
ADDED
|
@@ -0,0 +1,298 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import shutil
|
| 3 |
+
import json
|
| 4 |
+
import torch
|
| 5 |
+
import torch.nn.functional as F
|
| 6 |
+
from torch.utils.data import DataLoader
|
| 7 |
+
from fastapi import APIRouter, HTTPException
|
| 8 |
+
from pydantic import BaseModel
|
| 9 |
+
from typing import List, Dict, Any
|
| 10 |
+
from app.models.resnet34_siamese import SiameseResNet34
|
| 11 |
+
from app.training.pair_dataset import PairDataset
|
| 12 |
+
from app.training.loss_functions import ContrastiveLoss
|
| 13 |
+
from app.settings import GRAY_IMAGES_DIR
|
| 14 |
+
import asyncio
|
| 15 |
+
import json
|
| 16 |
+
from fastapi.responses import StreamingResponse
|
| 17 |
+
from typing import Optional
|
| 18 |
+
|
| 19 |
+
router = APIRouter(prefix="/sandbox", tags=["Training Sandbox"])
|
| 20 |
+
|
| 21 |
+
# Sandbox directories (relative to backend root)
|
| 22 |
+
SANDBOX_DIR = "sandbox"
|
| 23 |
+
SANDBOX_TRAIN_DIR = os.path.join(SANDBOX_DIR, "data", "train")
|
| 24 |
+
SANDBOX_VAL_DIR = os.path.join(SANDBOX_DIR, "data", "val")
|
| 25 |
+
SANDBOX_MODELS_DIR = os.path.join(SANDBOX_DIR, "models")
|
| 26 |
+
SANDBOX_RESULTS_DIR = os.path.join(SANDBOX_DIR, "results")
|
| 27 |
+
|
| 28 |
+
# Ensure directories exist
|
| 29 |
+
os.makedirs(SANDBOX_TRAIN_DIR, exist_ok=True)
|
| 30 |
+
os.makedirs(SANDBOX_VAL_DIR, exist_ok=True)
|
| 31 |
+
os.makedirs(SANDBOX_MODELS_DIR, exist_ok=True)
|
| 32 |
+
os.makedirs(SANDBOX_RESULTS_DIR, exist_ok=True)
|
| 33 |
+
|
| 34 |
+
# Families (same as main)
|
| 35 |
+
FAMILIES = ["benign", "banking", "smsware", "adware", "riskware"]
|
| 36 |
+
|
| 37 |
+
# How many images to copy for sandbox (train and validation)
|
| 38 |
+
TRAIN_COUNT = 5 # per family
|
| 39 |
+
VAL_COUNT = 2 # per family
|
| 40 |
+
|
| 41 |
+
def ensure_sandbox_data():
|
| 42 |
+
"""Copy a small subset of grayscale images to sandbox data folders (if not already present)."""
|
| 43 |
+
# Check if already copied (e.g., by checking one family's train folder)
|
| 44 |
+
sample_path = os.path.join(SANDBOX_TRAIN_DIR, FAMILIES[0])
|
| 45 |
+
if os.path.exists(sample_path) and len(os.listdir(sample_path)) >= TRAIN_COUNT:
|
| 46 |
+
return # already copied
|
| 47 |
+
|
| 48 |
+
# Otherwise, copy fresh
|
| 49 |
+
for family in FAMILIES:
|
| 50 |
+
src_dir = os.path.join(GRAY_IMAGES_DIR, family)
|
| 51 |
+
if not os.path.isdir(src_dir):
|
| 52 |
+
print(f"Warning: {src_dir} not found, skipping {family}")
|
| 53 |
+
continue
|
| 54 |
+
|
| 55 |
+
# Get all PNG images, sorted
|
| 56 |
+
images = [f for f in os.listdir(src_dir) if f.endswith(".png")]
|
| 57 |
+
images.sort()
|
| 58 |
+
if len(images) < TRAIN_COUNT + VAL_COUNT:
|
| 59 |
+
print(f"Warning: Not enough images for {family}, need {TRAIN_COUNT+VAL_COUNT}, have {len(images)}")
|
| 60 |
+
# Use what's available
|
| 61 |
+
train_imgs = images[:TRAIN_COUNT]
|
| 62 |
+
val_imgs = images[TRAIN_COUNT:TRAIN_COUNT+VAL_COUNT]
|
| 63 |
+
else:
|
| 64 |
+
train_imgs = images[:TRAIN_COUNT]
|
| 65 |
+
val_imgs = images[TRAIN_COUNT:TRAIN_COUNT+VAL_COUNT]
|
| 66 |
+
|
| 67 |
+
# Copy to train
|
| 68 |
+
train_dst = os.path.join(SANDBOX_TRAIN_DIR, family)
|
| 69 |
+
os.makedirs(train_dst, exist_ok=True)
|
| 70 |
+
for img in train_imgs:
|
| 71 |
+
shutil.copy(os.path.join(src_dir, img), os.path.join(train_dst, img))
|
| 72 |
+
|
| 73 |
+
# Copy to val
|
| 74 |
+
val_dst = os.path.join(SANDBOX_VAL_DIR, family)
|
| 75 |
+
os.makedirs(val_dst, exist_ok=True)
|
| 76 |
+
for img in val_imgs:
|
| 77 |
+
shutil.copy(os.path.join(src_dir, img), os.path.join(val_dst, img))
|
| 78 |
+
|
| 79 |
+
print("Sandbox data ready.")
|
| 80 |
+
|
| 81 |
+
class TrainRequest(BaseModel):
|
| 82 |
+
batch_size: int = 4
|
| 83 |
+
learning_rate: float = 0.0005
|
| 84 |
+
epochs: int = 10
|
| 85 |
+
loss_margin: float = 1.5
|
| 86 |
+
|
| 87 |
+
@router.post("/train")
|
| 88 |
+
async def sandbox_train(req: TrainRequest):
|
| 89 |
+
# Ensure sandbox data exists
|
| 90 |
+
ensure_sandbox_data()
|
| 91 |
+
|
| 92 |
+
# Create dataset and dataloader from sandbox train folder
|
| 93 |
+
train_dataset = PairDataset(
|
| 94 |
+
root_dir=SANDBOX_TRAIN_DIR,
|
| 95 |
+
families=FAMILIES,
|
| 96 |
+
image_size=224,
|
| 97 |
+
pairs_per_epoch=500 # smaller for speed
|
| 98 |
+
)
|
| 99 |
+
train_loader = DataLoader(
|
| 100 |
+
train_dataset,
|
| 101 |
+
batch_size=req.batch_size,
|
| 102 |
+
shuffle=True,
|
| 103 |
+
num_workers=0
|
| 104 |
+
)
|
| 105 |
+
|
| 106 |
+
# Create a fresh model (not loading any pretrained weights)
|
| 107 |
+
model = SiameseResNet34(
|
| 108 |
+
embed_dim=128,
|
| 109 |
+
pretrained=False, # no pretrained weights to keep training fast
|
| 110 |
+
freeze_backbone=False
|
| 111 |
+
)
|
| 112 |
+
# For speed, we can freeze most of the backbone except the last few layers
|
| 113 |
+
# (similar to main training but we can be simpler)
|
| 114 |
+
for name, param in model.encoder.feature_extractor.named_parameters():
|
| 115 |
+
param.requires_grad = False
|
| 116 |
+
if "7" in name: # only train the last block
|
| 117 |
+
param.requires_grad = True
|
| 118 |
+
for param in model.encoder.embedding_head.parameters():
|
| 119 |
+
param.requires_grad = True
|
| 120 |
+
|
| 121 |
+
optimizer = torch.optim.Adam(
|
| 122 |
+
filter(lambda p: p.requires_grad, model.parameters()),
|
| 123 |
+
lr=req.learning_rate
|
| 124 |
+
)
|
| 125 |
+
criterion = ContrastiveLoss(margin=req.loss_margin)
|
| 126 |
+
|
| 127 |
+
history = [] # list of {"epoch": e, "loss": l, "accuracy": a}
|
| 128 |
+
|
| 129 |
+
for epoch in range(req.epochs):
|
| 130 |
+
model.train()
|
| 131 |
+
running_loss = 0.0
|
| 132 |
+
correct_pairs = 0
|
| 133 |
+
total_pairs = 0
|
| 134 |
+
|
| 135 |
+
for img1, img2, labels in train_loader:
|
| 136 |
+
# Move to device (CPU for simplicity; can be changed)
|
| 137 |
+
img1 = img1
|
| 138 |
+
img2 = img2
|
| 139 |
+
labels = labels.float()
|
| 140 |
+
|
| 141 |
+
optimizer.zero_grad()
|
| 142 |
+
emb1, emb2 = model(img1, img2)
|
| 143 |
+
emb1 = F.normalize(emb1, p=2, dim=1)
|
| 144 |
+
emb2 = F.normalize(emb2, p=2, dim=1)
|
| 145 |
+
|
| 146 |
+
loss = criterion(emb1, emb2, labels)
|
| 147 |
+
loss.backward()
|
| 148 |
+
optimizer.step()
|
| 149 |
+
|
| 150 |
+
running_loss += loss.item()
|
| 151 |
+
|
| 152 |
+
# Compute accuracy for this batch (simple threshold)
|
| 153 |
+
distances = F.pairwise_distance(emb1, emb2)
|
| 154 |
+
predictions = (distances < req.loss_margin).float()
|
| 155 |
+
correct = (predictions == labels).sum().item()
|
| 156 |
+
correct_pairs += correct
|
| 157 |
+
total_pairs += len(labels)
|
| 158 |
+
|
| 159 |
+
epoch_loss = running_loss / len(train_loader)
|
| 160 |
+
epoch_acc = (correct_pairs / total_pairs) * 100.0
|
| 161 |
+
history.append({"epoch": epoch + 1, "loss": round(epoch_loss, 4), "accuracy": round(epoch_acc, 2)})
|
| 162 |
+
|
| 163 |
+
print(f"Sandbox Epoch [{epoch+1}/{req.epochs}] Loss: {epoch_loss:.4f}, Accuracy: {epoch_acc:.2f}%")
|
| 164 |
+
|
| 165 |
+
# After training, evaluate on validation set (simple)
|
| 166 |
+
val_dataset = PairDataset(
|
| 167 |
+
root_dir=SANDBOX_VAL_DIR,
|
| 168 |
+
families=FAMILIES,
|
| 169 |
+
image_size=224,
|
| 170 |
+
pairs_per_epoch=200 # small
|
| 171 |
+
)
|
| 172 |
+
val_loader = DataLoader(val_dataset, batch_size=req.batch_size, shuffle=False, num_workers=0)
|
| 173 |
+
model.eval()
|
| 174 |
+
correct_pairs = 0
|
| 175 |
+
total_pairs = 0
|
| 176 |
+
with torch.no_grad():
|
| 177 |
+
for img1, img2, labels in val_loader:
|
| 178 |
+
emb1, emb2 = model(img1, img2)
|
| 179 |
+
distances = F.pairwise_distance(emb1, emb2)
|
| 180 |
+
predictions = (distances < req.loss_margin).float()
|
| 181 |
+
correct = (predictions == labels).sum().item()
|
| 182 |
+
correct_pairs += correct
|
| 183 |
+
total_pairs += len(labels)
|
| 184 |
+
val_accuracy = (correct_pairs / total_pairs) * 100.0
|
| 185 |
+
|
| 186 |
+
# Optionally, compute per-family accuracy on validation (more complex, can skip for now)
|
| 187 |
+
# For simplicity, we just return overall validation accuracy.
|
| 188 |
+
|
| 189 |
+
# Save the model (optional)
|
| 190 |
+
model_path = os.path.join(SANDBOX_MODELS_DIR, f"student_model_epoch_{req.epochs}.pth")
|
| 191 |
+
torch.save(model.state_dict(), model_path)
|
| 192 |
+
|
| 193 |
+
# Save history to a JSON file
|
| 194 |
+
history_path = os.path.join(SANDBOX_RESULTS_DIR, f"history_{req.epochs}_{req.learning_rate}.json")
|
| 195 |
+
with open(history_path, "w") as f:
|
| 196 |
+
json.dump(history, f, indent=2)
|
| 197 |
+
|
| 198 |
+
return {
|
| 199 |
+
"history": history,
|
| 200 |
+
"final_loss": history[-1]["loss"] if history else 0,
|
| 201 |
+
"val_accuracy": round(val_accuracy, 2),
|
| 202 |
+
"message": f"Training completed on sandbox data. Validation accuracy: {val_accuracy:.2f}%"
|
| 203 |
+
}
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
# ... (keep existing imports and helper functions)
|
| 208 |
+
|
| 209 |
+
@router.get("/train-stream")
|
| 210 |
+
async def sandbox_train_stream(
|
| 211 |
+
batch_size: int = 4,
|
| 212 |
+
learning_rate: float = 0.0005,
|
| 213 |
+
epochs: int = 10,
|
| 214 |
+
loss_margin: float = 1.5
|
| 215 |
+
):
|
| 216 |
+
# Ensure sandbox data exists (same as before)
|
| 217 |
+
ensure_sandbox_data()
|
| 218 |
+
|
| 219 |
+
# Create a fresh model (same as in POST)
|
| 220 |
+
model = SiameseResNet34(embed_dim=128, pretrained=False, freeze_backbone=False)
|
| 221 |
+
for name, param in model.encoder.feature_extractor.named_parameters():
|
| 222 |
+
param.requires_grad = False
|
| 223 |
+
if "7" in name:
|
| 224 |
+
param.requires_grad = True
|
| 225 |
+
for param in model.encoder.embedding_head.parameters():
|
| 226 |
+
param.requires_grad = True
|
| 227 |
+
|
| 228 |
+
optimizer = torch.optim.Adam(
|
| 229 |
+
filter(lambda p: p.requires_grad, model.parameters()),
|
| 230 |
+
lr=learning_rate
|
| 231 |
+
)
|
| 232 |
+
criterion = ContrastiveLoss(margin=loss_margin)
|
| 233 |
+
|
| 234 |
+
train_dataset = PairDataset(
|
| 235 |
+
root_dir=SANDBOX_TRAIN_DIR,
|
| 236 |
+
families=FAMILIES,
|
| 237 |
+
image_size=224,
|
| 238 |
+
pairs_per_epoch=500
|
| 239 |
+
)
|
| 240 |
+
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=0)
|
| 241 |
+
|
| 242 |
+
async def event_generator():
|
| 243 |
+
try:
|
| 244 |
+
for epoch in range(epochs):
|
| 245 |
+
model.train()
|
| 246 |
+
running_loss = 0.0
|
| 247 |
+
correct_pairs = 0
|
| 248 |
+
total_pairs = 0
|
| 249 |
+
for img1, img2, labels in train_loader:
|
| 250 |
+
img1 = img1
|
| 251 |
+
img2 = img2
|
| 252 |
+
labels = labels.float()
|
| 253 |
+
optimizer.zero_grad()
|
| 254 |
+
emb1, emb2 = model(img1, img2)
|
| 255 |
+
emb1 = F.normalize(emb1, p=2, dim=1)
|
| 256 |
+
emb2 = F.normalize(emb2, p=2, dim=1)
|
| 257 |
+
loss = criterion(emb1, emb2, labels)
|
| 258 |
+
loss.backward()
|
| 259 |
+
optimizer.step()
|
| 260 |
+
running_loss += loss.item()
|
| 261 |
+
distances = F.pairwise_distance(emb1, emb2)
|
| 262 |
+
predictions = (distances < loss_margin).float()
|
| 263 |
+
correct = (predictions == labels).sum().item()
|
| 264 |
+
correct_pairs += correct
|
| 265 |
+
total_pairs += len(labels)
|
| 266 |
+
epoch_loss = running_loss / len(train_loader)
|
| 267 |
+
epoch_acc = (correct_pairs / total_pairs) * 100.0
|
| 268 |
+
|
| 269 |
+
# Send epoch update
|
| 270 |
+
yield f"data: {json.dumps({'epoch': epoch+1, 'loss': round(epoch_loss, 4), 'accuracy': round(epoch_acc, 2)})}\n\n"
|
| 271 |
+
await asyncio.sleep(0.01) # allow cancellation
|
| 272 |
+
|
| 273 |
+
# After all epochs, evaluate on validation set
|
| 274 |
+
val_dataset = PairDataset(
|
| 275 |
+
root_dir=SANDBOX_VAL_DIR,
|
| 276 |
+
families=FAMILIES,
|
| 277 |
+
image_size=224,
|
| 278 |
+
pairs_per_epoch=200
|
| 279 |
+
)
|
| 280 |
+
val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=0)
|
| 281 |
+
model.eval()
|
| 282 |
+
correct_pairs = 0
|
| 283 |
+
total_pairs = 0
|
| 284 |
+
with torch.no_grad():
|
| 285 |
+
for img1, img2, labels in val_loader:
|
| 286 |
+
emb1, emb2 = model(img1, img2)
|
| 287 |
+
distances = F.pairwise_distance(emb1, emb2)
|
| 288 |
+
predictions = (distances < loss_margin).float()
|
| 289 |
+
correct = (predictions == labels).sum().item()
|
| 290 |
+
correct_pairs += correct
|
| 291 |
+
total_pairs += len(labels)
|
| 292 |
+
val_accuracy = (correct_pairs / total_pairs) * 100.0
|
| 293 |
+
yield f"data: {json.dumps({'done': True, 'val_accuracy': round(val_accuracy, 2)})}\n\n"
|
| 294 |
+
except asyncio.CancelledError:
|
| 295 |
+
yield f"data: {json.dumps({'stopped': True, 'message': 'Training stopped by user'})}\n\n"
|
| 296 |
+
raise
|
| 297 |
+
|
| 298 |
+
return StreamingResponse(event_generator(), media_type="text/event-stream")
|
BACKEND/app/routers/scan.py
ADDED
|
File without changes
|
BACKEND/app/schemas.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Dict, List, Optional
|
| 2 |
+
from pydantic import BaseModel, Field
|
| 3 |
+
|
| 4 |
+
class ScanResponse(BaseModel):
|
| 5 |
+
success: bool
|
| 6 |
+
file_name: str
|
| 7 |
+
predicted_family: str
|
| 8 |
+
predicted_label: str
|
| 9 |
+
confidence: float
|
| 10 |
+
danger_score: float
|
| 11 |
+
model_name: str
|
| 12 |
+
grayscale_image_url: str
|
| 13 |
+
family_matches: Dict[str, float]
|
| 14 |
+
processing_steps: List[str]
|
| 15 |
+
permissions: List[str] = []
|
| 16 |
+
api_calls: List[str] = []
|
| 17 |
+
|
| 18 |
+
# ========== AUTH MODELS ==========
|
| 19 |
+
class UserCreate(BaseModel):
|
| 20 |
+
email: str = Field(..., pattern=r'^[^@]+@[^@]+\.[^@]+$')
|
| 21 |
+
password: str
|
| 22 |
+
full_name: str # ADDED
|
| 23 |
+
role: str
|
| 24 |
+
university: Optional[str] = None
|
| 25 |
+
organization: Optional[str] = None
|
| 26 |
+
org_details: Optional[str] = None
|
| 27 |
+
|
| 28 |
+
class UserLogin(BaseModel):
|
| 29 |
+
email: str
|
| 30 |
+
password: str
|
| 31 |
+
|
| 32 |
+
class Token(BaseModel):
|
| 33 |
+
access_token: str
|
| 34 |
+
token_type: str
|
| 35 |
+
|
| 36 |
+
class PasswordResetRequest(BaseModel):
|
| 37 |
+
email: str
|
| 38 |
+
|
| 39 |
+
class PasswordResetConfirm(BaseModel):
|
| 40 |
+
token: str
|
| 41 |
+
new_password: str
|
BACKEND/app/settings.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
|
| 3 |
+
APP_DIR = os.path.dirname(os.path.abspath(__file__))
|
| 4 |
+
BACKEND_DIR = os.path.dirname(APP_DIR)
|
| 5 |
+
|
| 6 |
+
DATA_DIR = os.path.join(BACKEND_DIR, "data")
|
| 7 |
+
|
| 8 |
+
RAW_APKS_DIR = os.path.join(DATA_DIR, "raw_apks")
|
| 9 |
+
GRAY_IMAGES_DIR = os.path.join(DATA_DIR, "grayscale_images")
|
| 10 |
+
TRAIN_IMAGES_DIR = os.path.join(DATA_DIR, "train_images")
|
| 11 |
+
SUPPORT_SET_DIR = os.path.join(DATA_DIR, "support_set")
|
| 12 |
+
TEST_IMAGES_DIR = os.path.join(DATA_DIR, "test_images")
|
| 13 |
+
|
| 14 |
+
UPLOADS_DIR = os.path.join(DATA_DIR, "uploads")
|
| 15 |
+
OUTPUTS_DIR = os.path.join(DATA_DIR, "outputs")
|
| 16 |
+
|
| 17 |
+
TRAINED_MODELS_DIR = os.path.join(BACKEND_DIR, "trained_models")
|
| 18 |
+
SUPPORT_EMBEDDINGS_DIR = os.path.join(BACKEND_DIR, "support_embeddings")
|
| 19 |
+
|
| 20 |
+
MODEL_PATH = os.path.join(TRAINED_MODELS_DIR, "resnet34_best.pth")
|
| 21 |
+
SUPPORT_EMBEDDINGS_PATH = os.path.join(SUPPORT_EMBEDDINGS_DIR, "support_embeddings.pkl")
|
| 22 |
+
|
| 23 |
+
IMAGE_SIZE = 224
|
| 24 |
+
EMBED_DIM = 128
|
| 25 |
+
|
| 26 |
+
DEVICE = "cpu"
|
BACKEND/app/static_analysis/__init__.py
ADDED
|
File without changes
|
BACKEND/app/static_analysis/apk_analyzer.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import zipfile
|
| 2 |
+
import os
|
| 3 |
+
import re
|
| 4 |
+
from xml.etree import ElementTree as ET
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def extract_permissions(apk_path):
|
| 8 |
+
return ['android.permission.INTERNET', 'android.permission.READ_EXTERNAL_STORAGE']
|
| 9 |
+
|
| 10 |
+
def extract_api_calls(apk_path):
|
| 11 |
+
return ['SmsManager.sendTextMessage', 'HttpURLConnection.connect']
|
BACKEND/app/support/build_support_set.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
print("Support set is already prepared via dataset_split.py")
|
BACKEND/app/support/save_support_embeddings.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import pickle
|
| 3 |
+
from collections import defaultdict
|
| 4 |
+
|
| 5 |
+
import torch.nn.functional as F
|
| 6 |
+
|
| 7 |
+
from app.constants import ALL_FAMILIES
|
| 8 |
+
from app.inference.make_embedding import load_model, make_embedding
|
| 9 |
+
from app.settings import (
|
| 10 |
+
TRAIN_IMAGES_DIR,
|
| 11 |
+
SUPPORT_SET_DIR,
|
| 12 |
+
SUPPORT_EMBEDDINGS_PATH,
|
| 13 |
+
SUPPORT_EMBEDDINGS_DIR,
|
| 14 |
+
)
|
| 15 |
+
|
| 16 |
+
# Seen families = jinke paas training images bhi hain
|
| 17 |
+
SEEN_FAMILIES = {"benign", "banking", "smsware"}
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def ensure_dir(path: str):
|
| 21 |
+
os.makedirs(path, exist_ok=True)
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def get_png_images(folder: str):
|
| 25 |
+
if not os.path.isdir(folder):
|
| 26 |
+
return []
|
| 27 |
+
return sorted(
|
| 28 |
+
f for f in os.listdir(folder)
|
| 29 |
+
if f.lower().endswith(".png")
|
| 30 |
+
)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def add_family_embeddings_from_folder(
|
| 34 |
+
family: str,
|
| 35 |
+
folder_path: str,
|
| 36 |
+
source_tag: str,
|
| 37 |
+
model,
|
| 38 |
+
support_db,
|
| 39 |
+
max_images: int = None,
|
| 40 |
+
):
|
| 41 |
+
if not os.path.isdir(folder_path):
|
| 42 |
+
print(f"[WARNING] Folder missing: {folder_path}")
|
| 43 |
+
return 0, 0
|
| 44 |
+
|
| 45 |
+
images = get_png_images(folder_path)
|
| 46 |
+
|
| 47 |
+
if max_images is not None:
|
| 48 |
+
images = images[:max_images]
|
| 49 |
+
|
| 50 |
+
if not images:
|
| 51 |
+
print(f"[WARNING] No PNG images found in: {folder_path}")
|
| 52 |
+
return 0, 0
|
| 53 |
+
|
| 54 |
+
print(f"\nProcessing family: {family} | source: {source_tag}")
|
| 55 |
+
print(f"Folder: {folder_path}")
|
| 56 |
+
print(f"Images found: {len(images)}")
|
| 57 |
+
|
| 58 |
+
success_count = 0
|
| 59 |
+
fail_count = 0
|
| 60 |
+
|
| 61 |
+
for image_name in images:
|
| 62 |
+
image_path = os.path.join(folder_path, image_name)
|
| 63 |
+
|
| 64 |
+
try:
|
| 65 |
+
emb = make_embedding(image_path, model)
|
| 66 |
+
emb = F.normalize(emb.unsqueeze(0), p=2, dim=1).squeeze(0)
|
| 67 |
+
|
| 68 |
+
support_db[family].append({
|
| 69 |
+
"image_name": image_name,
|
| 70 |
+
"embedding": emb.numpy(),
|
| 71 |
+
"source": source_tag,
|
| 72 |
+
})
|
| 73 |
+
|
| 74 |
+
success_count += 1
|
| 75 |
+
print(f"[OK] {source_tag}/{image_name}")
|
| 76 |
+
|
| 77 |
+
except Exception as e:
|
| 78 |
+
fail_count += 1
|
| 79 |
+
print(f"[FAILED] {source_tag}/{image_name}: {e}")
|
| 80 |
+
|
| 81 |
+
return success_count, fail_count
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def save_support_embeddings():
|
| 85 |
+
ensure_dir(SUPPORT_EMBEDDINGS_DIR)
|
| 86 |
+
|
| 87 |
+
model = load_model()
|
| 88 |
+
support_db = defaultdict(list)
|
| 89 |
+
|
| 90 |
+
for family in ALL_FAMILIES:
|
| 91 |
+
total_success = 0
|
| 92 |
+
total_fail = 0
|
| 93 |
+
|
| 94 |
+
# Always include support_set
|
| 95 |
+
support_family_dir = os.path.join(SUPPORT_SET_DIR, family)
|
| 96 |
+
s_ok, s_fail = add_family_embeddings_from_folder(
|
| 97 |
+
family=family,
|
| 98 |
+
folder_path=support_family_dir,
|
| 99 |
+
source_tag="support",
|
| 100 |
+
model=model,
|
| 101 |
+
support_db=support_db,
|
| 102 |
+
max_images=None, # support images sab use hongi
|
| 103 |
+
)
|
| 104 |
+
total_success += s_ok
|
| 105 |
+
total_fail += s_fail
|
| 106 |
+
|
| 107 |
+
# For seen families, also include limited train_images
|
| 108 |
+
if family in SEEN_FAMILIES:
|
| 109 |
+
train_family_dir = os.path.join(TRAIN_IMAGES_DIR, family)
|
| 110 |
+
t_ok, t_fail = add_family_embeddings_from_folder(
|
| 111 |
+
family=family,
|
| 112 |
+
folder_path=train_family_dir,
|
| 113 |
+
source_tag="train",
|
| 114 |
+
model=model,
|
| 115 |
+
support_db=support_db,
|
| 116 |
+
max_images=5, # only limited train refs
|
| 117 |
+
)
|
| 118 |
+
total_success += t_ok
|
| 119 |
+
total_fail += t_fail
|
| 120 |
+
|
| 121 |
+
print(
|
| 122 |
+
f"[SUMMARY] {family}: total_saved={len(support_db[family])}, "
|
| 123 |
+
f"success={total_success}, failed={total_fail}"
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
with open(SUPPORT_EMBEDDINGS_PATH, "wb") as f:
|
| 127 |
+
pickle.dump(dict(support_db), f)
|
| 128 |
+
|
| 129 |
+
print("\nExpanded gallery embeddings saved at:")
|
| 130 |
+
print(SUPPORT_EMBEDDINGS_PATH)
|
| 131 |
+
|
| 132 |
+
print("\nFinal family counts:")
|
| 133 |
+
for family in ALL_FAMILIES:
|
| 134 |
+
print(f"- {family}: {len(support_db[family])} embeddings")
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
if __name__ == "__main__":
|
| 138 |
+
save_support_embeddings()
|
BACKEND/app/training/dataset_loader.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
print("Dataset loader not used directly. PairDataset is used for training.")
|
BACKEND/app/training/loss_functions.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
import torch.nn.functional as F
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class ContrastiveLoss(nn.Module):
|
| 7 |
+
"""
|
| 8 |
+
label = 1 -> same class
|
| 9 |
+
label = 0 -> different class
|
| 10 |
+
"""
|
| 11 |
+
def __init__(self, margin: float = 1.0):
|
| 12 |
+
super().__init__()
|
| 13 |
+
self.margin = margin
|
| 14 |
+
|
| 15 |
+
def forward(self, emb1: torch.Tensor, emb2: torch.Tensor, label: torch.Tensor):
|
| 16 |
+
dist = F.pairwise_distance(emb1, emb2)
|
| 17 |
+
|
| 18 |
+
positive_loss = label * torch.pow(dist, 2)
|
| 19 |
+
negative_loss = (1 - label) * torch.pow(torch.clamp(self.margin - dist, min=0.0), 2)
|
| 20 |
+
|
| 21 |
+
loss = torch.mean(positive_loss + negative_loss)
|
| 22 |
+
return loss
|
BACKEND/app/training/pair_dataset.py
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import random
|
| 3 |
+
from typing import Dict, List
|
| 4 |
+
|
| 5 |
+
from PIL import Image
|
| 6 |
+
import torch
|
| 7 |
+
from torch.utils.data import Dataset
|
| 8 |
+
from torchvision import transforms
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class PairDataset(Dataset):
|
| 12 |
+
def __init__(
|
| 13 |
+
self,
|
| 14 |
+
root_dir: str,
|
| 15 |
+
families: List[str],
|
| 16 |
+
image_size: int = 224,
|
| 17 |
+
pairs_per_epoch: int = 1000,
|
| 18 |
+
):
|
| 19 |
+
self.root_dir = root_dir
|
| 20 |
+
self.families = families
|
| 21 |
+
self.pairs_per_epoch = pairs_per_epoch
|
| 22 |
+
|
| 23 |
+
self.transform = transforms.Compose([
|
| 24 |
+
transforms.Resize((image_size, image_size)),
|
| 25 |
+
transforms.ToTensor(),
|
| 26 |
+
])
|
| 27 |
+
|
| 28 |
+
self.family_to_images: Dict[str, List[str]] = {}
|
| 29 |
+
|
| 30 |
+
for family in families:
|
| 31 |
+
family_dir = os.path.join(root_dir, family)
|
| 32 |
+
if not os.path.isdir(family_dir):
|
| 33 |
+
continue
|
| 34 |
+
|
| 35 |
+
files = []
|
| 36 |
+
for f in os.listdir(family_dir):
|
| 37 |
+
if f.lower().endswith(".png"):
|
| 38 |
+
full_path = os.path.join(family_dir, f)
|
| 39 |
+
try:
|
| 40 |
+
with Image.open(full_path) as img:
|
| 41 |
+
img.verify()
|
| 42 |
+
files.append(full_path)
|
| 43 |
+
except Exception:
|
| 44 |
+
continue
|
| 45 |
+
|
| 46 |
+
if len(files) >= 2:
|
| 47 |
+
self.family_to_images[family] = files
|
| 48 |
+
|
| 49 |
+
self.valid_families = list(self.family_to_images.keys())
|
| 50 |
+
|
| 51 |
+
if len(self.valid_families) < 2:
|
| 52 |
+
raise ValueError("At least 2 valid families with >=2 images each are required.")
|
| 53 |
+
|
| 54 |
+
def __len__(self):
|
| 55 |
+
return self.pairs_per_epoch
|
| 56 |
+
|
| 57 |
+
def _load_image(self, path: str) -> torch.Tensor:
|
| 58 |
+
img = Image.open(path).convert("L")
|
| 59 |
+
return self.transform(img)
|
| 60 |
+
|
| 61 |
+
def __getitem__(self, index: int):
|
| 62 |
+
same_class = random.randint(0, 1)
|
| 63 |
+
|
| 64 |
+
if same_class == 1:
|
| 65 |
+
family = random.choice(self.valid_families)
|
| 66 |
+
img_paths = random.sample(self.family_to_images[family], 2)
|
| 67 |
+
label = 1.0
|
| 68 |
+
else:
|
| 69 |
+
fam1, fam2 = random.sample(self.valid_families, 2)
|
| 70 |
+
img_paths = [
|
| 71 |
+
random.choice(self.family_to_images[fam1]),
|
| 72 |
+
random.choice(self.family_to_images[fam2]),
|
| 73 |
+
]
|
| 74 |
+
label = 0.0
|
| 75 |
+
|
| 76 |
+
img1 = self._load_image(img_paths[0])
|
| 77 |
+
img2 = self._load_image(img_paths[1])
|
| 78 |
+
|
| 79 |
+
return img1, img2, torch.tensor(label, dtype=torch.float32)
|
BACKEND/app/training/train_resnet34.py
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import json
|
| 3 |
+
import torch
|
| 4 |
+
import torch.nn.functional as F
|
| 5 |
+
from torch.utils.data import DataLoader
|
| 6 |
+
|
| 7 |
+
from app.models.resnet34_siamese import SiameseResNet34
|
| 8 |
+
from app.training.pair_dataset import PairDataset
|
| 9 |
+
from app.training.loss_functions import ContrastiveLoss
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
TRAIN_DIR = "data/train_images"
|
| 13 |
+
MODEL_DIR = "trained_models"
|
| 14 |
+
MODEL_PATH = os.path.join(MODEL_DIR, "resnet34_best.pth")
|
| 15 |
+
HISTORY_PATH = os.path.join(MODEL_DIR, "train_history.json")
|
| 16 |
+
|
| 17 |
+
TRAIN_FAMILIES = ["benign", "banking", "smsware"]
|
| 18 |
+
|
| 19 |
+
IMAGE_SIZE = 224
|
| 20 |
+
PAIRS_PER_EPOCH = 1000
|
| 21 |
+
EPOCHS = 30
|
| 22 |
+
LR = 0.00005
|
| 23 |
+
TRAIN_BATCH_SIZE = 2
|
| 24 |
+
DEVICE = "cpu"
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def ensure_dir(path: str):
|
| 28 |
+
os.makedirs(path, exist_ok=True)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def train():
|
| 32 |
+
ensure_dir(MODEL_DIR)
|
| 33 |
+
|
| 34 |
+
dataset = PairDataset(
|
| 35 |
+
root_dir=TRAIN_DIR,
|
| 36 |
+
families=TRAIN_FAMILIES,
|
| 37 |
+
image_size=IMAGE_SIZE,
|
| 38 |
+
pairs_per_epoch=PAIRS_PER_EPOCH,
|
| 39 |
+
)
|
| 40 |
+
|
| 41 |
+
loader = DataLoader(
|
| 42 |
+
dataset,
|
| 43 |
+
batch_size=TRAIN_BATCH_SIZE,
|
| 44 |
+
shuffle=True,
|
| 45 |
+
num_workers=0,
|
| 46 |
+
)
|
| 47 |
+
|
| 48 |
+
model = SiameseResNet34(
|
| 49 |
+
embed_dim=128,
|
| 50 |
+
pretrained=True,
|
| 51 |
+
freeze_backbone=False
|
| 52 |
+
).to(DEVICE)
|
| 53 |
+
|
| 54 |
+
# Freeze most of the backbone, train only the deeper part
|
| 55 |
+
for name, param in model.encoder.feature_extractor.named_parameters():
|
| 56 |
+
param.requires_grad = False
|
| 57 |
+
if "7" in name:
|
| 58 |
+
param.requires_grad = True
|
| 59 |
+
|
| 60 |
+
# embedding head ہمیشہ trainable رہے
|
| 61 |
+
for param in model.encoder.embedding_head.parameters():
|
| 62 |
+
param.requires_grad = True
|
| 63 |
+
|
| 64 |
+
criterion = ContrastiveLoss(margin=1.5)
|
| 65 |
+
optimizer = torch.optim.Adam(
|
| 66 |
+
filter(lambda p: p.requires_grad, model.parameters()),
|
| 67 |
+
lr=LR
|
| 68 |
+
)
|
| 69 |
+
|
| 70 |
+
best_loss = float("inf")
|
| 71 |
+
history = []
|
| 72 |
+
|
| 73 |
+
for epoch in range(EPOCHS):
|
| 74 |
+
model.train()
|
| 75 |
+
running_loss = 0.0
|
| 76 |
+
|
| 77 |
+
for img1, img2, labels in loader:
|
| 78 |
+
img1 = img1.to(DEVICE)
|
| 79 |
+
img2 = img2.to(DEVICE)
|
| 80 |
+
labels = labels.to(DEVICE).float()
|
| 81 |
+
|
| 82 |
+
optimizer.zero_grad()
|
| 83 |
+
|
| 84 |
+
emb1, emb2 = model(img1, img2)
|
| 85 |
+
emb1 = F.normalize(emb1, p=2, dim=1)
|
| 86 |
+
emb2 = F.normalize(emb2, p=2, dim=1)
|
| 87 |
+
|
| 88 |
+
loss = criterion(emb1, emb2, labels)
|
| 89 |
+
loss.backward()
|
| 90 |
+
optimizer.step()
|
| 91 |
+
|
| 92 |
+
running_loss += loss.item()
|
| 93 |
+
|
| 94 |
+
epoch_loss = running_loss / len(loader)
|
| 95 |
+
history.append({"epoch": epoch + 1, "loss": epoch_loss})
|
| 96 |
+
|
| 97 |
+
print(f"Epoch [{epoch + 1}/{EPOCHS}] Loss: {epoch_loss:.4f}")
|
| 98 |
+
|
| 99 |
+
if epoch_loss < best_loss:
|
| 100 |
+
best_loss = epoch_loss
|
| 101 |
+
torch.save(model.state_dict(), MODEL_PATH)
|
| 102 |
+
print(f"Best model saved at epoch {epoch + 1}")
|
| 103 |
+
|
| 104 |
+
with open(HISTORY_PATH, "w") as f:
|
| 105 |
+
json.dump(history, f, indent=2)
|
| 106 |
+
|
| 107 |
+
print("\nTraining complete.")
|
| 108 |
+
print("Best model path:", MODEL_PATH)
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
if __name__ == "__main__":
|
| 112 |
+
train()
|
BACKEND/app/utils/file_utils.py
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
def ensure_dir(path: str):
|
| 5 |
+
os.makedirs(path, exist_ok=True)
|
BACKEND/app/utils/image_utils.py
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from PIL import Image
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
def open_grayscale(image_path: str):
|
| 5 |
+
return Image.open(image_path).convert("L")
|
BACKEND/data_split.py
ADDED
|
@@ -0,0 +1,270 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import shutil
|
| 3 |
+
from typing import List, Tuple
|
| 4 |
+
|
| 5 |
+
import numpy as np
|
| 6 |
+
from PIL import Image
|
| 7 |
+
|
| 8 |
+
# =========================
|
| 9 |
+
# SETTINGS
|
| 10 |
+
# =========================
|
| 11 |
+
|
| 12 |
+
SOURCE_DIR = "data/grayscale_images"
|
| 13 |
+
|
| 14 |
+
TRAIN_DIR = "data/train_images"
|
| 15 |
+
SUPPORT_DIR = "data/support_set"
|
| 16 |
+
TEST_DIR = "data/test_images"
|
| 17 |
+
|
| 18 |
+
TRAIN_FAMILIES = ["benign", "banking", "smsware"]
|
| 19 |
+
ALL_FAMILIES = ["benign", "banking", "smsware", "adware", "riskware"]
|
| 20 |
+
|
| 21 |
+
TRAIN_COUNT = 10
|
| 22 |
+
SUPPORT_COUNT = 5
|
| 23 |
+
TEST_COUNT = 5
|
| 24 |
+
|
| 25 |
+
# =========================
|
| 26 |
+
# MANUAL SUPPORT SELECTION
|
| 27 |
+
# =========================
|
| 28 |
+
# Agar kisi family ke liye yahan filenames di hui hon,
|
| 29 |
+
# to support set unhi files se banega.
|
| 30 |
+
# Baqi split automatically hoga.
|
| 31 |
+
#
|
| 32 |
+
# IMPORTANT:
|
| 33 |
+
# Ye filenames exact waise hi honi chahiye jaisi
|
| 34 |
+
# SOURCE_DIR/family folder me موجود hain.
|
| 35 |
+
#
|
| 36 |
+
# Abhi smsware ke liye manual support ON hai.
|
| 37 |
+
MANUAL_SUPPORT = {
|
| 38 |
+
"smsware": [
|
| 39 |
+
"020cdc2d622af016d7cbfcee797e078884380a6635ebe70b36a5c527608ec07f.png",
|
| 40 |
+
"0221511d597a5ab7b6303e12675dabadf6f48db968fa26403ee70a041e3a6826.png",
|
| 41 |
+
"043b4fbc2b58040754a20844e8bc85139ce38daffd40ce53ba0a91ba052ca84b.png",
|
| 42 |
+
"0454a5c0ff9fea30a5084af2354ef142f0ee5dbbf3545edb4bf0d07b2242bbeb.png",
|
| 43 |
+
"015b473e1d56054bed16899430ea95f9ac940a45ab0ec4888a119279667e7916.png",
|
| 44 |
+
]
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
def ensure_dir(path):
|
| 48 |
+
os.makedirs(path, exist_ok=True)
|
| 49 |
+
|
| 50 |
+
def reset_family_dir(path):
|
| 51 |
+
if os.path.isdir(path):
|
| 52 |
+
shutil.rmtree(path)
|
| 53 |
+
os.makedirs(path, exist_ok=True)
|
| 54 |
+
|
| 55 |
+
def copy_files(files, src_dir, dst_dir):
|
| 56 |
+
ensure_dir(dst_dir)
|
| 57 |
+
|
| 58 |
+
for f in files:
|
| 59 |
+
src = os.path.join(src_dir, f)
|
| 60 |
+
dst = os.path.join(dst_dir, f)
|
| 61 |
+
shutil.copy(src, dst)
|
| 62 |
+
|
| 63 |
+
def get_image_score(image_path: str) -> float:
|
| 64 |
+
"""
|
| 65 |
+
Higher score = better / more informative image.
|
| 66 |
+
Prefer images with reasonable contrast and non-extreme brightness.
|
| 67 |
+
"""
|
| 68 |
+
try:
|
| 69 |
+
img = Image.open(image_path).convert("L")
|
| 70 |
+
arr = np.array(img, dtype=np.float32)
|
| 71 |
+
|
| 72 |
+
mean_val = float(arr.mean())
|
| 73 |
+
std_val = float(arr.std())
|
| 74 |
+
|
| 75 |
+
mean_penalty = abs(mean_val - 127.5) / 127.5
|
| 76 |
+
score = std_val - (mean_penalty * 20.0)
|
| 77 |
+
return score
|
| 78 |
+
except Exception:
|
| 79 |
+
return -1e9
|
| 80 |
+
|
| 81 |
+
def get_ranked_images(family_src: str) -> List[str]:
|
| 82 |
+
images = [
|
| 83 |
+
f for f in os.listdir(family_src)
|
| 84 |
+
if f.lower().endswith(".png")
|
| 85 |
+
]
|
| 86 |
+
|
| 87 |
+
scored_images: List[Tuple[str, float]] = []
|
| 88 |
+
|
| 89 |
+
for f in images:
|
| 90 |
+
path = os.path.join(family_src, f)
|
| 91 |
+
score = get_image_score(path)
|
| 92 |
+
if score > -1e8:
|
| 93 |
+
scored_images.append((f, score))
|
| 94 |
+
|
| 95 |
+
scored_images.sort(key=lambda x: x[1], reverse=True)
|
| 96 |
+
return [f for f, _ in scored_images]
|
| 97 |
+
|
| 98 |
+
def pick_spread_items(images: List[str], count: int) -> List[str]:
|
| 99 |
+
"""
|
| 100 |
+
Pick evenly spread samples from a ranked list so support is diverse.
|
| 101 |
+
"""
|
| 102 |
+
if len(images) <= count:
|
| 103 |
+
return images[:count]
|
| 104 |
+
|
| 105 |
+
indices = np.linspace(0, len(images) - 1, count, dtype=int)
|
| 106 |
+
picked = [images[i] for i in indices]
|
| 107 |
+
|
| 108 |
+
unique_picked = []
|
| 109 |
+
for item in picked:
|
| 110 |
+
if item not in unique_picked:
|
| 111 |
+
unique_picked.append(item)
|
| 112 |
+
|
| 113 |
+
if len(unique_picked) < count:
|
| 114 |
+
for item in images:
|
| 115 |
+
if item not in unique_picked:
|
| 116 |
+
unique_picked.append(item)
|
| 117 |
+
if len(unique_picked) == count:
|
| 118 |
+
break
|
| 119 |
+
|
| 120 |
+
return unique_picked[:count]
|
| 121 |
+
|
| 122 |
+
def validate_manual_support(family: str, family_src: str, manual_files: List[str]) -> List[str]:
|
| 123 |
+
"""
|
| 124 |
+
Keep only valid manual support files that actually exist.
|
| 125 |
+
"""
|
| 126 |
+
valid = []
|
| 127 |
+
missing = []
|
| 128 |
+
|
| 129 |
+
for f in manual_files:
|
| 130 |
+
full_path = os.path.join(family_src, f)
|
| 131 |
+
if os.path.isfile(full_path):
|
| 132 |
+
valid.append(f)
|
| 133 |
+
else:
|
| 134 |
+
missing.append(f)
|
| 135 |
+
|
| 136 |
+
if missing:
|
| 137 |
+
print(f"[WARNING] Missing manual support files for {family}:")
|
| 138 |
+
for f in missing:
|
| 139 |
+
print(f" - {f}")
|
| 140 |
+
|
| 141 |
+
if len(valid) < SUPPORT_COUNT:
|
| 142 |
+
print(
|
| 143 |
+
f"[WARNING] Manual support for {family} has only {len(valid)} valid files. "
|
| 144 |
+
f"Need {SUPPORT_COUNT}. Falling back to auto-fill for remaining."
|
| 145 |
+
)
|
| 146 |
+
|
| 147 |
+
return valid
|
| 148 |
+
|
| 149 |
+
def split_seen_family(images: List[str], family: str, family_src: str):
|
| 150 |
+
"""
|
| 151 |
+
Seen family split:
|
| 152 |
+
- support = 5
|
| 153 |
+
- train = 10
|
| 154 |
+
- test = 5
|
| 155 |
+
|
| 156 |
+
smsware ke liye manual support allow hai.
|
| 157 |
+
"""
|
| 158 |
+
required = SUPPORT_COUNT + TRAIN_COUNT + TEST_COUNT
|
| 159 |
+
if len(images) < required:
|
| 160 |
+
print(f"[WARNING] Seen family has fewer than required images: {len(images)} < {required}")
|
| 161 |
+
|
| 162 |
+
pool = images[:max(required, 20)]
|
| 163 |
+
|
| 164 |
+
# =========================
|
| 165 |
+
# Manual support mode
|
| 166 |
+
# =========================
|
| 167 |
+
if family in MANUAL_SUPPORT:
|
| 168 |
+
manual_support = validate_manual_support(family, family_src, MANUAL_SUPPORT[family])
|
| 169 |
+
|
| 170 |
+
remaining_candidates = [img for img in pool if img not in manual_support]
|
| 171 |
+
|
| 172 |
+
# Agar manual support 5 se kam ho to auto-fill kar do
|
| 173 |
+
if len(manual_support) < SUPPORT_COUNT:
|
| 174 |
+
needed = SUPPORT_COUNT - len(manual_support)
|
| 175 |
+
auto_fill = remaining_candidates[:needed]
|
| 176 |
+
support = manual_support + auto_fill
|
| 177 |
+
else:
|
| 178 |
+
support = manual_support[:SUPPORT_COUNT]
|
| 179 |
+
|
| 180 |
+
remaining = [img for img in pool if img not in support]
|
| 181 |
+
train = remaining[:TRAIN_COUNT]
|
| 182 |
+
test = remaining[TRAIN_COUNT:TRAIN_COUNT + TEST_COUNT]
|
| 183 |
+
|
| 184 |
+
return train, support, test
|
| 185 |
+
|
| 186 |
+
# =========================
|
| 187 |
+
# Auto split for seen families
|
| 188 |
+
# =========================
|
| 189 |
+
support_candidates = pool[:15] if len(pool) >= 15 else pool
|
| 190 |
+
support = pick_spread_items(support_candidates, SUPPORT_COUNT)
|
| 191 |
+
|
| 192 |
+
remaining = [img for img in pool if img not in support]
|
| 193 |
+
train = remaining[:TRAIN_COUNT]
|
| 194 |
+
test = remaining[TRAIN_COUNT:TRAIN_COUNT + TEST_COUNT]
|
| 195 |
+
|
| 196 |
+
return train, support, test
|
| 197 |
+
|
| 198 |
+
def split_unseen_family(images: List[str]):
|
| 199 |
+
"""
|
| 200 |
+
Unseen family split:
|
| 201 |
+
- train = 0
|
| 202 |
+
- support = 5
|
| 203 |
+
- test = 5
|
| 204 |
+
"""
|
| 205 |
+
required = SUPPORT_COUNT + TEST_COUNT
|
| 206 |
+
if len(images) < required:
|
| 207 |
+
print(f"[WARNING] Unseen family has fewer than required images: {len(images)} < {required}")
|
| 208 |
+
|
| 209 |
+
pool = images[:max(required, 15)]
|
| 210 |
+
|
| 211 |
+
support_candidates = pool[:10] if len(pool) >= 10 else pool
|
| 212 |
+
support = pick_spread_items(support_candidates, SUPPORT_COUNT)
|
| 213 |
+
|
| 214 |
+
remaining = [img for img in pool if img not in support]
|
| 215 |
+
test = remaining[:TEST_COUNT]
|
| 216 |
+
|
| 217 |
+
train = []
|
| 218 |
+
return train, support, test
|
| 219 |
+
|
| 220 |
+
def main():
|
| 221 |
+
print("\n========== DATASET SPLIT START ==========\n")
|
| 222 |
+
|
| 223 |
+
ensure_dir(TRAIN_DIR)
|
| 224 |
+
ensure_dir(SUPPORT_DIR)
|
| 225 |
+
ensure_dir(TEST_DIR)
|
| 226 |
+
|
| 227 |
+
for family in ALL_FAMILIES:
|
| 228 |
+
family_src = os.path.join(SOURCE_DIR, family)
|
| 229 |
+
|
| 230 |
+
if not os.path.isdir(family_src):
|
| 231 |
+
print(f"[WARNING] Missing source folder: {family_src}")
|
| 232 |
+
continue
|
| 233 |
+
|
| 234 |
+
images = get_ranked_images(family_src)
|
| 235 |
+
|
| 236 |
+
print(f"\nProcessing: {family}")
|
| 237 |
+
print("Valid images found:", len(images))
|
| 238 |
+
|
| 239 |
+
if family in TRAIN_FAMILIES:
|
| 240 |
+
train, support, test = split_seen_family(images, family, family_src)
|
| 241 |
+
else:
|
| 242 |
+
train, support, test = split_unseen_family(images)
|
| 243 |
+
|
| 244 |
+
reset_family_dir(os.path.join(TRAIN_DIR, family))
|
| 245 |
+
reset_family_dir(os.path.join(SUPPORT_DIR, family))
|
| 246 |
+
reset_family_dir(os.path.join(TEST_DIR, family))
|
| 247 |
+
|
| 248 |
+
copy_files(train, family_src, os.path.join(TRAIN_DIR, family))
|
| 249 |
+
copy_files(support, family_src, os.path.join(SUPPORT_DIR, family))
|
| 250 |
+
copy_files(test, family_src, os.path.join(TEST_DIR, family))
|
| 251 |
+
|
| 252 |
+
print("Train :", len(train))
|
| 253 |
+
print("Support:", len(support))
|
| 254 |
+
print("Test :", len(test))
|
| 255 |
+
|
| 256 |
+
if family in TRAIN_FAMILIES:
|
| 257 |
+
if family in MANUAL_SUPPORT:
|
| 258 |
+
print("Seen split -> manual support(5), train(10), test(5)")
|
| 259 |
+
print("Manual support files:")
|
| 260 |
+
for f in support:
|
| 261 |
+
print(f" - {f}")
|
| 262 |
+
else:
|
| 263 |
+
print("Seen split -> support(diverse 5), train(10), test(5)")
|
| 264 |
+
else:
|
| 265 |
+
print("Unseen split -> support(diverse 5), test(5), no train")
|
| 266 |
+
|
| 267 |
+
print("\n========== DATASET SPLIT DONE ==========\n")
|
| 268 |
+
|
| 269 |
+
if __name__ == "__main__":
|
| 270 |
+
main()
|
BACKEND/dockerfile
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ----- Stage 1: React Frontend Build -----
|
| 2 |
+
FROM node:18 AS frontend-build
|
| 3 |
+
WORKDIR /app/frontend
|
| 4 |
+
COPY malware-detection-frontend/package*.json ./
|
| 5 |
+
RUN npm install
|
| 6 |
+
COPY malware-detection-frontend/ ./
|
| 7 |
+
# React build karein (static files 'build' folder mein aayengi)
|
| 8 |
+
RUN npm run build
|
| 9 |
+
|
| 10 |
+
# ----- Stage 2: Python Backend -----
|
| 11 |
+
FROM python:3.10-slim
|
| 12 |
+
WORKDIR /app/backend
|
| 13 |
+
|
| 14 |
+
# Build folder ko frontend se copy karein
|
| 15 |
+
COPY --from=frontend-build /app/frontend/build /app/backend/frontend_build
|
| 16 |
+
|
| 17 |
+
# Backend dependencies install karein
|
| 18 |
+
COPY BACKEND/requirements.txt ./
|
| 19 |
+
RUN pip install --no-cache-dir -r requirements.txt
|
| 20 |
+
|
| 21 |
+
# Poora backend code copy karein
|
| 22 |
+
COPY BACKEND/ ./
|
| 23 |
+
|
| 24 |
+
# CRITICAL: Hugging Face Space sirf port 7860 expose karta hai
|
| 25 |
+
EXPOSE 7860
|
| 26 |
+
|
| 27 |
+
# FastAPI chalayein (port 7860 par)
|
| 28 |
+
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"]
|
BACKEND/list_models.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import google.generativeai as genai
|
| 2 |
+
import os
|
| 3 |
+
from dotenv import load_dotenv
|
| 4 |
+
|
| 5 |
+
load_dotenv()
|
| 6 |
+
genai.configure(api_key=os.getenv("GEMINI_API_KEY"))
|
| 7 |
+
|
| 8 |
+
print("Available models:")
|
| 9 |
+
for model in genai.list_models():
|
| 10 |
+
print(f" {model.name}")
|
BACKEND/requirements.txt
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
fastapi
|
| 2 |
+
uvicorn
|
| 3 |
+
python-multipart
|
| 4 |
+
numpy
|
| 5 |
+
pillow
|
| 6 |
+
torch
|
| 7 |
+
torchvision
|
| 8 |
+
pydantic
|
| 9 |
+
python-jose[cryptography]
|
| 10 |
+
passlib[bcrypt]
|
| 11 |
+
python-multipart
|
| 12 |
+
email-validator
|
| 13 |
+
aiosmtplib
|
| 14 |
+
python-dotenv
|
| 15 |
+
werkzeug
|
| 16 |
+
python-jose[cryptography]
|
| 17 |
+
python-dotenv
|
| 18 |
+
sqlalchemy
|
| 19 |
+
aiosqlite
|
| 20 |
+
email-validator
|
| 21 |
+
fastapi
|
| 22 |
+
uvicorn
|
| 23 |
+
python-multipart
|
| 24 |
+
numpy
|
| 25 |
+
pillow
|
| 26 |
+
pydantic
|
| 27 |
+
python-jose[cryptography]
|
| 28 |
+
werkzeug
|
| 29 |
+
sqlalchemy
|
| 30 |
+
python-dotenv
|
| 31 |
+
scikit-learn
|
| 32 |
+
torch --index-url https://download.pytorch.org/whl/cpu
|
| 33 |
+
torchvision --index-url https://download.pytorch.org/whl/cpu
|
| 34 |
+
gradio
|
| 35 |
+
pandas
|
| 36 |
+
scikit-learn
|
| 37 |
+
# Add any other libraries you use, e.g., torch, tensorflow, joblib
|
BACKEND/run_backend.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import uvicorn
|
| 2 |
+
import os
|
| 3 |
+
import sys
|
| 4 |
+
print(sys.executable)
|
| 5 |
+
|
| 6 |
+
def main():
|
| 7 |
+
"""
|
| 8 |
+
Start FastAPI backend server
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
print("\n====================================")
|
| 12 |
+
print(" Android Malware FSL Backend Server ")
|
| 13 |
+
print("====================================\n")
|
| 14 |
+
|
| 15 |
+
print("Server starting...")
|
| 16 |
+
print("API URL: http://127.0.0.1:8000")
|
| 17 |
+
print("Docs: http://127.0.0.1:8000/docs\n")
|
| 18 |
+
|
| 19 |
+
uvicorn.run(
|
| 20 |
+
"app.main:app",
|
| 21 |
+
host="127.0.0.1",
|
| 22 |
+
port=8000,
|
| 23 |
+
reload=True
|
| 24 |
+
)
|
| 25 |
+
|
| 26 |
+
if __name__ == "__main__":
|
| 27 |
+
main()
|
BACKEND/setup_dataset.py
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import zipfile
|
| 3 |
+
from app.preprocessing.apk_pipeline import apk_to_image_pipeline
|
| 4 |
+
|
| 5 |
+
# =========================
|
| 6 |
+
# SETTINGS
|
| 7 |
+
# =========================
|
| 8 |
+
RAW_APKS_DIR = "data/raw_apks"
|
| 9 |
+
GRAY_IMAGES_DIR = "data/grayscale_images"
|
| 10 |
+
TEMP_DIR = "temp"
|
| 11 |
+
FINAL_SIZE = (300, 300)
|
| 12 |
+
|
| 13 |
+
# Target number of grayscale images per family
|
| 14 |
+
TARGET_IMAGES_PER_FAMILY = 20
|
| 15 |
+
|
| 16 |
+
# Families to process
|
| 17 |
+
FAMILIES = ["benign", "banking", "smsware", "adware", "riskware"]
|
| 18 |
+
|
| 19 |
+
def ensure_dir(path: str):
|
| 20 |
+
os.makedirs(path, exist_ok=True)
|
| 21 |
+
|
| 22 |
+
def is_valid_apk(file_path: str) -> bool:
|
| 23 |
+
"""
|
| 24 |
+
Check file validity by trying to open it as APK/ZIP
|
| 25 |
+
and verifying that at least one .dex exists.
|
| 26 |
+
Extension does not matter.
|
| 27 |
+
"""
|
| 28 |
+
try:
|
| 29 |
+
with zipfile.ZipFile(file_path, "r") as zip_ref:
|
| 30 |
+
names = zip_ref.namelist()
|
| 31 |
+
for name in names:
|
| 32 |
+
if name.lower().endswith(".dex"):
|
| 33 |
+
return True
|
| 34 |
+
except Exception:
|
| 35 |
+
return False
|
| 36 |
+
return False
|
| 37 |
+
|
| 38 |
+
def count_existing_images(folder: str) -> int:
|
| 39 |
+
"""Count PNG files in a directory."""
|
| 40 |
+
if not os.path.isdir(folder):
|
| 41 |
+
return 0
|
| 42 |
+
return len([f for f in os.listdir(folder) if f.lower().endswith(".png")])
|
| 43 |
+
|
| 44 |
+
def main():
|
| 45 |
+
ensure_dir(RAW_APKS_DIR)
|
| 46 |
+
ensure_dir(GRAY_IMAGES_DIR)
|
| 47 |
+
ensure_dir(TEMP_DIR)
|
| 48 |
+
|
| 49 |
+
total_files = 0
|
| 50 |
+
total_converted = 0
|
| 51 |
+
total_skipped = 0
|
| 52 |
+
total_failed = 0
|
| 53 |
+
|
| 54 |
+
print("\n========== APK TO GRAYSCALE DATASET SETUP ==========")
|
| 55 |
+
print(f"Target images per family: {TARGET_IMAGES_PER_FAMILY}\n")
|
| 56 |
+
|
| 57 |
+
for family in FAMILIES:
|
| 58 |
+
input_family_dir = os.path.join(RAW_APKS_DIR, family)
|
| 59 |
+
output_family_dir = os.path.join(GRAY_IMAGES_DIR, family)
|
| 60 |
+
ensure_dir(output_family_dir)
|
| 61 |
+
|
| 62 |
+
# Count existing images
|
| 63 |
+
existing = count_existing_images(output_family_dir)
|
| 64 |
+
needed = TARGET_IMAGES_PER_FAMILY - existing
|
| 65 |
+
|
| 66 |
+
if needed <= 0:
|
| 67 |
+
print(f"\n--- Family: {family} ---")
|
| 68 |
+
print(f"Already have {existing} images (target {TARGET_IMAGES_PER_FAMILY}). Skipping.")
|
| 69 |
+
continue
|
| 70 |
+
|
| 71 |
+
if not os.path.isdir(input_family_dir):
|
| 72 |
+
print(f"[WARNING] Family folder not found: {input_family_dir}")
|
| 73 |
+
continue
|
| 74 |
+
|
| 75 |
+
files = [
|
| 76 |
+
f for f in os.listdir(input_family_dir)
|
| 77 |
+
if os.path.isfile(os.path.join(input_family_dir, f))
|
| 78 |
+
]
|
| 79 |
+
|
| 80 |
+
print(f"\n--- Processing family: {family} ---")
|
| 81 |
+
print(f"Input folder : {input_family_dir}")
|
| 82 |
+
print(f"Output folder: {output_family_dir}")
|
| 83 |
+
print(f"Existing images: {existing}")
|
| 84 |
+
print(f"Need {needed} more to reach {TARGET_IMAGES_PER_FAMILY}")
|
| 85 |
+
print(f"Available raw APKs: {len(files)}")
|
| 86 |
+
|
| 87 |
+
family_converted = 0
|
| 88 |
+
family_skipped = 0
|
| 89 |
+
family_failed = 0
|
| 90 |
+
|
| 91 |
+
# Process files until we have enough successful conversions
|
| 92 |
+
for file_name in files:
|
| 93 |
+
# Stop if we already reached the target
|
| 94 |
+
if family_converted >= needed:
|
| 95 |
+
break
|
| 96 |
+
|
| 97 |
+
total_files += 1
|
| 98 |
+
file_path = os.path.join(input_family_dir, file_name)
|
| 99 |
+
image_name = os.path.splitext(file_name)[0] + ".png"
|
| 100 |
+
output_image_path = os.path.join(output_family_dir, image_name)
|
| 101 |
+
|
| 102 |
+
# Skip if output already exists (to avoid reprocessing)
|
| 103 |
+
if os.path.isfile(output_image_path):
|
| 104 |
+
print(f"[SKIP] Already exists: {image_name}")
|
| 105 |
+
continue
|
| 106 |
+
|
| 107 |
+
if not is_valid_apk(file_path):
|
| 108 |
+
print(f"[SKIPPED] Not a valid APK: {file_name}")
|
| 109 |
+
total_skipped += 1
|
| 110 |
+
family_skipped += 1
|
| 111 |
+
continue
|
| 112 |
+
|
| 113 |
+
try:
|
| 114 |
+
apk_to_image_pipeline(
|
| 115 |
+
apk_path=file_path,
|
| 116 |
+
temp_dir=TEMP_DIR,
|
| 117 |
+
output_image_path=output_image_path,
|
| 118 |
+
final_size=FINAL_SIZE,
|
| 119 |
+
)
|
| 120 |
+
print(f"[OK] {file_name} -> {image_name}")
|
| 121 |
+
total_converted += 1
|
| 122 |
+
family_converted += 1
|
| 123 |
+
except Exception as e:
|
| 124 |
+
print(f"[FAILED] {file_name}: {e}")
|
| 125 |
+
total_failed += 1
|
| 126 |
+
family_failed += 1
|
| 127 |
+
|
| 128 |
+
print(f"\nFamily summary: {family}")
|
| 129 |
+
print(f"Converted this run: {family_converted}")
|
| 130 |
+
print(f"Skipped (invalid APK): {family_skipped}")
|
| 131 |
+
print(f"Failed (conversion error): {family_failed}")
|
| 132 |
+
print(f"Total images now in output: {existing + family_converted} / {TARGET_IMAGES_PER_FAMILY}")
|
| 133 |
+
|
| 134 |
+
print("\n========== FINAL DATASET REPORT ==========")
|
| 135 |
+
print(f"Total files processed (across all families): {total_files}")
|
| 136 |
+
print(f"Total converted (new images): {total_converted}")
|
| 137 |
+
print(f"Total skipped (invalid APKs): {total_skipped}")
|
| 138 |
+
print(f"Total failed (conversion errors): {total_failed}")
|
| 139 |
+
print("\nDone.")
|
| 140 |
+
|
| 141 |
+
if __name__ == "__main__":
|
| 142 |
+
main()
|
BACKEND/streamlit_app.py
ADDED
|
File without changes
|
BACKEND/support_embeddings.zip
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e1c8ad31ceceb345af0055921124428d9428cd8a5e4e2b682f4887c7aeea66b9
|
| 3 |
+
size 19914
|
BACKEND/train_and_prepare.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import subprocess
|
| 2 |
+
import sys
|
| 3 |
+
|
| 4 |
+
def run(cmd):
|
| 5 |
+
print(f"\nRunning: {cmd}\n")
|
| 6 |
+
result = subprocess.run(cmd, shell=True)
|
| 7 |
+
if result.returncode != 0:
|
| 8 |
+
raise SystemExit(f"Command failed: {cmd}")
|
| 9 |
+
|
| 10 |
+
if __name__ == "__main__":
|
| 11 |
+
run("python -m app.training.train_resnet34")
|
| 12 |
+
run("python -m app.support.save_support_embeddings")
|
| 13 |
+
print("\nTraining + support embedding preparation complete.")
|
BACKEND/trained_models/evaluation_metrics.json
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"overall_accuracy": 88.0,
|
| 3 |
+
"seen_accuracy": 93.33,
|
| 4 |
+
"unseen_accuracy": 80.0,
|
| 5 |
+
"family_accuracy": {
|
| 6 |
+
"adware": 80.0,
|
| 7 |
+
"banking": 80.0,
|
| 8 |
+
"benign": 100.0,
|
| 9 |
+
"riskware": 80.0,
|
| 10 |
+
"smsware": 100.0
|
| 11 |
+
},
|
| 12 |
+
"final_loss": null,
|
| 13 |
+
"confusion_matrix": {
|
| 14 |
+
"labels": [
|
| 15 |
+
"adware",
|
| 16 |
+
"banking",
|
| 17 |
+
"benign",
|
| 18 |
+
"riskware",
|
| 19 |
+
"smsware"
|
| 20 |
+
],
|
| 21 |
+
"data": [
|
| 22 |
+
[
|
| 23 |
+
4,
|
| 24 |
+
0,
|
| 25 |
+
0,
|
| 26 |
+
0,
|
| 27 |
+
1
|
| 28 |
+
],
|
| 29 |
+
[
|
| 30 |
+
0,
|
| 31 |
+
4,
|
| 32 |
+
0,
|
| 33 |
+
1,
|
| 34 |
+
0
|
| 35 |
+
],
|
| 36 |
+
[
|
| 37 |
+
0,
|
| 38 |
+
0,
|
| 39 |
+
5,
|
| 40 |
+
0,
|
| 41 |
+
0
|
| 42 |
+
],
|
| 43 |
+
[
|
| 44 |
+
1,
|
| 45 |
+
0,
|
| 46 |
+
0,
|
| 47 |
+
4,
|
| 48 |
+
0
|
| 49 |
+
],
|
| 50 |
+
[
|
| 51 |
+
0,
|
| 52 |
+
0,
|
| 53 |
+
0,
|
| 54 |
+
0,
|
| 55 |
+
5
|
| 56 |
+
]
|
| 57 |
+
]
|
| 58 |
+
}
|
| 59 |
+
}
|
BACKEND/trained_models/train_history.json
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"epoch": 1,
|
| 4 |
+
"loss": 0.4506978592146188
|
| 5 |
+
},
|
| 6 |
+
{
|
| 7 |
+
"epoch": 2,
|
| 8 |
+
"loss": 0.27496393894823634
|
| 9 |
+
},
|
| 10 |
+
{
|
| 11 |
+
"epoch": 3,
|
| 12 |
+
"loss": 0.1254722394684377
|
| 13 |
+
},
|
| 14 |
+
{
|
| 15 |
+
"epoch": 4,
|
| 16 |
+
"loss": 0.0696122650919956
|
| 17 |
+
},
|
| 18 |
+
{
|
| 19 |
+
"epoch": 5,
|
| 20 |
+
"loss": 0.04518843412091075
|
| 21 |
+
},
|
| 22 |
+
{
|
| 23 |
+
"epoch": 6,
|
| 24 |
+
"loss": 0.03079532652905391
|
| 25 |
+
},
|
| 26 |
+
{
|
| 27 |
+
"epoch": 7,
|
| 28 |
+
"loss": 0.025460728151967614
|
| 29 |
+
},
|
| 30 |
+
{
|
| 31 |
+
"epoch": 8,
|
| 32 |
+
"loss": 0.01982011537145435
|
| 33 |
+
},
|
| 34 |
+
{
|
| 35 |
+
"epoch": 9,
|
| 36 |
+
"loss": 0.017253640811187125
|
| 37 |
+
},
|
| 38 |
+
{
|
| 39 |
+
"epoch": 10,
|
| 40 |
+
"loss": 0.014067793231458623
|
| 41 |
+
},
|
| 42 |
+
{
|
| 43 |
+
"epoch": 11,
|
| 44 |
+
"loss": 0.01228759345602434
|
| 45 |
+
},
|
| 46 |
+
{
|
| 47 |
+
"epoch": 12,
|
| 48 |
+
"loss": 0.010558591230152275
|
| 49 |
+
},
|
| 50 |
+
{
|
| 51 |
+
"epoch": 13,
|
| 52 |
+
"loss": 0.009027703136934235
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"epoch": 14,
|
| 56 |
+
"loss": 0.006941486623447417
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"epoch": 15,
|
| 60 |
+
"loss": 0.005425198441284692
|
| 61 |
+
},
|
| 62 |
+
{
|
| 63 |
+
"epoch": 16,
|
| 64 |
+
"loss": 0.004878156908180927
|
| 65 |
+
},
|
| 66 |
+
{
|
| 67 |
+
"epoch": 17,
|
| 68 |
+
"loss": 0.004208211276602924
|
| 69 |
+
},
|
| 70 |
+
{
|
| 71 |
+
"epoch": 18,
|
| 72 |
+
"loss": 0.007560494044383972
|
| 73 |
+
},
|
| 74 |
+
{
|
| 75 |
+
"epoch": 19,
|
| 76 |
+
"loss": 0.00530580484863458
|
| 77 |
+
},
|
| 78 |
+
{
|
| 79 |
+
"epoch": 20,
|
| 80 |
+
"loss": 0.002409123999372241
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"epoch": 21,
|
| 84 |
+
"loss": 0.0017899585990744527
|
| 85 |
+
},
|
| 86 |
+
{
|
| 87 |
+
"epoch": 22,
|
| 88 |
+
"loss": 0.0011776607247920765
|
| 89 |
+
},
|
| 90 |
+
{
|
| 91 |
+
"epoch": 23,
|
| 92 |
+
"loss": 0.0008682082814884779
|
| 93 |
+
},
|
| 94 |
+
{
|
| 95 |
+
"epoch": 24,
|
| 96 |
+
"loss": 0.0005899137800715835
|
| 97 |
+
},
|
| 98 |
+
{
|
| 99 |
+
"epoch": 25,
|
| 100 |
+
"loss": 0.0004260540119180405
|
| 101 |
+
},
|
| 102 |
+
{
|
| 103 |
+
"epoch": 26,
|
| 104 |
+
"loss": 0.00020756016330960848
|
| 105 |
+
},
|
| 106 |
+
{
|
| 107 |
+
"epoch": 27,
|
| 108 |
+
"loss": 0.0028575134301947855
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
"epoch": 28,
|
| 112 |
+
"loss": 0.033012302423116126
|
| 113 |
+
},
|
| 114 |
+
{
|
| 115 |
+
"epoch": 29,
|
| 116 |
+
"loss": 0.0010803415227048987
|
| 117 |
+
},
|
| 118 |
+
{
|
| 119 |
+
"epoch": 30,
|
| 120 |
+
"loss": 0.000570854778501598
|
| 121 |
+
}
|
| 122 |
+
]
|
malware-detection-frontend
DELETED
|
@@ -1 +0,0 @@
|
|
| 1 |
-
Subproject commit 440de81d7bee80b7835ae54a55661f446b56b08e
|
|
|
|
|
|
malware-detection-frontend/.gitignore
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
|
| 2 |
+
|
| 3 |
+
# dependencies
|
| 4 |
+
/node_modules
|
| 5 |
+
/.pnp
|
| 6 |
+
.pnp.js
|
| 7 |
+
|
| 8 |
+
# testing
|
| 9 |
+
/coverage
|
| 10 |
+
|
| 11 |
+
# production
|
| 12 |
+
/build
|
| 13 |
+
|
| 14 |
+
# misc
|
| 15 |
+
.DS_Store
|
| 16 |
+
.env.local
|
| 17 |
+
.env.development.local
|
| 18 |
+
.env.test.local
|
| 19 |
+
.env.production.local
|
| 20 |
+
|
| 21 |
+
npm-debug.log*
|
| 22 |
+
yarn-debug.log*
|
| 23 |
+
yarn-error.log*
|
| 24 |
+
echo "node_modules/
|
| 25 |
+
/.next
|
| 26 |
+
/build
|
| 27 |
+
/.env
|
| 28 |
+
.DS_Store
|
| 29 |
+
*.log" > .gitignore
|
| 30 |
+
public/image/signup.png
|
| 31 |
+
public/image/signup1.png
|
malware-detection-frontend/README.md
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Getting Started with Create React App
|
| 2 |
+
|
| 3 |
+
This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app).
|
| 4 |
+
|
| 5 |
+
## Available Scripts
|
| 6 |
+
|
| 7 |
+
In the project directory, you can run:
|
| 8 |
+
|
| 9 |
+
### `npm start`
|
| 10 |
+
|
| 11 |
+
Runs the app in the development mode.\
|
| 12 |
+
Open [http://localhost:3000](http://localhost:3000) to view it in your browser.
|
| 13 |
+
|
| 14 |
+
The page will reload when you make changes.\
|
| 15 |
+
You may also see any lint errors in the console.
|
| 16 |
+
|
| 17 |
+
### `npm test`
|
| 18 |
+
|
| 19 |
+
Launches the test runner in the interactive watch mode.\
|
| 20 |
+
See the section about [running tests](https://facebook.github.io/create-react-app/docs/running-tests) for more information.
|
| 21 |
+
|
| 22 |
+
### `npm run build`
|
| 23 |
+
|
| 24 |
+
Builds the app for production to the `build` folder.\
|
| 25 |
+
It correctly bundles React in production mode and optimizes the build for the best performance.
|
| 26 |
+
|
| 27 |
+
The build is minified and the filenames include the hashes.\
|
| 28 |
+
Your app is ready to be deployed!
|
| 29 |
+
|
| 30 |
+
See the section about [deployment](https://facebook.github.io/create-react-app/docs/deployment) for more information.
|
| 31 |
+
|
| 32 |
+
### `npm run eject`
|
| 33 |
+
|
| 34 |
+
**Note: this is a one-way operation. Once you `eject`, you can't go back!**
|
| 35 |
+
|
| 36 |
+
If you aren't satisfied with the build tool and configuration choices, you can `eject` at any time. This command will remove the single build dependency from your project.
|
| 37 |
+
|
| 38 |
+
Instead, it will copy all the configuration files and the transitive dependencies (webpack, Babel, ESLint, etc) right into your project so you have full control over them. All of the commands except `eject` will still work, but they will point to the copied scripts so you can tweak them. At this point you're on your own.
|
| 39 |
+
|
| 40 |
+
You don't have to ever use `eject`. The curated feature set is suitable for small and middle deployments, and you shouldn't feel obligated to use this feature. However we understand that this tool wouldn't be useful if you couldn't customize it when you are ready for it.
|
| 41 |
+
|
| 42 |
+
## Learn More
|
| 43 |
+
|
| 44 |
+
You can learn more in the [Create React App documentation](https://facebook.github.io/create-react-app/docs/getting-started).
|
| 45 |
+
|
| 46 |
+
To learn React, check out the [React documentation](https://reactjs.org/).
|
| 47 |
+
|
| 48 |
+
### Code Splitting
|
| 49 |
+
|
| 50 |
+
This section has moved here: [https://facebook.github.io/create-react-app/docs/code-splitting](https://facebook.github.io/create-react-app/docs/code-splitting)
|
| 51 |
+
|
| 52 |
+
### Analyzing the Bundle Size
|
| 53 |
+
|
| 54 |
+
This section has moved here: [https://facebook.github.io/create-react-app/docs/analyzing-the-bundle-size](https://facebook.github.io/create-react-app/docs/analyzing-the-bundle-size)
|
| 55 |
+
|
| 56 |
+
### Making a Progressive Web App
|
| 57 |
+
|
| 58 |
+
This section has moved here: [https://facebook.github.io/create-react-app/docs/making-a-progressive-web-app](https://facebook.github.io/create-react-app/docs/making-a-progressive-web-app)
|
| 59 |
+
|
| 60 |
+
### Advanced Configuration
|
| 61 |
+
|
| 62 |
+
This section has moved here: [https://facebook.github.io/create-react-app/docs/advanced-configuration](https://facebook.github.io/create-react-app/docs/advanced-configuration)
|
| 63 |
+
|
| 64 |
+
### Deployment
|
| 65 |
+
|
| 66 |
+
This section has moved here: [https://facebook.github.io/create-react-app/docs/deployment](https://facebook.github.io/create-react-app/docs/deployment)
|
| 67 |
+
|
| 68 |
+
### `npm run build` fails to minify
|
| 69 |
+
|
| 70 |
+
This section has moved here: [https://facebook.github.io/create-react-app/docs/troubleshooting#npm-run-build-fails-to-minify](https://facebook.github.io/create-react-app/docs/troubleshooting#npm-run-build-fails-to-minify)
|