Upload 43 files

.gitignore

@@ -0,0 +1,10 @@
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.env
+.venv
+venv/
+ENV/
+*.sqlite3
+.pytest_cache/

Dockerfile

@@ -0,0 +1,24 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Install system dependencies required for psycopg2 / C extensions
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    gcc \
+    libpq-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy source code and models directory
+COPY . .
+COPY models_dir /app/models_dir
+
+# Set environment variables for Hugging Face Spaces
+ENV MODELS_DIR=/app/models_dir
+ENV PORT=7860
+
+EXPOSE 7860
+
+CMD ["sh", "-c", "uvicorn app.main:app --host 0.0.0.0 --port ${PORT:-7860}"]

README.md

@@ -1,13 +0,0 @@
----
-title: Arxen Backend
-emoji: 🌍
-colorFrom: yellow
-colorTo: blue
-sdk: gradio
-sdk_version: 6.20.0
-python_version: '3.12'
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,8 @@
+import uvicorn
+import os
+from app.main import app
+
+if __name__ == "__main__":
+    # Hugging Face Spaces expect services to run on port 7860
+    port = int(os.environ.get("PORT", 7860))
+    uvicorn.run(app, host="0.0.0.0", port=port)

app/api/routes_quarantine.py

@@ -0,0 +1,193 @@
+from fastapi import APIRouter, Depends, HTTPException
+from sqlalchemy.orm import Session
+from sqlalchemy import func
+from app.db.database import get_db
+from app.models.db_models import Quarantine, ScanLog, WhitelistDomain
+from app.schemas.schemas import QuarantineValidateRequest
+from app.models.cnn_model import predict_cnn
+from app.services.clustering_service import run_quarantine_clustering
+from app.services.retrain_service import export_retraining_dataset
+import datetime
+import urllib.parse
+
+router = APIRouter()
+
+@router.get("/")
+def get_quarantine_cases(db: Session = Depends(get_db)):
+    cases = db.query(Quarantine, ScanLog).join(ScanLog, Quarantine.scan_log_id == ScanLog.id).filter(Quarantine.status == 'pending').all()
+    
+    result = []
+    for q, s in cases:
+        result.append({
+            "quarantine_id": q.id,
+            "url": s.url,
+            "cnn_score": s.cnn_score,
+            "bert_score": s.bert_score,
+            "cluster_id": q.cluster_id,
+            "created_at": s.created_at
+        })
+    return result
+
+@router.post("/run-clustering")
+def trigger_clustering(db: Session = Depends(get_db)):
+    """
+    Menjalankan algoritma clustering berbasis kemiripan leksikal n-gram
+    untuk mengelompokkan URL pending karantina ke dalam cluster_id.
+    """
+    res = run_quarantine_clustering(db)
+    return res
+
+@router.get("/clusters")
+def get_quarantine_clusters(db: Session = Depends(get_db)):
+    """
+    Mengambil semua kasus karantina pending yang sudah dikelompokkan per cluster_id.
+    """
+    cases = db.query(Quarantine, ScanLog).join(ScanLog, Quarantine.scan_log_id == ScanLog.id).filter(Quarantine.status == 'pending').all()
+
+    clusters_map = {}
+    unclustered = []
+
+    for q, s in cases:
+        item = {
+            "quarantine_id": q.id,
+            "url": s.url,
+            "cnn_score": s.cnn_score,
+            "bert_score": s.bert_score,
+            "created_at": s.created_at
+        }
+        if q.cluster_id:
+            c_id = f"Cluster #{q.cluster_id}"
+            if c_id not in clusters_map:
+                clusters_map[c_id] = []
+            clusters_map[c_id].append(item)
+        else:
+            unclustered.append(item)
+
+    result = [
+        {"cluster_name": c_name, "count": len(items), "items": items}
+        for c_name, items in clusters_map.items()
+    ]
+
+    if unclustered:
+        result.append({"cluster_name": "Belum Dikelompokkan", "count": len(unclustered), "items": unclustered})
+
+    return result
+
+@router.post("/cluster/{cluster_id}/validate")
+def validate_cluster(cluster_id: int, request: QuarantineValidateRequest, db: Session = Depends(get_db)):
+    """
+    Validasi massal berbasis Cluster (Human-in-the-Loop 1-Klik).
+    Memvalidasi seluruh URL di dalam satu cluster_id sekaligus.
+    """
+    q_cases = db.query(Quarantine).filter(Quarantine.cluster_id == cluster_id, Quarantine.status == 'pending').all()
+
+    if not q_cases:
+        raise HTTPException(status_code=404, detail=f"Tidak ada URL pending di Cluster #{cluster_id}")
+
+    validated_count = 0
+    for q in q_cases:
+        q.status = request.status
+        q.validated_by = request.validated_by
+        q.validated_at = datetime.datetime.utcnow()
+
+        s = db.query(ScanLog).filter(ScanLog.id == q.scan_log_id).first()
+        if s:
+            if request.status == 'validated_safe':
+                s.final_verdict = 'safe'
+                domain = urllib.parse.urlparse(s.url).hostname
+                if domain:
+                    domain = domain.lower()
+                    existing_wl = db.query(WhitelistDomain).filter(WhitelistDomain.domain == domain).first()
+                    if not existing_wl:
+                        new_wl = WhitelistDomain(domain=domain, is_strict=0)
+                        db.add(new_wl)
+            elif request.status == 'validated_phishing':
+                s.final_verdict = 'phishing'
+        
+        validated_count += 1
+
+    db.commit()
+    return {
+        "message": f"Berhasil memvalidasi {validated_count} URL di Cluster #{cluster_id} sebagai '{request.status}'.",
+        "validated_count": validated_count
+    }
+
+@router.get("/export-dataset")
+def export_dataset_for_retraining(db: Session = Depends(get_db)):
+    """
+    Mengekspor seluruh data karantina yang telah divalidasi HITL menjadi
+    dataset baru untuk siklus Batch Retraining di Colab / Cloud.
+    """
+    data = export_retraining_dataset(db)
+    return data
+
+@router.post("/report")
+def report_url_to_quarantine(payload: dict, db: Session = Depends(get_db)):
+    """
+    Dipanggil dari ekstensi Chrome saat pengguna mengklik 'Laporkan ke Karantina'.
+    Membuat ScanLog + entri Quarantine agar admin bisa meninjau.
+    """
+    url = payload.get("url")
+    if not url:
+        raise HTTPException(status_code=400, detail="URL wajib diisi")
+
+    try:
+        cnn_score = predict_cnn(url)
+    except Exception:
+        cnn_score = 0.0
+
+    existing = db.query(Quarantine).join(ScanLog).filter(
+        ScanLog.url == url,
+        Quarantine.status == 'pending'
+    ).first()
+    if existing:
+        return {"message": "URL sudah ada di karantina", "quarantine_id": existing.id}
+
+    new_log = ScanLog(
+        url=url,
+        url_decoded=url,
+        cnn_score=cnn_score,
+        bert_score=0.0,
+        final_verdict="quarantine",
+        stage_triggered="cnn_only"
+    )
+    db.add(new_log)
+    db.commit()
+    db.refresh(new_log)
+
+    new_q = Quarantine(scan_log_id=new_log.id, status='pending')
+    db.add(new_q)
+    db.commit()
+    db.refresh(new_q)
+
+    # Otomatis jalankan clustering agar URL baru langsung mendapat cluster_id jika mirip dengan yang lain
+    run_quarantine_clustering(db)
+
+    return {"message": "URL berhasil dilaporkan ke karantina", "quarantine_id": new_q.id}
+
+@router.post("/{id}/validate")
+def validate_quarantine(id: int, request: QuarantineValidateRequest, db: Session = Depends(get_db)):
+    q = db.query(Quarantine).filter(Quarantine.id == id).first()
+    if not q:
+        raise HTTPException(status_code=404, detail="Quarantine case not found")
+        
+    q.status = request.status
+    q.validated_by = request.validated_by
+    q.validated_at = datetime.datetime.utcnow()
+    
+    s = db.query(ScanLog).filter(ScanLog.id == q.scan_log_id).first()
+    if s:
+        if request.status == 'validated_safe':
+            s.final_verdict = 'safe'
+            domain = urllib.parse.urlparse(s.url).hostname
+            if domain:
+                domain = domain.lower()
+                existing_wl = db.query(WhitelistDomain).filter(WhitelistDomain.domain == domain).first()
+                if not existing_wl:
+                    new_wl = WhitelistDomain(domain=domain, is_strict=0)
+                    db.add(new_wl)
+        elif request.status == 'validated_phishing':
+            s.final_verdict = 'phishing'
+            
+    db.commit()
+    return {"message": "Validation successful"}

app/api/routes_scan.py

@@ -0,0 +1,174 @@
+from fastapi import APIRouter, Depends, HTTPException
+from sqlalchemy.orm import Session
+from app.db.database import get_db
+from app.schemas.schemas import ScanUrlRequest, ScanUrlResponse, ScanDomRequest, ScanDomResponse
+from app.models.db_models import ScanLog, Quarantine, WhitelistDomain
+from app.models.cnn_model import predict_cnn
+import urllib.parse
+from app.models.bert_model import predict_bert
+import hashlib
+
+router = APIRouter()
+
+@router.post("/url", response_model=ScanUrlResponse)
+def scan_url(request: ScanUrlRequest, db: Session = Depends(get_db)):
+    # 0. Check Whitelist
+    domain = urllib.parse.urlparse(request.url).hostname
+    if domain:
+        domain = domain.lower()
+        whitelisted = db.query(WhitelistDomain).filter(WhitelistDomain.domain == domain).first()
+        if whitelisted:
+            if whitelisted.is_strict == 1:
+                return ScanUrlResponse(
+                    verdict="safe",
+                    cnn_score=0.0,
+                    stage="strict_whitelist",
+                    needs_dom_analysis=False
+                )
+            else:
+                return ScanUrlResponse(
+                    verdict="safe",
+                    cnn_score=0.0,
+                    stage="smart_whitelist",
+                    needs_dom_analysis=True
+                )
+
+    # 1. Run 1D-CNN inference
+    cnn_score = predict_cnn(request.url)
+    
+    # ============================================================
+    # MODE: CNN-ONLY (BERT dimatikan sementara)
+    # Set CNN_ONLY_MODE = False untuk mengaktifkan kembali BERT.
+    # ============================================================
+    CNN_ONLY_MODE = True
+
+    cnn_threshold_phishing = 0.904
+    cnn_threshold_safe = 0.10
+
+    # === HIGH-RISK HOSTING PLATFORMS ===
+    HIGH_RISK_HOSTING = {
+        "ipfs.io", "dweb.link", "cf-ipfs.com",
+        "web.app", "firebaseapp.com",
+        "sites.google.com",
+        "github.io",
+        "vercel.app", "netlify.app",
+        "workers.dev", "pages.dev",
+        "000webhostapp.com", "lima-city.de",
+        "glitch.me", "repl.co", "onrender.com",
+    }
+    is_high_risk_hosting = domain is not None and any(
+        domain == d or domain.endswith('.' + d) for d in HIGH_RISK_HOSTING
+    )
+
+    # === TRUSTED GOVERNMENT & INSTITUTION TLDs ===
+    TRUSTED_GOV_TLDS = (
+        ".go.id", ".ac.id", ".sch.id", ".mil.id", ".net.id", ".or.id",
+        ".gov", ".edu", ".mil",
+        ".gov.uk", ".ac.uk", ".gov.au", ".edu.au",
+        ".gov.sg", ".edu.sg",
+        ".gov.my", ".edu.my",
+    )
+    is_trusted_gov = domain is not None and any(
+        domain.endswith(tld) for tld in TRUSTED_GOV_TLDS
+    )
+
+    if cnn_score >= cnn_threshold_phishing and not is_trusted_gov:
+        # CNN sangat yakin ini phishing DAN bukan domain pemerintah — langsung blokir
+        verdict = "phishing"
+        stage = "cnn_only"
+        needs_dom = False
+    elif cnn_score <= cnn_threshold_safe or is_trusted_gov:
+        # CNN yakin aman, ATAU domain pemerintah terpercaya — anggap aman
+        verdict = "safe"
+        stage = "cnn_only"
+        needs_dom = False
+    else:
+        # Zona ambiguous atau high-risk hosting
+        if CNN_ONLY_MODE:
+            # Mode CNN-only: tandai sebagai quarantine, tidak kirim ke BERT
+            verdict = "quarantine"
+            stage = "cnn_only"
+            needs_dom = False
+        else:
+            # Mode normal: kirim ke BERT untuk analisis DOM
+            verdict = "safe"  # verdict sementara, BERT akan memutuskan
+            stage = "cnn_and_bert"
+            needs_dom = True
+        
+    return ScanUrlResponse(
+        verdict=verdict,
+        cnn_score=cnn_score,
+        stage=stage,
+        needs_dom_analysis=needs_dom
+    )
+
+@router.post("/dom", response_model=ScanDomResponse)
+def scan_dom(request: ScanDomRequest, db: Session = Depends(get_db)):
+    # 0. Check Smart Whitelist
+    domain = urllib.parse.urlparse(request.url).hostname
+    if domain:
+        domain = domain.lower()
+        whitelisted = db.query(WhitelistDomain).filter(WhitelistDomain.domain == domain).first()
+        if whitelisted and whitelisted.is_strict == 0:
+            current_len = len(request.sanitized_text)
+            if not whitelisted.dom_hash:
+                whitelisted.dom_hash = str(current_len)
+                db.commit()
+                return ScanDomResponse(verdict="safe", bert_score=0.0)
+            else:
+                try:
+                    old_len = int(whitelisted.dom_hash)
+                    if old_len > 0:
+                        ratio = current_len / old_len
+                        if 0.7 <= ratio <= 1.3:
+                            return ScanDomResponse(verdict="safe", bert_score=0.0)
+                        else:
+                            # Hijacked / Drastically changed! Remove from whitelist
+                            db.delete(whitelisted)
+                            db.commit()
+                except ValueError:
+                    pass
+
+    # 1. Run DistilBERT inference on request.sanitized_text
+    bert_score = predict_bert(request.sanitized_text)
+    
+    # 2. Threshold check
+    bert_threshold = 0.75
+    
+    if bert_score >= bert_threshold:
+        verdict = "phishing"
+    elif bert_score < 0.4:
+        verdict = "safe"
+    else:
+        verdict = "quarantine"
+        
+    # Re-calculate CNN score for logging purposes
+    from app.models.cnn_model import predict_cnn
+    actual_cnn_score = predict_cnn(request.url)
+
+    # Log to DB
+    new_log = ScanLog(
+        url=request.url,
+        url_decoded=request.url,
+        cnn_score=actual_cnn_score,
+        bert_score=bert_score,
+        final_verdict=verdict,
+        stage_triggered="cnn_and_bert"
+    )
+    db.add(new_log)
+    db.commit()
+    db.refresh(new_log)
+    
+    if verdict == "quarantine":
+        # Check if this URL is already pending in quarantine to prevent duplicates
+        existing_q = db.query(Quarantine).join(ScanLog).filter(
+            ScanLog.url == request.url,
+            Quarantine.status == 'pending'
+        ).first()
+        
+        if not existing_q:
+            new_q = Quarantine(scan_log_id=new_log.id, status='pending')
+            db.add(new_q)
+            db.commit()
+        
+    return ScanDomResponse(verdict=verdict, bert_score=bert_score)

app/api/routes_stats.py

@@ -0,0 +1,51 @@
+from fastapi import APIRouter, Depends
+from sqlalchemy.orm import Session
+from sqlalchemy import func
+from datetime import datetime, timedelta
+from app.db.database import get_db
+from app.models.db_models import ScanLog, Quarantine
+
+router = APIRouter()
+
+@router.get("/")
+def get_stats(db: Session = Depends(get_db)):
+    total_scans = db.query(func.count(ScanLog.id)).scalar()
+    total_phishing = db.query(func.count(ScanLog.id)).filter(ScanLog.final_verdict == 'phishing').scalar()
+    total_quarantine = db.query(func.count(Quarantine.id)).filter(Quarantine.status == 'pending').scalar()
+    
+    return {
+        "total_scans": total_scans or 0,
+        "total_blocked": total_phishing or 0,
+        "pending_quarantine": total_quarantine or 0
+    }
+
+@router.get("/daily")
+def get_daily_stats(db: Session = Depends(get_db)):
+    ten_days_ago = datetime.utcnow() - timedelta(days=10)
+    
+    results = (
+        db.query(
+            func.date(ScanLog.created_at).label('date'),
+            ScanLog.final_verdict,
+            func.count(ScanLog.id).label('count')
+        )
+        .filter(ScanLog.created_at >= ten_days_ago)
+        .group_by(func.date(ScanLog.created_at), ScanLog.final_verdict)
+        .all()
+    )
+    
+    daily_data = {}
+    
+    # Pre-fill last 10 days
+    for i in range(9, -1, -1):
+        day = (datetime.utcnow() - timedelta(days=i)).date()
+        daily_data[str(day)] = {"date": str(day), "safe": 0, "phishing": 0, "quarantine": 0}
+        
+    for row in results:
+        date_str = str(row.date)
+        if date_str in daily_data:
+            verdict = row.final_verdict
+            if verdict in daily_data[date_str]:
+                daily_data[date_str][verdict] = row.count
+                
+    return sorted(list(daily_data.values()), key=lambda x: x['date'])

app/api/routes_whitelist.py

@@ -0,0 +1,44 @@
+from fastapi import APIRouter, Depends, HTTPException
+from sqlalchemy.orm import Session
+from app.db.database import get_db
+from app.models.db_models import WhitelistDomain
+from app.schemas.schemas import WhitelistAddRequest
+import datetime
+
+router = APIRouter()
+
+@router.get("/")
+def get_whitelist(strict: str = None, db: Session = Depends(get_db)):
+    query = db.query(WhitelistDomain)
+    if strict is not None:
+        is_strict_val = 1 if strict.lower() == 'true' else 0
+        query = query.filter(WhitelistDomain.is_strict == is_strict_val)
+    domains = query.order_by(WhitelistDomain.created_at.desc()).all()
+    return [{"id": d.id, "domain": d.domain, "is_strict": d.is_strict == 1, "created_at": d.created_at} for d in domains]
+
+@router.post("/")
+def add_whitelist(request: WhitelistAddRequest, db: Session = Depends(get_db)):
+    domain = request.domain.strip().lower()
+    if not domain:
+        raise HTTPException(status_code=400, detail="Domain cannot be empty")
+        
+    existing = db.query(WhitelistDomain).filter(WhitelistDomain.domain == domain).first()
+    if existing:
+        raise HTTPException(status_code=400, detail="Domain is already in whitelist")
+        
+    new_domain = WhitelistDomain(domain=domain, is_strict=1)
+    db.add(new_domain)
+    db.commit()
+    db.refresh(new_domain)
+    return {"message": "Domain added to whitelist", "domain": new_domain.domain}
+
+@router.delete("/{domain}")
+def remove_whitelist(domain: str, db: Session = Depends(get_db)):
+    domain = domain.strip().lower()
+    existing = db.query(WhitelistDomain).filter(WhitelistDomain.domain == domain).first()
+    if not existing:
+        raise HTTPException(status_code=404, detail="Domain not found in whitelist")
+        
+    db.delete(existing)
+    db.commit()
+    return {"message": "Domain removed from whitelist"}

app/db/database.py

@@ -0,0 +1,21 @@
+import os
+from sqlalchemy import create_engine
+from sqlalchemy.orm import declarative_base, sessionmaker
+
+DATABASE_URL = os.environ.get("DATABASE_URL", "sqlite:///./arxen.db")
+
+# Fix Supabase / Heroku postgres:// -> postgresql:// for SQLAlchemy compatibility
+if DATABASE_URL.startswith("postgres://"):
+    DATABASE_URL = DATABASE_URL.replace("postgres://", "postgresql://", 1)
+
+engine = create_engine(DATABASE_URL)
+SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+
+Base = declarative_base()
+
+def get_db():
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()

app/main.py

@@ -0,0 +1,27 @@
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from app.api import routes_scan, routes_quarantine, routes_stats, routes_whitelist
+from app.db.database import engine, Base
+
+# Create tables
+Base.metadata.create_all(bind=engine)
+
+app = FastAPI(title="Arxen API", version="1.0.0")
+
+# Setup CORS for extension and Vercel frontend
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+app.include_router(routes_scan.router, prefix="/api/scan", tags=["Scan"])
+app.include_router(routes_quarantine.router, prefix="/api/quarantine", tags=["Quarantine"])
+app.include_router(routes_stats.router, prefix="/api/stats", tags=["Stats"])
+app.include_router(routes_whitelist.router, prefix="/api/whitelist", tags=["Whitelist"])
+
+@app.get("/")
+def read_root():
+    return {"message": "Arxen Backend is running"}

app/models/bert_model.py

@@ -0,0 +1,53 @@
+import os
+import logging
+from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification
+
+logger = logging.getLogger(__name__)
+
+MODELS_DIR = os.environ.get("MODELS_DIR", "/app/models_dir")
+BERT_MODEL_PATH = os.path.join(MODELS_DIR, "bert_model") # Should be a directory saved by save_pretrained
+
+_pipeline_loaded = False
+_bert_pipeline = None
+
+def load_bert_model():
+    global _pipeline_loaded, _bert_pipeline
+    if os.path.exists(BERT_MODEL_PATH) and os.path.isdir(BERT_MODEL_PATH):
+        try:
+            tokenizer = AutoTokenizer.from_pretrained(BERT_MODEL_PATH)
+            model = AutoModelForSequenceClassification.from_pretrained(BERT_MODEL_PATH)
+            _bert_pipeline = pipeline("text-classification", model=model, tokenizer=tokenizer, max_length=512, truncation=True)
+            _pipeline_loaded = True
+            logger.info("DistilBERT model loaded successfully.")
+        except Exception as e:
+            logger.error(f"Failed to load DistilBERT model: {e}")
+            _pipeline_loaded = False
+    else:
+        logger.warning(f"DistilBERT model directory not found at {BERT_MODEL_PATH}. Using dummy inference fallback.")
+        _pipeline_loaded = False
+
+# Initialize on startup
+load_bert_model()
+
+def predict_bert(text: str) -> float:
+    if not _pipeline_loaded:
+        # Dummy logic: look for suspicious keywords
+        score = 0.1
+        text_lower = text.lower()
+        suspicious_words = ['urgent', 'password', 'verify', 'account', 'login', 'suspend', 'blokir', 'verifikasi', 'akun']
+        found = sum(1 for w in suspicious_words if w in text_lower)
+        score += min(found * 0.15, 0.8)
+        return score
+        
+    result = _bert_pipeline(text)
+    # Pipeline usually returns [{'label': 'LABEL_1', 'score': 0.9}]
+    # We assume LABEL_1 means phishing (needs to match actual training)
+    if result and isinstance(result, list):
+        res = result[0]
+        # Map label to score
+        if "phish" in res['label'].lower() or res['label'] == 'LABEL_1':
+            return res['score']
+        else:
+            return 1.0 - res['score']
+            
+    return 0.0

app/models/cnn_model.py

@@ -0,0 +1,138 @@
+import os
+import json
+import torch
+import torch.nn as nn
+import logging
+import idna
+
+logger = logging.getLogger(__name__)
+
+# Constants
+MODELS_DIR = os.environ.get("MODELS_DIR", "/app/models_dir")
+CNN_MODEL_PATH = os.path.join(MODELS_DIR, "cnn_model.pt")
+CNN_CONFIG_PATH = os.path.join(MODELS_DIR, "preprocessing_config.json")
+CNN_VOCAB_PATH = os.path.join(MODELS_DIR, "vocab.json")
+
+# Dummy Fallback State
+_model_loaded = False
+_model = None
+_vocab = {}
+_max_length = 200
+_padding_strategy = "post" # "post" or "pre"
+
+class SimpleCNN(nn.Module):
+    def __init__(self, vocab_size, embedding_dim=32, max_length=200):
+        super(SimpleCNN, self).__init__()
+        self.embedding = nn.Embedding(vocab_size, embedding_dim)
+        self.conv1 = nn.Conv1d(embedding_dim, 64, kernel_size=3, padding=1)
+        self.pool = nn.AdaptiveMaxPool1d(1)
+        self.fc1 = nn.Linear(64, 32)
+        self.fc2 = nn.Linear(32, 1)
+        self.sigmoid = nn.Sigmoid()
+
+    def forward(self, x):
+        # x: (batch_size, seq_len)
+        x = self.embedding(x)
+        # x: (batch_size, seq_len, embedding_dim)
+        x = x.permute(0, 2, 1) # (batch_size, embedding_dim, seq_len)
+        x = torch.relu(self.conv1(x))
+        x = self.pool(x).squeeze(-1) # (batch_size, 64)
+        x = torch.relu(self.fc1(x))
+        x = self.fc2(x)
+        return self.sigmoid(x).squeeze(-1)
+
+def load_cnn_model():
+    global _model_loaded, _model, _vocab, _max_length, _padding_strategy
+    
+    config_path = CNN_CONFIG_PATH if os.path.exists(CNN_CONFIG_PATH) else CNN_VOCAB_PATH
+    
+    if os.path.exists(config_path) and os.path.exists(CNN_MODEL_PATH):
+        try:
+            with open(config_path, "r") as f:
+                config = json.load(f)
+                _vocab = config.get("vocab", {})
+                _max_length = config.get("max_length", 200)
+                _padding_strategy = config.get("padding_strategy", "post")
+            
+            vocab_size = len(_vocab) # Exact length from vocab.json (includes <PAD> and <UNK>)
+            
+            _model = SimpleCNN(vocab_size=vocab_size, max_length=_max_length)
+            _model.load_state_dict(torch.load(CNN_MODEL_PATH, map_location=torch.device('cpu')))
+            _model.eval()
+            _model_loaded = True
+            logger.info(f"1D-CNN Model and config loaded successfully from {config_path}.")
+        except Exception as e:
+            logger.error(f"Failed to load CNN model: {e}")
+            _model_loaded = False
+    else:
+        logger.error(
+            "[CNN] Model tidak terload! "
+            f"Pastikan file '{CNN_MODEL_PATH}' dan '{CNN_VOCAB_PATH}' ada dan dapat dibaca. "
+            "Sistem tidak dapat melakukan prediksi CNN."
+        )
+        _model_loaded = False
+
+# Initialize on startup
+load_cnn_model()
+
+def preprocess_url(url: str) -> str:
+    
+    try:
+        import urllib.parse
+        parsed = urllib.parse.urlparse(url)
+        
+        # Ambil scheme (http/https) — PENTING untuk model, persis seperti di Colab
+        scheme = parsed.scheme if parsed.scheme else 'http'
+        
+        # Ambil hostname saja (buang path, query, fragment)
+        hostname = parsed.hostname if parsed.hostname else url.split('/')[0]
+        
+        # Decode punycode (internationalized domains)
+        try:
+            hostname = idna.decode(hostname).encode('utf-8').decode('utf-8')
+        except Exception:
+            pass
+        
+        # Output: scheme://hostname — sama seperti format di Colab
+        # Path sengaja dibuang agar URL panjang (mis. Google Search) tidak mengacaukan model
+        return f"{scheme}://{hostname}"
+    except Exception:
+        return url
+
+def encode_url(url: str, vocab: dict, max_length: int, padding: str = "post") -> list:
+    encoded = []
+    for char in url:
+        encoded.append(vocab.get(char, 0)) # 0 for unknown
+        
+    if len(encoded) > max_length:
+        encoded = encoded[:max_length]
+    elif len(encoded) < max_length:
+        pad_len = max_length - len(encoded)
+        if padding == "post":
+            encoded.extend([0] * pad_len)
+        else:
+            encoded = ([0] * pad_len) + encoded
+            
+    return encoded
+
+def predict_cnn(url: str) -> float:
+    decoded_url = preprocess_url(url)
+    
+    if not _model_loaded:
+        logger.error("[CNN] Model tidak terload — prediksi tidak dapat dilakukan.")
+        raise RuntimeError(
+            "Model CNN tidak terload. "
+            f"Pastikan file model ada di '{MODELS_DIR}' dan container sudah direstart."
+        )
+        
+    encoded = encode_url(decoded_url, _vocab, _max_length, _padding_strategy)
+    tensor_input = torch.tensor([encoded], dtype=torch.long)
+    
+    with torch.no_grad():
+        output = _model(tensor_input)
+        # Model outputs 1 for Legitimate, 0 for Phishing.
+        # We invert it so that 1 means Phishing and 0 means Legitimate.
+        raw_score = output.item()
+        score = 1.0 - raw_score
+        
+    return score

app/models/db_models.py

@@ -0,0 +1,50 @@
+from sqlalchemy import Column, Integer, String, Float, Text, Enum, ForeignKey, TIMESTAMP, BigInteger
+from sqlalchemy.sql import func
+from app.db.database import Base
+
+class ScanLog(Base):
+    __tablename__ = "scan_logs"
+
+    id = Column(BigInteger, primary_key=True, index=True, autoincrement=True)
+    url = Column(Text, nullable=False)
+    url_decoded = Column(Text, nullable=False)
+    cnn_score = Column(Float, nullable=False)
+    bert_score = Column(Float, nullable=True)
+    final_verdict = Column(Enum('safe', 'phishing', 'quarantine'), nullable=False)
+    stage_triggered = Column(Enum('cnn_only', 'cnn_and_bert'), nullable=False)
+    created_at = Column(TIMESTAMP, server_default=func.now())
+
+class Quarantine(Base):
+    __tablename__ = "quarantine"
+
+    id = Column(BigInteger, primary_key=True, index=True, autoincrement=True)
+    scan_log_id = Column(BigInteger, ForeignKey("scan_logs.id"), nullable=False)
+    cluster_id = Column(Integer, nullable=True)
+    status = Column(Enum('pending', 'validated_safe', 'validated_phishing'), default='pending')
+    validated_by = Column(String(255), nullable=True)
+    validated_at = Column(TIMESTAMP, nullable=True)
+
+class ModelThreshold(Base):
+    __tablename__ = "model_thresholds"
+
+    id = Column(Integer, primary_key=True, index=True, autoincrement=True)
+    model_name = Column(String(50), nullable=False)
+    threshold_value = Column(Float, nullable=False)
+    updated_at = Column(TIMESTAMP, server_default=func.now(), onupdate=func.now())
+
+class LocalBlacklistCache(Base):
+    __tablename__ = "local_blacklist_cache"
+
+    id = Column(BigInteger, primary_key=True, index=True, autoincrement=True)
+    domain = Column(String(255), unique=True, nullable=False)
+    verdict = Column(Enum('safe', 'phishing'), nullable=False)
+    last_verified = Column(TIMESTAMP, server_default=func.now())
+
+class WhitelistDomain(Base):
+    __tablename__ = "whitelist_domains"
+
+    id = Column(Integer, primary_key=True, index=True, autoincrement=True)
+    domain = Column(String(255), unique=True, nullable=False)
+    is_strict = Column(Integer, default=1) # 1 for True, 0 for False (using Integer for sqlite/mysql compat)
+    dom_hash = Column(String(255), nullable=True)
+    created_at = Column(TIMESTAMP, server_default=func.now())

app/schemas/schemas.py

@@ -0,0 +1,26 @@
+from pydantic import BaseModel
+from typing import Optional
+
+class ScanUrlRequest(BaseModel):
+    url: str
+
+class ScanUrlResponse(BaseModel):
+    verdict: str
+    cnn_score: float
+    stage: str
+    needs_dom_analysis: bool = False
+
+class ScanDomRequest(BaseModel):
+    url: str
+    sanitized_text: str
+
+class ScanDomResponse(BaseModel):
+    verdict: str
+    bert_score: float
+
+class QuarantineValidateRequest(BaseModel):
+    status: str
+    validated_by: str
+
+class WhitelistAddRequest(BaseModel):
+    domain: str

app/services/clustering_service.py

@@ -0,0 +1,94 @@
+import math
+import urllib.parse
+from typing import List, Dict, Any
+from sqlalchemy.orm import Session
+from app.models.db_models import Quarantine, ScanLog
+
+def get_char_ngrams(text: str, n: int = 3) -> set:
+    """Generate character n-grams for a given string."""
+    text = text.lower().strip()
+    if len(text) < n:
+        return {text}
+    return {text[i:i+n] for i in range(len(text) - n + 1)}
+
+def jaccard_similarity(set1: set, set2: set) -> float:
+    """Calculate Jaccard similarity between two sets of n-grams."""
+    if not set1 or not set2:
+        return 0.0
+    intersection = len(set1.intersection(set2))
+    union = len(set1.union(set2))
+    return intersection / union if union > 0 else 0.0
+
+def hybrid_url_similarity(url1: str, url2: str) -> float:
+    """
+    Calculates hybrid similarity between two URLs by combining:
+    1. 60% Domain/Hostname N-gram similarity (detects domain ecosystems / campaign patterns)
+    2. 40% Full URL N-gram similarity (detects path & structure patterns)
+    """
+    h1 = urllib.parse.urlparse(url1).hostname or url1
+    h2 = urllib.parse.urlparse(url2).hostname or url2
+
+    sim_host = jaccard_similarity(get_char_ngrams(h1, 3), get_char_ngrams(h2, 3))
+    sim_full = jaccard_similarity(get_char_ngrams(url1, 3), get_char_ngrams(url2, 3))
+
+    return 0.6 * sim_host + 0.4 * sim_full
+
+def run_quarantine_clustering(db: Session, similarity_threshold: float = 0.18) -> Dict[str, Any]:
+    """
+    Groups pending quarantine URLs into clusters based on Hybrid N-Gram Similarity.
+    Threshold default = 0.18 (18%).
+    - Unrelated domains (e.g. bisa.ai vs wikipedia.org) have < 5% similarity -> Separate clusters.
+    - Related domain variants / campaigns (e.g. id.wikipedia, wikisource, wikimedia) have >= 18% similarity -> Same cluster.
+    """
+    # Fetch all pending quarantine cases
+    cases = db.query(Quarantine, ScanLog).join(ScanLog, Quarantine.scan_log_id == ScanLog.id).filter(Quarantine.status == 'pending').all()
+
+    if not cases:
+        return {
+            "message": "Tidak ada kasus karantina pending yang perlu dikelompokkan.",
+            "clustered_count": 0,
+            "clusters_created": 0
+        }
+
+    items = [
+        {"quarantine": q, "scan_log": s, "url": s.url}
+        for q, s in cases
+    ]
+
+    # Clustering algorithm (Greedy Connected Components based on Hybrid Similarity)
+    clusters = []
+    visited = set()
+
+    for i in range(len(items)):
+        if i in visited:
+            continue
+
+        current_cluster = [i]
+        visited.add(i)
+
+        for j in range(i + 1, len(items)):
+            if j in visited:
+                continue
+
+            sim = hybrid_url_similarity(items[i]["url"], items[j]["url"])
+            if sim >= similarity_threshold:
+                current_cluster.append(j)
+                visited.add(j)
+
+        clusters.append(current_cluster)
+
+    # Assign cluster_ids to database records
+    total_assigned = 0
+    for cluster_idx, member_indices in enumerate(clusters, start=1):
+        for idx in member_indices:
+            q_rec = items[idx]["quarantine"]
+            q_rec.cluster_id = cluster_idx
+            total_assigned += 1
+
+    db.commit()
+
+    return {
+        "message": f"Berhasil mengelompokkan {total_assigned} URL ke dalam {len(clusters)} cluster.",
+        "clustered_count": total_assigned,
+        "clusters_created": len(clusters)
+    }

app/services/retrain_service.py

@@ -0,0 +1,36 @@
+from typing import List, Dict, Any
+from sqlalchemy.orm import Session
+from app.models.db_models import Quarantine, ScanLog
+
+def export_retraining_dataset(db: Session) -> Dict[str, Any]:
+    """
+    Exports all validated quarantine records (validated_safe / validated_phishing)
+    into a structured dataset format ready for Batch Retraining in Google Colab.
+    
+    Label Mapping:
+    - Label 1 = Legitimate (Safe)
+    - Label 0 = Phishing
+    (Matches PhiUSIIL Dataset Standard)
+    """
+    cases = db.query(Quarantine, ScanLog).join(ScanLog, Quarantine.scan_log_id == ScanLog.id).filter(
+        Quarantine.status.in_(['validated_safe', 'validated_phishing'])
+    ).all()
+
+    dataset = []
+    for q, s in cases:
+        label = 1 if q.status == 'validated_safe' else 0
+        dataset.append({
+            "quarantine_id": q.id,
+            "url": s.url,
+            "label": label,
+            "status": q.status,
+            "validated_by": q.validated_by,
+            "validated_at": q.validated_at.isoformat() if q.validated_at else None,
+            "cnn_score": s.cnn_score,
+            "bert_score": s.bert_score
+        })
+
+    return {
+        "total_records": len(dataset),
+        "dataset": dataset
+    }

models_dir/cnn_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:538121299dad478885ffaf2d057d694bb85fe0b004f8204382581b14f43f5bcc
+size 46059

models_dir/vocab.json

@@ -0,0 +1 @@
+{"vocab": {"a": 1, "s": 2, "7": 3, "h": 4, "3": 5, "\u00a4": 6, "l": 7, "6": 8, "i": 9, "e": 10, "*": 11, ",": 12, "!": 13, ".": 14, "\u00c3": 15, "u": 16, "\u00ac": 17, "0": 18, "8": 19, "q": 20, "\u00c5": 21, "~": 22, "z": 23, "=": 24, "]": 25, "1": 26, "5": 27, "c": 28, "\u201c": 29, ":": 30, "(": 31, "o": 32, "%": 33, "w": 34, "$": 35, "g": 36, "&": 37, "t": 38, "4": 39, "f": 40, "'": 41, "\u00c2": 42, "y": 43, "n": 44, "p": 45, "9": 46, ";": 47, "2": 48, "@": 49, "#": 50, "r": 51, "m": 52, "_": 53, "\u201a": 54, "\u20ac": 55, "/": 56, "k": 57, "\u00a2": 58, "d": 59, "+": 60, ")": 61, "\u00b9": 62, "[": 63, "-": 64, "j": 65, "\u00a3": 66, "?": 67, "x": 68, "\u00e2": 69, "v": 70, "b": 71, "<PAD>": 0, "<UNK>": 72}, "max_length": 150}

requirements.txt

@@ -0,0 +1,10 @@
+fastapi==0.111.0
+uvicorn==0.29.0
+sqlalchemy==2.0.30
+pymysql==1.1.0
+psycopg2-binary==2.9.9
+pydantic==2.7.1
+transformers==4.40.1
+torch==2.3.0
+idna==3.7
+cryptography