models.py

# models.py
# 此檔案負責機器學習模型的載入、資料處理和異常偵測邏輯
# 包含 HDBSCAN 模型、特徵工程和預測功能

import os
import pandas as pd
import numpy as np
import joblib
import datetime
from config import MODEL_FILES, ACTION_MAP, SYSTEM_WEIGHTS, ENABLE_HDBSCAN
from utils import get_ip_category, get_geo_level, get_current_time
import gradio as gr

# 全域變數
le_action = None
baseline_df = None
baseline_X = None
AI_READY = False
DATA_READY = False # 新增：標記資料是否載入成功

# 全域統計與狀態變數
global_stats = {
    "total_count": 0,
    "abnormal_count": 0,
    "start_time": datetime.datetime.now()
}

# 日誌歷史
log_history = []

def load_models_and_data():
    """載入模型和歷史資料"""
    global le_action, baseline_df, baseline_X, AI_READY, DATA_READY, global_stats, log_history

    # 1. 優先載入資料 (圖表顯示核心)
    try:
        print("📂 正在從雲端獲取最新資料 (Hugging Face API)...")
        try:
            import logger_service
            cloud_df = logger_service.sync_from_hf()
            if cloud_df is not None:
                baseline_df = cloud_df
            else:
                raise Exception("Cloud sync returned None")
        except Exception as e:
            print(f"ℹ️ 使用本地備份資料庫 (雲端同步跳過: {e})")
            if os.path.exists(MODEL_FILES["baseline_logs"]):
                baseline_df = pd.read_csv(MODEL_FILES["baseline_logs"])
            else:
                baseline_df = pd.DataFrame(columns=["timestamp", "ip_address", "cookie_id", "account", "role", "action", "status"])
                baseline_df.to_csv(MODEL_FILES["baseline_logs"], index=False)

        baseline_df['timestamp'] = pd.to_datetime(baseline_df['timestamp'])
        baseline_df['clean_ip'] = baseline_df['ip_address'].astype(str).apply(lambda x: x.split(" ")[0])

        global_stats["total_count"] = len(baseline_df)
        global_stats["abnormal_count"] = baseline_df[baseline_df['status'].str.contains("異常", na=False)].shape[0]
        print(f"📊 統計初始化：總監測量 {global_stats['total_count']}, 異常數 {global_stats['abnormal_count']}")

        # 準備顯示資料
        display_df = baseline_df.sort_values(by='timestamp', ascending=False).head(20).copy()
        display_df['timestamp'] = display_df['timestamp'].dt.strftime('%Y-%m-%d %H:%M:%S')
        new_logs = display_df[['timestamp', 'ip_address', 'cookie_id', 'account', 'role', 'action', 'status']].to_dict('records')
        log_history.clear()
        log_history.extend(new_logs)
        
        DATA_READY = True
        print("✅ 歷史基準資料載入成功！")
    except Exception as e:
        DATA_READY = False
        print(f"❌ 資料載入失敗: {e}")

    # 2. 嘗試載入 AI 模型 (異常偵測核心)
    try:
        # 載入標籤編碼器
        le_action = joblib.load(MODEL_FILES["label_encoder"])

        # 準備特徵矩陣
        baseline_df['month'] = baseline_df['timestamp'].dt.month
        baseline_df['day'] = baseline_df['timestamp'].dt.day
        baseline_df['hour'] = baseline_df['timestamp'].dt.hour

        ip_codes = np.array([get_ip_category(ip) for ip in baseline_df['clean_ip'].values])

        actions = baseline_df['action'].astype(str).values
        known_actions = set(le_action.classes_)
        unknown_actions = set(actions) - known_actions
        if unknown_actions:
            le_action.classes_ = np.append(le_action.classes_, list(unknown_actions))

        baseline_X = np.column_stack((
            baseline_df['month'].values, baseline_df['day'].values, baseline_df['hour'].values,
            ip_codes, le_action.transform(actions)
        ))

        AI_READY = True
        print("✅ AI 模組載入成功！")

    except Exception as e:
        AI_READY = False
        if "KeyError: 118" in str(e) or "version https://git-lfs" in str(e):
             print(f"⚠️ AI 模組載入失敗：偵測到 Git LFS 指標檔案。請確保已安裝 git-lfs 並執行 `git lfs pull` 下載實際模型檔。")
        else:
             print(f"⚠️ AI 模組載入失敗 (不影響基本圖表顯示)。錯誤: {e}")

def check_anomaly(ip, account, action, log_time):
    """核心異常偵測邏輯"""
    if not AI_READY:
        return "success"
    if action in ["login_attempt", "logout"]:
        return "success"

    if not ENABLE_HDBSCAN:
        return "success"

    try:
        import hdbscan  # lazy import
    except Exception:
        return "success"

    try:
        ow = float(SYSTEM_WEIGHTS["ow"])
        tw = float(SYSTEM_WEIGHTS["tw"])
        iw = float(SYSTEM_WEIGHTS["iw"])
        dw = float(SYSTEM_WEIGHTS["dw"])
        gw = float(SYSTEM_WEIGHTS["gw"])
        fw = float(SYSTEM_WEIGHTS["fw"])

        geo_level = get_geo_level(ip)

        # 地理位置檢查
        if gw == 100 and geo_level > 0:
            return "⚠️異常_地理位置受限 (100%絕對鎖定: 僅限校內專網存取)"
        elif gw >= 90 and geo_level > 1:
            return "⚠️異常_地理位置受限 (高敏感防護: 僅限校園網路與宿舍)"
        elif gw >= 70 and geo_level > 2:
            return "⚠️異常_地理位置受限 (進階防護: 禁止公共場所與海外連線)"
        elif gw >= 50 and geo_level > 3:
            return "⚠️異常_地理位置受限 (預設防護: 禁止海外異常 IP)"

        current_dt = pd.to_datetime(str(log_time))
        cutoff_dt = current_dt - pd.Timedelta(seconds=60)

        # 頻率檢查
        recent_clicks = 0
        for log in log_history:
            log_dt = pd.to_datetime(log['timestamp'])
            if log_dt >= cutoff_dt:
                if log.get('account') == account and log.get('action') == action:
                    recent_clicks += 1
            else:
                break

        if fw <= 0:
            max_allowed_clicks = 100
        elif fw <= 10:
            max_allowed_clicks = 90
        elif fw <= 20:
            max_allowed_clicks = 80
        elif fw <= 30:
            max_allowed_clicks = 70
        elif fw <= 40:
            max_allowed_clicks = 60
        elif fw <= 50:
            max_allowed_clicks = 50
        elif fw <= 60:
            max_allowed_clicks = 40
        elif fw <= 70:
            max_allowed_clicks = 30
        elif fw <= 80:
            max_allowed_clicks = 15
        elif fw <= 90:
            max_allowed_clicks = 9
        else:
            max_allowed_clicks = 3

        if (recent_clicks + 1) > max_allowed_clicks:
            return f"⚠️異常_單一操作頻率過高 ({recent_clicks+1}次/分)"

        # HDBSCAN 異常偵測
        ip_code = float(get_ip_category(ip))
        action_code = float(le_action.transform([action])[0])

        current_month = float(current_dt.month)
        current_day = float(current_dt.day)
        current_hour = float(current_dt.hour)

        time_multiplier = tw / 100.0
        action_scale = (dw / 100.0) * 100.0

        scaled_baseline = baseline_X.copy().astype(float)
        scaled_baseline[:, 0] *= time_multiplier * 10.0
        scaled_baseline[:, 1] *= time_multiplier * 0.5
        scaled_baseline[:, 2] *= time_multiplier * 2.0
        scaled_baseline[:, 3] *= (iw / 100.0) * 10.0
        scaled_baseline[:, 4] *= action_scale

        v_month = current_month * time_multiplier * 10.0
        v_day = current_day * time_multiplier * 0.5
        v_hour = current_hour * time_multiplier * 2.0
        v_ip = ip_code * (iw / 100.0) * 10.0
        v_action = action_code * action_scale

        new_data_point = np.array([[v_month, v_day, v_hour, v_ip, v_action]], dtype=float)

        same_action_mask = (baseline_X[:, 4] == action_code)
        relevant_history = scaled_baseline[same_action_mask]

        other_history = scaled_baseline[~same_action_mask]
        if len(other_history) > 800:
            np.random.seed(int(current_day + current_hour))
            indices = np.random.choice(len(other_history), 800, replace=False)
            other_history = other_history[indices]

        fit_X = np.vstack((other_history, relevant_history, new_data_point))
        strictness = max(2, int(2 + (ow / 100.0) * 4))

        model = hdbscan.HDBSCAN(min_cluster_size=strictness, min_samples=2)
        clusters = model.fit_predict(fit_X)

        if clusters[-1] == -1:
            return "⚠️異常_作息或行為不符"

        if np.any(same_action_mask):
            distances = np.linalg.norm(relevant_history - new_data_point[0], axis=1)
            min_distance = np.min(distances)
            max_tolerance_distance = time_multiplier * 15.0 + 1.0

            if ip_code >= 3.0:
                max_tolerance_distance *= (1 - (iw / 100.0) * 0.6)

            if min_distance > max_tolerance_distance:
                return "⚠️異常_作息或行為不符"

        return "success"
    except ValueError:
        return "⚠️異常_未知特徵 (出現未授權的 IP 或操作)"
    except Exception as e:
        return f"⚠️系統錯誤: {str(e)}"

def record_log(ip, cookie, account, role, action, status, time_mode, custom_time):
    """記錄日誌"""
    global global_stats, log_history, baseline_X

    if time_mode == "自訂時間 (模擬過去/未來)" and custom_time and str(custom_time).strip() != "":
        try:
            valid_time = pd.to_datetime(str(custom_time))
            log_time = valid_time.strftime("%Y-%m-%d %H:%M:%S")
        except Exception:
            log_time = get_current_time()
            status = "⚠️格式錯誤_無效的自訂時間"
    else:
        log_time = get_current_time()

    if status == "success":
        ai_judgment = check_anomaly(ip, account, action, log_time)
        if ai_judgment != "success":
            status = ai_judgment
            
            # 如果是異常行為，觸發安全警報與驗證流程（以帳戶名稱作為 session key）
            if "異常" in status or "受限" in status:
                try:
                    from security import trigger_identity_verification
                    from config import FIXED_TEST_EMAIL
                    # 以帳戶名稱作為識別 key，信件收件人統一使用 FIXED_TEST_EMAIL
                    trigger_identity_verification(account, FIXED_TEST_EMAIL, log_time)
                    print(f"🚨 [Security Alert Triggered] Account: {account}, Status: {status}")
                except Exception as e:
                    print(f"❌ [Security Alert Error] {e}")

    new_log = {
        "timestamp": log_time,
        "ip_address": ip,
        "cookie_id": cookie,
        "account": account,
        "role": role,
        "action": action,
        "status": status
    }

    try:
        import logger_service
        logger_service.push_new_log(new_log)
    except:
        pass

    global_stats["total_count"] += 1
    if "異常" in str(status) or "錯誤" in str(status) or "受限" in str(status):
        global_stats["abnormal_count"] += 1

    log_history.insert(0, new_log)

    if status == "success" and AI_READY:
        try:
            current_time = pd.to_datetime(str(log_time))
            if action not in le_action.classes_:
                le_action.classes_ = np.append(le_action.classes_, action)
            new_row = np.array([[
                float(current_time.month),
                float(current_time.day),
                float(current_time.hour),
                float(get_ip_category(ip)),
                float(le_action.transform([action])[0])
            ]], dtype=float)
            baseline_X = np.vstack((baseline_X, new_row))
        except:
            pass

    df_all = pd.DataFrame(log_history)
    abnormal_logs = [log for log in log_history if "異常" in str(log.get("status", "")) or "錯誤" in str(log.get("status", "")) or "受限" in str(log.get("status", ""))]
    df_abnormal = pd.DataFrame(abnormal_logs) if abnormal_logs else pd.DataFrame(columns=df_all.columns)
    
    # 建立 Gradio 更新物件，用於更新管理員介面的異常 Log 文字框
    if "異常" in str(status) or "錯誤" in str(status) or "受限" in str(status):
        inp_upd = gr.update(value=f"時間: {log_time} | IP: {ip} | 帳號: {account} | 動作: {action} | 狀態: {status}")
    else:
        inp_upd = gr.update(value="")

    return df_all, df_abnormal, inp_upd