Spaces:
Sleeping
Sleeping
Commit
·
e3a165a
1
Parent(s):
419bca9
Upd dynamic local/cloud mode rotator from ENV
Browse files- requirements-dev.txt +1 -0
- utils/cloud_llm.py +26 -1
- utils/drive_saver.py +47 -12
- utils/token.py +53 -3
requirements-dev.txt
CHANGED
|
@@ -22,6 +22,7 @@ scipy>=1.10.0
|
|
| 22 |
scikit-learn>=1.3.0
|
| 23 |
sentencepiece
|
| 24 |
sacremoses
|
|
|
|
| 25 |
|
| 26 |
# Vietnamese translation (Opus model fallback)
|
| 27 |
# Note: google-genai and google-auth packages are excluded for local mode
|
|
|
|
| 22 |
scikit-learn>=1.3.0
|
| 23 |
sentencepiece
|
| 24 |
sacremoses
|
| 25 |
+
protobuf
|
| 26 |
|
| 27 |
# Vietnamese translation (Opus model fallback)
|
| 28 |
# Note: google-genai and google-auth packages are excluded for local mode
|
utils/cloud_llm.py
CHANGED
|
@@ -3,7 +3,27 @@ import os
|
|
| 3 |
import logging
|
| 4 |
import requests
|
| 5 |
from typing import Optional
|
| 6 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
|
| 8 |
logger = logging.getLogger("llm")
|
| 9 |
if not logger.handlers:
|
|
@@ -49,8 +69,13 @@ class GeminiClient:
|
|
| 49 |
def __init__(self, rotator: KeyRotator, default_model: str):
|
| 50 |
self.rotator = rotator
|
| 51 |
self.default_model = default_model
|
|
|
|
| 52 |
|
| 53 |
def generate(self, prompt: str, model: Optional[str] = None, temperature: float = 0.2, max_output_tokens: int = 512) -> Optional[str]:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
key = self.rotator.next_key()
|
| 55 |
if not key:
|
| 56 |
return None
|
|
|
|
| 3 |
import logging
|
| 4 |
import requests
|
| 5 |
from typing import Optional
|
| 6 |
+
|
| 7 |
+
# Dynamic import for Google GenAI (only when not in local mode)
|
| 8 |
+
def _import_google_genai():
|
| 9 |
+
"""Dynamically import Google GenAI only when needed"""
|
| 10 |
+
try:
|
| 11 |
+
from google import genai
|
| 12 |
+
return genai
|
| 13 |
+
except ImportError as e:
|
| 14 |
+
raise ImportError(f"Google GenAI not available: {e}. Make sure IS_LOCAL=false and google-genai is installed.")
|
| 15 |
+
|
| 16 |
+
# Check if we're in local mode
|
| 17 |
+
IS_LOCAL = os.getenv("IS_LOCAL", "false").lower() == "true"
|
| 18 |
+
|
| 19 |
+
# Only import Google GenAI if not in local mode
|
| 20 |
+
if not IS_LOCAL:
|
| 21 |
+
try:
|
| 22 |
+
genai = _import_google_genai()
|
| 23 |
+
except ImportError:
|
| 24 |
+
genai = None
|
| 25 |
+
else:
|
| 26 |
+
genai = None
|
| 27 |
|
| 28 |
logger = logging.getLogger("llm")
|
| 29 |
if not logger.handlers:
|
|
|
|
| 69 |
def __init__(self, rotator: KeyRotator, default_model: str):
|
| 70 |
self.rotator = rotator
|
| 71 |
self.default_model = default_model
|
| 72 |
+
self.available = genai is not None and not IS_LOCAL
|
| 73 |
|
| 74 |
def generate(self, prompt: str, model: Optional[str] = None, temperature: float = 0.2, max_output_tokens: int = 512) -> Optional[str]:
|
| 75 |
+
if not self.available:
|
| 76 |
+
logger.warning("[LLM][Gemini] Google GenAI not available (local mode or import failed)")
|
| 77 |
+
return None
|
| 78 |
+
|
| 79 |
key = self.rotator.next_key()
|
| 80 |
if not key:
|
| 81 |
return None
|
utils/drive_saver.py
CHANGED
|
@@ -2,29 +2,51 @@
|
|
| 2 |
import os, json, logging
|
| 3 |
from typing import Optional
|
| 4 |
|
| 5 |
-
#
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
GOOGLE_DRIVE_AVAILABLE = False
|
| 13 |
-
# Create dummy classes for
|
| 14 |
class service_account:
|
| 15 |
class Credentials:
|
| 16 |
@staticmethod
|
| 17 |
def from_service_account_info(*args, **kwargs):
|
| 18 |
-
raise ImportError("Google Drive
|
| 19 |
|
| 20 |
class build:
|
| 21 |
@staticmethod
|
| 22 |
def build(*args, **kwargs):
|
| 23 |
-
raise ImportError("Google Drive
|
| 24 |
|
| 25 |
class MediaFileUpload:
|
| 26 |
def __init__(self, *args, **kwargs):
|
| 27 |
-
raise ImportError("Google Drive
|
| 28 |
|
| 29 |
from utils.token import get_credentials
|
| 30 |
|
|
@@ -45,6 +67,11 @@ class DriveSaver:
|
|
| 45 |
self.supports_all_drives = os.getenv("GDRIVE_FOLDER_IS_SHARED", "false").lower() in ("1","true","yes")
|
| 46 |
self.allow_sa_fallback = os.getenv("GDRIVE_ALLOW_SA_FALLBACK", "false").lower() in ("1","true","yes")
|
| 47 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
# Check if Google Drive is available
|
| 49 |
if not GOOGLE_DRIVE_AVAILABLE:
|
| 50 |
logger.warning("⚠️ Google Drive dependencies not available - DriveSaver will be disabled")
|
|
@@ -55,6 +82,10 @@ class DriveSaver:
|
|
| 55 |
self._initialize_service()
|
| 56 |
|
| 57 |
def _initialize_service(self):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 58 |
if not GOOGLE_DRIVE_AVAILABLE:
|
| 59 |
logger.warning("⚠️ Google Drive dependencies not available - skipping service initialization")
|
| 60 |
return
|
|
@@ -90,6 +121,10 @@ class DriveSaver:
|
|
| 90 |
logger.info("✅ Google Drive service initialized")
|
| 91 |
|
| 92 |
def upload_file_to_drive(self, file_path: str, folder_id: Optional[str] = None, mimetype: Optional[str] = None) -> bool:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 93 |
if not GOOGLE_DRIVE_AVAILABLE:
|
| 94 |
logger.warning("⚠️ Google Drive dependencies not available - upload skipped")
|
| 95 |
return False
|
|
@@ -116,7 +151,7 @@ class DriveSaver:
|
|
| 116 |
return False
|
| 117 |
|
| 118 |
def is_service_available(self) -> bool:
|
| 119 |
-
return GOOGLE_DRIVE_AVAILABLE and self.service is not None
|
| 120 |
|
| 121 |
def set_folder_id(self, folder_id: str):
|
| 122 |
self.folder_id = folder_id
|
|
|
|
| 2 |
import os, json, logging
|
| 3 |
from typing import Optional
|
| 4 |
|
| 5 |
+
# Check if we're in local mode
|
| 6 |
+
IS_LOCAL = os.getenv("IS_LOCAL", "false").lower() == "true"
|
| 7 |
+
|
| 8 |
+
# Conditional imports for Google Drive (only when not in local mode)
|
| 9 |
+
if not IS_LOCAL:
|
| 10 |
+
try:
|
| 11 |
+
from google.oauth2 import service_account
|
| 12 |
+
from googleapiclient.discovery import build
|
| 13 |
+
from googleapiclient.http import MediaFileUpload
|
| 14 |
+
GOOGLE_DRIVE_AVAILABLE = True
|
| 15 |
+
except ImportError:
|
| 16 |
+
GOOGLE_DRIVE_AVAILABLE = False
|
| 17 |
+
# Create dummy classes for when Google Drive is not available
|
| 18 |
+
class service_account:
|
| 19 |
+
class Credentials:
|
| 20 |
+
@staticmethod
|
| 21 |
+
def from_service_account_info(*args, **kwargs):
|
| 22 |
+
raise ImportError("Google Drive dependencies not available")
|
| 23 |
+
|
| 24 |
+
class build:
|
| 25 |
+
@staticmethod
|
| 26 |
+
def build(*args, **kwargs):
|
| 27 |
+
raise ImportError("Google Drive dependencies not available")
|
| 28 |
+
|
| 29 |
+
class MediaFileUpload:
|
| 30 |
+
def __init__(self, *args, **kwargs):
|
| 31 |
+
raise ImportError("Google Drive dependencies not available")
|
| 32 |
+
else:
|
| 33 |
+
# Local mode: Google Drive is not available
|
| 34 |
GOOGLE_DRIVE_AVAILABLE = False
|
| 35 |
+
# Create dummy classes for local mode
|
| 36 |
class service_account:
|
| 37 |
class Credentials:
|
| 38 |
@staticmethod
|
| 39 |
def from_service_account_info(*args, **kwargs):
|
| 40 |
+
raise ImportError("Google Drive not available in local mode")
|
| 41 |
|
| 42 |
class build:
|
| 43 |
@staticmethod
|
| 44 |
def build(*args, **kwargs):
|
| 45 |
+
raise ImportError("Google Drive not available in local mode")
|
| 46 |
|
| 47 |
class MediaFileUpload:
|
| 48 |
def __init__(self, *args, **kwargs):
|
| 49 |
+
raise ImportError("Google Drive not available in local mode")
|
| 50 |
|
| 51 |
from utils.token import get_credentials
|
| 52 |
|
|
|
|
| 67 |
self.supports_all_drives = os.getenv("GDRIVE_FOLDER_IS_SHARED", "false").lower() in ("1","true","yes")
|
| 68 |
self.allow_sa_fallback = os.getenv("GDRIVE_ALLOW_SA_FALLBACK", "false").lower() in ("1","true","yes")
|
| 69 |
|
| 70 |
+
# Check if we're in local mode
|
| 71 |
+
if IS_LOCAL:
|
| 72 |
+
logger.info("🏠 Local mode: Google Drive integration disabled")
|
| 73 |
+
return
|
| 74 |
+
|
| 75 |
# Check if Google Drive is available
|
| 76 |
if not GOOGLE_DRIVE_AVAILABLE:
|
| 77 |
logger.warning("⚠️ Google Drive dependencies not available - DriveSaver will be disabled")
|
|
|
|
| 82 |
self._initialize_service()
|
| 83 |
|
| 84 |
def _initialize_service(self):
|
| 85 |
+
if IS_LOCAL:
|
| 86 |
+
logger.info("🏠 Local mode: Skipping Google Drive service initialization")
|
| 87 |
+
return
|
| 88 |
+
|
| 89 |
if not GOOGLE_DRIVE_AVAILABLE:
|
| 90 |
logger.warning("⚠️ Google Drive dependencies not available - skipping service initialization")
|
| 91 |
return
|
|
|
|
| 121 |
logger.info("✅ Google Drive service initialized")
|
| 122 |
|
| 123 |
def upload_file_to_drive(self, file_path: str, folder_id: Optional[str] = None, mimetype: Optional[str] = None) -> bool:
|
| 124 |
+
if IS_LOCAL:
|
| 125 |
+
logger.info("🏠 Local mode: File upload to Google Drive skipped")
|
| 126 |
+
return False
|
| 127 |
+
|
| 128 |
if not GOOGLE_DRIVE_AVAILABLE:
|
| 129 |
logger.warning("⚠️ Google Drive dependencies not available - upload skipped")
|
| 130 |
return False
|
|
|
|
| 151 |
return False
|
| 152 |
|
| 153 |
def is_service_available(self) -> bool:
|
| 154 |
+
return not IS_LOCAL and GOOGLE_DRIVE_AVAILABLE and self.service is not None
|
| 155 |
|
| 156 |
def set_folder_id(self, folder_id: str):
|
| 157 |
self.folder_id = folder_id
|
utils/token.py
CHANGED
|
@@ -1,9 +1,49 @@
|
|
| 1 |
# GCS credential token refresher
|
| 2 |
import os, json, logging
|
| 3 |
from typing import Optional
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
|
| 8 |
logger = logging.getLogger("token")
|
| 9 |
if not logger.handlers:
|
|
@@ -31,6 +71,10 @@ def _ensure_dirs():
|
|
| 31 |
os.makedirs(base, exist_ok=True)
|
| 32 |
|
| 33 |
def get_credentials() -> Optional[Credentials]:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
# 1) Token file
|
| 35 |
if os.path.exists(TOKEN_FILE):
|
| 36 |
try:
|
|
@@ -68,6 +112,9 @@ def get_credentials() -> Optional[Credentials]:
|
|
| 68 |
return None
|
| 69 |
|
| 70 |
def build_auth_url(redirect_uri: str) -> str:
|
|
|
|
|
|
|
|
|
|
| 71 |
web = _load_oauth_client_web()
|
| 72 |
if not web:
|
| 73 |
raise RuntimeError("GDRIVE_CREDENTIALS_JSON missing or invalid ('web' section required)")
|
|
@@ -80,6 +127,9 @@ def build_auth_url(redirect_uri: str) -> str:
|
|
| 80 |
return auth_url
|
| 81 |
|
| 82 |
def exchange_code(code: str, redirect_uri: str) -> Credentials:
|
|
|
|
|
|
|
|
|
|
| 83 |
web = _load_oauth_client_web()
|
| 84 |
if not web:
|
| 85 |
raise RuntimeError("GDRIVE_CREDENTIALS_JSON missing or invalid ('web' section required)")
|
|
|
|
| 1 |
# GCS credential token refresher
|
| 2 |
import os, json, logging
|
| 3 |
from typing import Optional
|
| 4 |
+
|
| 5 |
+
# Dynamic imports for Google OAuth (only when not in local mode)
|
| 6 |
+
def _import_google_oauth():
|
| 7 |
+
"""Dynamically import Google OAuth libraries only when needed"""
|
| 8 |
+
try:
|
| 9 |
+
from google.oauth2.credentials import Credentials
|
| 10 |
+
from google_auth_oauthlib.flow import Flow
|
| 11 |
+
from google.auth.transport.requests import Request
|
| 12 |
+
return Credentials, Flow, Request
|
| 13 |
+
except ImportError as e:
|
| 14 |
+
raise ImportError(f"Google OAuth libraries not available: {e}. Make sure IS_LOCAL=false and google-auth packages are installed.")
|
| 15 |
+
|
| 16 |
+
# Check if we're in local mode
|
| 17 |
+
IS_LOCAL = os.getenv("IS_LOCAL", "false").lower() == "true"
|
| 18 |
+
|
| 19 |
+
# Only import Google OAuth libraries if not in local mode
|
| 20 |
+
if not IS_LOCAL:
|
| 21 |
+
try:
|
| 22 |
+
Credentials, Flow, Request = _import_google_oauth()
|
| 23 |
+
except ImportError:
|
| 24 |
+
# Create dummy classes for when Google OAuth is not available
|
| 25 |
+
class Credentials:
|
| 26 |
+
@staticmethod
|
| 27 |
+
def from_authorized_user_info(*args, **kwargs):
|
| 28 |
+
raise ImportError("Google OAuth not available")
|
| 29 |
+
class Flow:
|
| 30 |
+
@staticmethod
|
| 31 |
+
def from_client_config(*args, **kwargs):
|
| 32 |
+
raise ImportError("Google OAuth not available")
|
| 33 |
+
class Request:
|
| 34 |
+
pass
|
| 35 |
+
else:
|
| 36 |
+
# Create dummy classes for local mode
|
| 37 |
+
class Credentials:
|
| 38 |
+
@staticmethod
|
| 39 |
+
def from_authorized_user_info(*args, **kwargs):
|
| 40 |
+
raise ImportError("Google OAuth not available in local mode")
|
| 41 |
+
class Flow:
|
| 42 |
+
@staticmethod
|
| 43 |
+
def from_client_config(*args, **kwargs):
|
| 44 |
+
raise ImportError("Google OAuth not available in local mode")
|
| 45 |
+
class Request:
|
| 46 |
+
pass
|
| 47 |
|
| 48 |
logger = logging.getLogger("token")
|
| 49 |
if not logger.handlers:
|
|
|
|
| 71 |
os.makedirs(base, exist_ok=True)
|
| 72 |
|
| 73 |
def get_credentials() -> Optional[Credentials]:
|
| 74 |
+
if IS_LOCAL:
|
| 75 |
+
logger.info("🏠 Local mode: Google OAuth credentials not needed")
|
| 76 |
+
return None
|
| 77 |
+
|
| 78 |
# 1) Token file
|
| 79 |
if os.path.exists(TOKEN_FILE):
|
| 80 |
try:
|
|
|
|
| 112 |
return None
|
| 113 |
|
| 114 |
def build_auth_url(redirect_uri: str) -> str:
|
| 115 |
+
if IS_LOCAL:
|
| 116 |
+
raise RuntimeError("Google OAuth not available in local mode")
|
| 117 |
+
|
| 118 |
web = _load_oauth_client_web()
|
| 119 |
if not web:
|
| 120 |
raise RuntimeError("GDRIVE_CREDENTIALS_JSON missing or invalid ('web' section required)")
|
|
|
|
| 127 |
return auth_url
|
| 128 |
|
| 129 |
def exchange_code(code: str, redirect_uri: str) -> Credentials:
|
| 130 |
+
if IS_LOCAL:
|
| 131 |
+
raise RuntimeError("Google OAuth not available in local mode")
|
| 132 |
+
|
| 133 |
web = _load_oauth_client_web()
|
| 134 |
if not web:
|
| 135 |
raise RuntimeError("GDRIVE_CREDENTIALS_JSON missing or invalid ('web' section required)")
|