Spaces:
Running
Running
π± Imperial Deployment: Shadow Brain Core ignition
Browse files- VERSION +1 -1
- scripts/tools/cloud_tool.py +43 -27
VERSION
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
v2.9.
|
|
|
|
| 1 |
+
ο»Ώv2.9.71
|
scripts/tools/cloud_tool.py
CHANGED
|
@@ -71,32 +71,39 @@ import time as _time
|
|
| 71 |
_R2_LIST_CACHE: dict = {} # key: (bucket, path) β {'files': [...], 'ts': float}
|
| 72 |
_R2_LIST_TTL = 30 # μ΄
|
| 73 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
def _get_bw_credential(name):
|
| 75 |
-
"""Fetches a credential
|
| 76 |
-
κ²°κ³Όλ μΈλ©λͺ¨λ¦¬ μΊμ(_BW_CREDENTIAL_CACHE)μ μ μ₯
|
| 77 |
"""
|
| 78 |
-
# If it's already a value (not starting with Imperial:), return as is
|
| 79 |
if not isinstance(name, str) or "Imperial:" not in str(name):
|
| 80 |
return name
|
| 81 |
|
| 82 |
-
# π± μΊμ ννΈ: μ΄λ―Έ μΈμΆν μ격μ¦λͺ
μ μ¦μ λ°ν (BW CLI μλ΅)
|
| 83 |
if name in _BW_CREDENTIAL_CACHE:
|
| 84 |
return _BW_CREDENTIAL_CACHE[name]
|
| 85 |
|
| 86 |
-
#
|
| 87 |
-
# Strip prefix for env lookup: Imperial:R2_KEY -> R2_KEY
|
| 88 |
env_key = name.replace("Imperial:", "").replace("Imperial: ", "").strip()
|
| 89 |
env_val = os.environ.get(env_key)
|
|
|
|
| 90 |
if env_val and "Imperial:" not in str(env_val):
|
| 91 |
_BW_CREDENTIAL_CACHE[name] = env_val
|
| 92 |
return env_val
|
| 93 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 94 |
# Standardize both ways: 'Imperial: KEY' and 'Imperial:KEY'
|
| 95 |
name_no_space = name.replace("Imperial: ", "Imperial:").strip()
|
| 96 |
name_with_space = name_no_space.replace("Imperial:", "Imperial: ").strip()
|
| 97 |
|
| 98 |
candidates = [name, name_no_space, name_with_space]
|
| 99 |
-
# Remove duplicates while preserving order
|
| 100 |
unique_candidates = []
|
| 101 |
for c in candidates:
|
| 102 |
if c not in unique_candidates:
|
|
@@ -104,22 +111,20 @@ def _get_bw_credential(name):
|
|
| 104 |
|
| 105 |
for cand in unique_candidates:
|
| 106 |
try:
|
| 107 |
-
# Use shell=True for Windows to better find 'bw' in PATH
|
| 108 |
result = subprocess.run(
|
| 109 |
f'bw get password "{cand}" --raw',
|
| 110 |
capture_output=True, text=True, check=True, shell=True, timeout=30
|
| 111 |
)
|
| 112 |
val = result.stdout.strip()
|
| 113 |
if val:
|
| 114 |
-
# π± μΊμμ μ μ₯: λͺ¨λ λ³ν ν€λ₯Ό λμΌ κ°μΌλ‘ μΊμ±
|
| 115 |
for key in unique_candidates:
|
| 116 |
_BW_CREDENTIAL_CACHE[key] = val
|
| 117 |
return val
|
| 118 |
except:
|
| 119 |
continue
|
| 120 |
|
| 121 |
-
print(f"[Aegis Error] Failed to fetch {name}
|
| 122 |
-
return None
|
| 123 |
|
| 124 |
|
| 125 |
def clear_bw_credential_cache():
|
|
@@ -132,21 +137,17 @@ def clear_bw_credential_cache():
|
|
| 132 |
|
| 133 |
|
| 134 |
def _prewarm_credentials():
|
| 135 |
-
"""π± μλ² κΈ°λ μ λͺ¨λ R2 κ³μ μ μ격μ¦λͺ
μ μ μ λ‘λν©λλ€.
|
| 136 |
-
|
| 137 |
-
|
|
|
|
|
|
|
| 138 |
all_keys = set()
|
| 139 |
for acc in R2_ACCOUNTS.values():
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
if key_id and "Imperial:" in key_id:
|
| 145 |
-
all_keys.add(key_id)
|
| 146 |
-
if secret_id and "Imperial:" in secret_id:
|
| 147 |
-
all_keys.add(secret_id)
|
| 148 |
-
if acc_id and "Imperial:" in acc_id:
|
| 149 |
-
all_keys.add(acc_id)
|
| 150 |
|
| 151 |
loaded = 0
|
| 152 |
for key in all_keys:
|
|
@@ -154,11 +155,14 @@ def _prewarm_credentials():
|
|
| 154 |
val = _get_bw_credential(key)
|
| 155 |
if val and "Imperial:" not in str(val):
|
| 156 |
loaded += 1
|
| 157 |
-
|
|
|
|
| 158 |
|
| 159 |
|
| 160 |
-
# π₯ μλ²/λͺ¨λ κΈ°λ μ μλ Pre-warm (
|
| 161 |
def _prewarm_async():
|
|
|
|
|
|
|
| 162 |
import threading
|
| 163 |
t = threading.Thread(target=_prewarm_credentials, name="aegis-prewarm", daemon=True)
|
| 164 |
t.start()
|
|
@@ -346,11 +350,22 @@ def get_r2_bucket_size(account_config, bucket_name="cache"):
|
|
| 346 |
access_key = _get_bw_credential(account_config.get("access_key_id"))
|
| 347 |
secret_key = _get_bw_credential(account_config.get("secret_access_key"))
|
| 348 |
account_id = _get_bw_credential(account_config.get("account_id"))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 349 |
s3 = boto3.client(
|
| 350 |
service_name='s3',
|
| 351 |
endpoint_url=f'https://{account_id}.r2.cloudflarestorage.com',
|
| 352 |
aws_access_key_id=access_key,
|
| 353 |
aws_secret_access_key=secret_key,
|
|
|
|
|
|
|
| 354 |
)
|
| 355 |
total_bytes = 0
|
| 356 |
total_count = 0
|
|
@@ -360,7 +375,8 @@ def get_r2_bucket_size(account_config, bucket_name="cache"):
|
|
| 360 |
if not obj["Key"].endswith("/"): # ν΄λ μ체 μ μΈ
|
| 361 |
total_bytes += obj["Size"]
|
| 362 |
total_count += 1
|
| 363 |
-
|
|
|
|
| 364 |
except Exception as e:
|
| 365 |
raise RuntimeError(f"get_r2_bucket_size failed: {e}")
|
| 366 |
|
|
|
|
| 71 |
_R2_LIST_CACHE: dict = {} # key: (bucket, path) β {'files': [...], 'ts': float}
|
| 72 |
_R2_LIST_TTL = 30 # μ΄
|
| 73 |
|
| 74 |
+
# π± [Imperial Cloud Guard] Determine if running in cloud
|
| 75 |
+
IS_RENDER = os.environ.get('RENDER') == 'true'
|
| 76 |
+
IS_CLOUD_RUN = os.environ.get('K_SERVICE') is not None
|
| 77 |
+
IS_CLOUD = IS_RENDER or IS_CLOUD_RUN
|
| 78 |
+
|
| 79 |
def _get_bw_credential(name):
|
| 80 |
+
"""Fetches a credential. On Cloud (Render/GCP), skips BW and uses Env only.
|
| 81 |
+
κ²°κ³Όλ μΈλ©λͺ¨λ¦¬ μΊμ(_BW_CREDENTIAL_CACHE)μ μ μ₯λ©λλ€.
|
| 82 |
"""
|
|
|
|
| 83 |
if not isinstance(name, str) or "Imperial:" not in str(name):
|
| 84 |
return name
|
| 85 |
|
|
|
|
| 86 |
if name in _BW_CREDENTIAL_CACHE:
|
| 87 |
return _BW_CREDENTIAL_CACHE[name]
|
| 88 |
|
| 89 |
+
# Standardize key for Env lookup
|
|
|
|
| 90 |
env_key = name.replace("Imperial:", "").replace("Imperial: ", "").strip()
|
| 91 |
env_val = os.environ.get(env_key)
|
| 92 |
+
|
| 93 |
if env_val and "Imperial:" not in str(env_val):
|
| 94 |
_BW_CREDENTIAL_CACHE[name] = env_val
|
| 95 |
return env_val
|
| 96 |
|
| 97 |
+
# π± Cloud Environment: BW CLI is not available.
|
| 98 |
+
if IS_CLOUD:
|
| 99 |
+
print(f"[π± Cloud Alert] Missing Environment Variable: {env_key}")
|
| 100 |
+
return None
|
| 101 |
+
|
| 102 |
# Standardize both ways: 'Imperial: KEY' and 'Imperial:KEY'
|
| 103 |
name_no_space = name.replace("Imperial: ", "Imperial:").strip()
|
| 104 |
name_with_space = name_no_space.replace("Imperial:", "Imperial: ").strip()
|
| 105 |
|
| 106 |
candidates = [name, name_no_space, name_with_space]
|
|
|
|
| 107 |
unique_candidates = []
|
| 108 |
for c in candidates:
|
| 109 |
if c not in unique_candidates:
|
|
|
|
| 111 |
|
| 112 |
for cand in unique_candidates:
|
| 113 |
try:
|
|
|
|
| 114 |
result = subprocess.run(
|
| 115 |
f'bw get password "{cand}" --raw',
|
| 116 |
capture_output=True, text=True, check=True, shell=True, timeout=30
|
| 117 |
)
|
| 118 |
val = result.stdout.strip()
|
| 119 |
if val:
|
|
|
|
| 120 |
for key in unique_candidates:
|
| 121 |
_BW_CREDENTIAL_CACHE[key] = val
|
| 122 |
return val
|
| 123 |
except:
|
| 124 |
continue
|
| 125 |
|
| 126 |
+
print(f"[Aegis Error] Failed to fetch {name} from BW/Env. (Key: {env_key})")
|
| 127 |
+
return None
|
| 128 |
|
| 129 |
|
| 130 |
def clear_bw_credential_cache():
|
|
|
|
| 137 |
|
| 138 |
|
| 139 |
def _prewarm_credentials():
|
| 140 |
+
"""π± μλ² κΈ°λ μ λͺ¨λ R2 κ³μ μ μ격μ¦λͺ
μ μ μ λ‘λν©λλ€."""
|
| 141 |
+
# Cloudμμλ μ΄μ°¨νΌ Envμμ κ°μ Έμ€λ―λ‘ Pre-warmμ΄ λΆνμνκ±°λ μλ¬ λ©μμ§λ§ λ¨λ°ν¨
|
| 142 |
+
if IS_CLOUD:
|
| 143 |
+
return
|
| 144 |
+
|
| 145 |
all_keys = set()
|
| 146 |
for acc in R2_ACCOUNTS.values():
|
| 147 |
+
for k in ["access_key_id", "secret_access_key", "account_id"]:
|
| 148 |
+
key_val = acc.get(k, "")
|
| 149 |
+
if key_val and "Imperial:" in str(key_val):
|
| 150 |
+
all_keys.add(key_val)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 151 |
|
| 152 |
loaded = 0
|
| 153 |
for key in all_keys:
|
|
|
|
| 155 |
val = _get_bw_credential(key)
|
| 156 |
if val and "Imperial:" not in str(val):
|
| 157 |
loaded += 1
|
| 158 |
+
if loaded > 0:
|
| 159 |
+
print(f"[Aegis Pre-warm] R2 μ격μ¦λͺ
{loaded}/{len(all_keys)}κ° μ μ λ‘λ μλ£.")
|
| 160 |
|
| 161 |
|
| 162 |
+
# π₯ μλ²/λͺ¨λ κΈ°λ μ μλ Pre-warm (λ‘컬 μ μ©)
|
| 163 |
def _prewarm_async():
|
| 164 |
+
if IS_CLOUD:
|
| 165 |
+
return
|
| 166 |
import threading
|
| 167 |
t = threading.Thread(target=_prewarm_credentials, name="aegis-prewarm", daemon=True)
|
| 168 |
t.start()
|
|
|
|
| 350 |
access_key = _get_bw_credential(account_config.get("access_key_id"))
|
| 351 |
secret_key = _get_bw_credential(account_config.get("secret_access_key"))
|
| 352 |
account_id = _get_bw_credential(account_config.get("account_id"))
|
| 353 |
+
|
| 354 |
+
if not all([access_key, secret_key, account_id]):
|
| 355 |
+
missing = []
|
| 356 |
+
if not access_key: missing.append("ACCESS_KEY_ID")
|
| 357 |
+
if not secret_key: missing.append("SECRET_ACCESS_KEY")
|
| 358 |
+
if not account_id: missing.append("ACCOUNT_ID")
|
| 359 |
+
raise RuntimeError(f"Missing R2 Credentials in Environment: {', '.join(missing)}")
|
| 360 |
+
|
| 361 |
+
from botocore.config import Config
|
| 362 |
s3 = boto3.client(
|
| 363 |
service_name='s3',
|
| 364 |
endpoint_url=f'https://{account_id}.r2.cloudflarestorage.com',
|
| 365 |
aws_access_key_id=access_key,
|
| 366 |
aws_secret_access_key=secret_key,
|
| 367 |
+
region_name='auto',
|
| 368 |
+
config=Config(signature_version='s3v4')
|
| 369 |
)
|
| 370 |
total_bytes = 0
|
| 371 |
total_count = 0
|
|
|
|
| 375 |
if not obj["Key"].endswith("/"): # ν΄λ μ체 μ μΈ
|
| 376 |
total_bytes += obj["Size"]
|
| 377 |
total_count += 1
|
| 378 |
+
formatted = f"{total_bytes / (1024*1024):.2f} MB" if total_bytes < 1024**3 else f"{total_bytes / (1024**3):.2f} GB"
|
| 379 |
+
return {"count": total_count, "bytes": total_bytes, "formatted": formatted}
|
| 380 |
except Exception as e:
|
| 381 |
raise RuntimeError(f"get_r2_bucket_size failed: {e}")
|
| 382 |
|