fix: sync checks local management status before downloading from R2
Browse filesWhen objectstore sync is about to download an auth file from R2,
it first queries CLIProxyAPI's local management API for runtime status.
If the auth is already marked invalid (status=error, unavailable,
status_message contains 401/unauthorized/token_invalidated etc),
the file is deleted from R2 instead of downloaded β preventing
invalid auths from being resurrected by sync.
- entrypoint.sh +2 -0
- objectstore_sync.py +78 -2
entrypoint.sh
CHANGED
|
@@ -140,6 +140,8 @@ ensure_objectstore_config() {
|
|
| 140 |
OBJECTSTORE_ROOT="$OBJECTSTORE_MIRROR_ROOT" \
|
| 141 |
OBJECTSTORE_CONFIG_FALLBACK="$CONFIG_PATH" \
|
| 142 |
MC_CONFIG_DIR="$MC_CONFIG_DIR" \
|
|
|
|
|
|
|
| 143 |
/usr/local/bin/python3 /opt/daili/objectstore_sync.py "$1" >/dev/null 2>&1
|
| 144 |
}
|
| 145 |
|
|
|
|
| 140 |
OBJECTSTORE_ROOT="$OBJECTSTORE_MIRROR_ROOT" \
|
| 141 |
OBJECTSTORE_CONFIG_FALLBACK="$CONFIG_PATH" \
|
| 142 |
MC_CONFIG_DIR="$MC_CONFIG_DIR" \
|
| 143 |
+
MANAGEMENT_PASSWORD="$MGMT_KEY_VALUE" \
|
| 144 |
+
PORT="$APP_PORT" \
|
| 145 |
/usr/local/bin/python3 /opt/daili/objectstore_sync.py "$1" >/dev/null 2>&1
|
| 146 |
}
|
| 147 |
|
objectstore_sync.py
CHANGED
|
@@ -9,6 +9,8 @@ import subprocess
|
|
| 9 |
import sys
|
| 10 |
from datetime import datetime, timezone
|
| 11 |
from pathlib import Path
|
|
|
|
|
|
|
| 12 |
|
| 13 |
|
| 14 |
def env_required(name: str) -> str:
|
|
@@ -160,6 +162,63 @@ def upload_file(rel: str) -> None:
|
|
| 160 |
run_mc(["cp", str(src), remote_path(rel)])
|
| 161 |
|
| 162 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 163 |
def restore() -> None:
|
| 164 |
ensure_alias()
|
| 165 |
ROOT.mkdir(parents=True, exist_ok=True)
|
|
@@ -184,12 +243,23 @@ def sync() -> None:
|
|
| 184 |
ensure_alias()
|
| 185 |
ROOT.mkdir(parents=True, exist_ok=True)
|
| 186 |
|
|
|
|
|
|
|
| 187 |
remote = remote_inventory()
|
| 188 |
local = local_inventory()
|
| 189 |
|
| 190 |
-
# Remote has, local doesn't β download
|
| 191 |
-
# Both have, md5 differs β compare mtime, newer wins
|
| 192 |
for rel, meta in remote.items():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 193 |
local_meta = local.get(rel)
|
| 194 |
if local_meta is None:
|
| 195 |
download_file(rel, meta)
|
|
@@ -203,6 +273,12 @@ def sync() -> None:
|
|
| 203 |
|
| 204 |
# Local has, remote doesn't β upload to remote (new file from management UI)
|
| 205 |
for rel in sorted(set(local) - set(remote)):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 206 |
upload_file(rel)
|
| 207 |
|
| 208 |
prune_empty_dirs(local_path(REMOTE_AUTHS_PREFIX))
|
|
|
|
| 9 |
import sys
|
| 10 |
from datetime import datetime, timezone
|
| 11 |
from pathlib import Path
|
| 12 |
+
from urllib.request import Request, urlopen
|
| 13 |
+
from urllib.error import HTTPError, URLError
|
| 14 |
|
| 15 |
|
| 16 |
def env_required(name: str) -> str:
|
|
|
|
| 162 |
run_mc(["cp", str(src), remote_path(rel)])
|
| 163 |
|
| 164 |
|
| 165 |
+
def delete_remote(rel: str) -> None:
|
| 166 |
+
run_mc(["rm", "--force", remote_path(rel)], check=False)
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
INVALID_STATUS_KEYWORDS = [
|
| 170 |
+
"token_invalidated",
|
| 171 |
+
"token_revoked",
|
| 172 |
+
"invalidated oauth token",
|
| 173 |
+
"authentication token has been invalidated",
|
| 174 |
+
"account has been deactivated",
|
| 175 |
+
"no_organization",
|
| 176 |
+
"unauthorized",
|
| 177 |
+
"401",
|
| 178 |
+
]
|
| 179 |
+
|
| 180 |
+
|
| 181 |
+
def _fetch_invalid_auth_names() -> set[str]:
|
| 182 |
+
"""Query local CLIProxyAPI management API for auth names with invalid status."""
|
| 183 |
+
mgmt_key = os.environ.get("MANAGEMENT_PASSWORD") or os.environ.get("API_KEY") or ""
|
| 184 |
+
port = os.environ.get("PORT", "8317")
|
| 185 |
+
if not mgmt_key:
|
| 186 |
+
return set()
|
| 187 |
+
url = f"http://127.0.0.1:{port}/v0/management/auth-files"
|
| 188 |
+
req = Request(url)
|
| 189 |
+
req.add_header("Authorization", f"Bearer {mgmt_key}")
|
| 190 |
+
try:
|
| 191 |
+
with urlopen(req, timeout=10) as resp:
|
| 192 |
+
data = json.loads(resp.read())
|
| 193 |
+
except (HTTPError, URLError, OSError, ValueError):
|
| 194 |
+
return set()
|
| 195 |
+
if not isinstance(data, dict):
|
| 196 |
+
return set()
|
| 197 |
+
files = data.get("files", [])
|
| 198 |
+
if not isinstance(files, list):
|
| 199 |
+
return set()
|
| 200 |
+
|
| 201 |
+
invalid_names: set[str] = set()
|
| 202 |
+
for entry in files:
|
| 203 |
+
if not isinstance(entry, dict):
|
| 204 |
+
continue
|
| 205 |
+
name = str(entry.get("name") or "").strip()
|
| 206 |
+
if not name:
|
| 207 |
+
continue
|
| 208 |
+
status = str(entry.get("status") or "").strip().lower()
|
| 209 |
+
status_message = str(entry.get("status_message") or "").strip().lower()
|
| 210 |
+
unavailable = entry.get("unavailable", False)
|
| 211 |
+
if status == "error" or unavailable:
|
| 212 |
+
invalid_names.add(name)
|
| 213 |
+
continue
|
| 214 |
+
if status_message:
|
| 215 |
+
for kw in INVALID_STATUS_KEYWORDS:
|
| 216 |
+
if kw in status_message:
|
| 217 |
+
invalid_names.add(name)
|
| 218 |
+
break
|
| 219 |
+
return invalid_names
|
| 220 |
+
|
| 221 |
+
|
| 222 |
def restore() -> None:
|
| 223 |
ensure_alias()
|
| 224 |
ROOT.mkdir(parents=True, exist_ok=True)
|
|
|
|
| 243 |
ensure_alias()
|
| 244 |
ROOT.mkdir(parents=True, exist_ok=True)
|
| 245 |
|
| 246 |
+
invalid_names = _fetch_invalid_auth_names()
|
| 247 |
+
|
| 248 |
remote = remote_inventory()
|
| 249 |
local = local_inventory()
|
| 250 |
|
| 251 |
+
# Remote has, local doesn't β download (unless invalid)
|
| 252 |
+
# Both have, md5 differs β compare mtime, newer wins (unless invalid)
|
| 253 |
for rel, meta in remote.items():
|
| 254 |
+
# Extract auth file name from rel path (e.g. "auths/codex-xxx-free.json" β "codex-xxx-free.json")
|
| 255 |
+
file_name = rel.rsplit("/", 1)[-1] if "/" in rel else rel
|
| 256 |
+
if file_name in invalid_names:
|
| 257 |
+
delete_remote(rel)
|
| 258 |
+
dest = local_path(rel)
|
| 259 |
+
if dest.is_file():
|
| 260 |
+
dest.unlink()
|
| 261 |
+
continue
|
| 262 |
+
|
| 263 |
local_meta = local.get(rel)
|
| 264 |
if local_meta is None:
|
| 265 |
download_file(rel, meta)
|
|
|
|
| 273 |
|
| 274 |
# Local has, remote doesn't β upload to remote (new file from management UI)
|
| 275 |
for rel in sorted(set(local) - set(remote)):
|
| 276 |
+
file_name = rel.rsplit("/", 1)[-1] if "/" in rel else rel
|
| 277 |
+
if file_name in invalid_names:
|
| 278 |
+
dest = local_path(rel)
|
| 279 |
+
if dest.is_file():
|
| 280 |
+
dest.unlink()
|
| 281 |
+
continue
|
| 282 |
upload_file(rel)
|
| 283 |
|
| 284 |
prune_empty_dirs(local_path(REMOTE_AUTHS_PREFIX))
|