Spaces:
Running
Running
Upload 云端_定时版本检测引擎.py
Browse files- 云端_定时版本检测引擎.py +15 -1
云端_定时版本检测引擎.py
CHANGED
|
@@ -106,23 +106,29 @@ async def precache_github_zip(repo_url: str, token: Optional[str], item_id: str,
|
|
| 106 |
client = None
|
| 107 |
response = None
|
| 108 |
resources = None
|
|
|
|
| 109 |
try:
|
| 110 |
storage_repo = archive_repository(db.DATASET_REPO_ID)
|
| 111 |
from proxy_access import read_proxy_item
|
|
|
|
| 112 |
item = await run_owned_thread(read_proxy_item, db, item_id)
|
|
|
|
| 113 |
current_token = select_github_token(item)
|
| 114 |
if repository_parts(repo_url) != repository_parts(item['link']) or token != current_token:
|
| 115 |
return False
|
| 116 |
owner, repo = repository_parts(repo_url)
|
|
|
|
| 117 |
ref = commit_ref(version_hash)
|
| 118 |
sha = commit_sha(ref) if len(ref) == 40 else await fetch_github_commit(repo_url, token, ref)
|
| 119 |
if not sha:
|
| 120 |
return False
|
|
|
|
| 121 |
signature = source_signature(item, token)
|
| 122 |
path_in_repo = archive_path(item_id, sha, signature)
|
| 123 |
hf_token = os.environ.get("HF_TOKEN")
|
| 124 |
if not hf_token:
|
| 125 |
return False
|
|
|
|
| 126 |
storage = await run_owned_thread(HfApi(token=hf_token).repo_info, repo_id=storage_repo, repo_type="dataset")
|
| 127 |
if getattr(storage, 'private', None) is not True:
|
| 128 |
logger.warning("ZIP预缓存要求私有Dataset,跳过发布")
|
|
@@ -130,6 +136,7 @@ async def precache_github_zip(repo_url: str, token: Optional[str], item_id: str,
|
|
| 130 |
headers = {"Accept": "application/vnd.github.v3+json", "User-Agent": "ComfyUI-Hub", "Accept-Encoding": "identity"}
|
| 131 |
if token:
|
| 132 |
headers["Authorization"] = f"Bearer {token}"
|
|
|
|
| 133 |
client = httpx.AsyncClient(follow_redirects=False, trust_env=False, transport=ValidatedAsyncHTTPTransport())
|
| 134 |
resources = ProxyResources(client)
|
| 135 |
# 共用逐跳公网/DNS/TLS与跨origin凭据隔离;归档重定向仍需验证。
|
|
@@ -138,6 +145,7 @@ async def precache_github_zip(repo_url: str, token: Optional[str], item_id: str,
|
|
| 138 |
f"https://api.github.com/repos/{owner}/{repo}/zipball/{sha}", headers=headers, timeout=300))
|
| 139 |
if response.status_code != 200:
|
| 140 |
return False
|
|
|
|
| 141 |
fd, temp_path = tempfile.mkstemp(suffix=".zip", prefix="ranking-precache-")
|
| 142 |
size = 0
|
| 143 |
with os.fdopen(fd, 'wb') as output:
|
|
@@ -146,11 +154,16 @@ async def precache_github_zip(repo_url: str, token: Optional[str], item_id: str,
|
|
| 146 |
if size > ZIP_MAX_SIZE:
|
| 147 |
return False
|
| 148 |
output.write(chunk)
|
|
|
|
| 149 |
await resources.close()
|
|
|
|
| 150 |
await run_owned_thread(validate_zip_archive, temp_path, size)
|
|
|
|
| 151 |
uploaded = await run_owned_thread(upload_file, path_or_fileobj=temp_path, path_in_repo=path_in_repo,
|
| 152 |
repo_id=storage_repo, repo_type="dataset", token=hf_token)
|
|
|
|
| 153 |
hf_commit = commit_sha(getattr(uploaded, 'oid', None))
|
|
|
|
| 154 |
latest_item = await run_owned_thread(read_proxy_item, db, item_id)
|
| 155 |
if source_signature(latest_item, select_github_token(latest_item)) != signature:
|
| 156 |
return False
|
|
@@ -161,9 +174,10 @@ async def precache_github_zip(repo_url: str, token: Optional[str], item_id: str,
|
|
| 161 |
if db.DATASET_REPO_ID != storage_repo:
|
| 162 |
return False
|
| 163 |
return publish_archive_metadata(versions, item_id, ref, metadata)
|
|
|
|
| 164 |
return bool(await run_owned_thread(db.atomic_update, "versions.json", update, default_data={}))
|
| 165 |
except Exception as error:
|
| 166 |
-
logger.warning("预缓存失败: error_type=%s", type(error).__name__)
|
| 167 |
return False
|
| 168 |
finally:
|
| 169 |
try:
|
|
|
|
| 106 |
client = None
|
| 107 |
response = None
|
| 108 |
resources = None
|
| 109 |
+
stage = "storage_identity"
|
| 110 |
try:
|
| 111 |
storage_repo = archive_repository(db.DATASET_REPO_ID)
|
| 112 |
from proxy_access import read_proxy_item
|
| 113 |
+
stage = "resource_read"
|
| 114 |
item = await run_owned_thread(read_proxy_item, db, item_id)
|
| 115 |
+
stage = "source_validation"
|
| 116 |
current_token = select_github_token(item)
|
| 117 |
if repository_parts(repo_url) != repository_parts(item['link']) or token != current_token:
|
| 118 |
return False
|
| 119 |
owner, repo = repository_parts(repo_url)
|
| 120 |
+
stage = "commit_resolution"
|
| 121 |
ref = commit_ref(version_hash)
|
| 122 |
sha = commit_sha(ref) if len(ref) == 40 else await fetch_github_commit(repo_url, token, ref)
|
| 123 |
if not sha:
|
| 124 |
return False
|
| 125 |
+
stage = "archive_identity"
|
| 126 |
signature = source_signature(item, token)
|
| 127 |
path_in_repo = archive_path(item_id, sha, signature)
|
| 128 |
hf_token = os.environ.get("HF_TOKEN")
|
| 129 |
if not hf_token:
|
| 130 |
return False
|
| 131 |
+
stage = "storage_permission"
|
| 132 |
storage = await run_owned_thread(HfApi(token=hf_token).repo_info, repo_id=storage_repo, repo_type="dataset")
|
| 133 |
if getattr(storage, 'private', None) is not True:
|
| 134 |
logger.warning("ZIP预缓存要求私有Dataset,跳过发布")
|
|
|
|
| 136 |
headers = {"Accept": "application/vnd.github.v3+json", "User-Agent": "ComfyUI-Hub", "Accept-Encoding": "identity"}
|
| 137 |
if token:
|
| 138 |
headers["Authorization"] = f"Bearer {token}"
|
| 139 |
+
stage = "archive_request"
|
| 140 |
client = httpx.AsyncClient(follow_redirects=False, trust_env=False, transport=ValidatedAsyncHTTPTransport())
|
| 141 |
resources = ProxyResources(client)
|
| 142 |
# 共用逐跳公网/DNS/TLS与跨origin凭据隔离;归档重定向仍需验证。
|
|
|
|
| 145 |
f"https://api.github.com/repos/{owner}/{repo}/zipball/{sha}", headers=headers, timeout=300))
|
| 146 |
if response.status_code != 200:
|
| 147 |
return False
|
| 148 |
+
stage = "archive_download"
|
| 149 |
fd, temp_path = tempfile.mkstemp(suffix=".zip", prefix="ranking-precache-")
|
| 150 |
size = 0
|
| 151 |
with os.fdopen(fd, 'wb') as output:
|
|
|
|
| 154 |
if size > ZIP_MAX_SIZE:
|
| 155 |
return False
|
| 156 |
output.write(chunk)
|
| 157 |
+
stage = "archive_close"
|
| 158 |
await resources.close()
|
| 159 |
+
stage = "archive_validation"
|
| 160 |
await run_owned_thread(validate_zip_archive, temp_path, size)
|
| 161 |
+
stage = "hf_upload"
|
| 162 |
uploaded = await run_owned_thread(upload_file, path_or_fileobj=temp_path, path_in_repo=path_in_repo,
|
| 163 |
repo_id=storage_repo, repo_type="dataset", token=hf_token)
|
| 164 |
+
stage = "hf_commit"
|
| 165 |
hf_commit = commit_sha(getattr(uploaded, 'oid', None))
|
| 166 |
+
stage = "source_recheck"
|
| 167 |
latest_item = await run_owned_thread(read_proxy_item, db, item_id)
|
| 168 |
if source_signature(latest_item, select_github_token(latest_item)) != signature:
|
| 169 |
return False
|
|
|
|
| 174 |
if db.DATASET_REPO_ID != storage_repo:
|
| 175 |
return False
|
| 176 |
return publish_archive_metadata(versions, item_id, ref, metadata)
|
| 177 |
+
stage = "metadata_publish"
|
| 178 |
return bool(await run_owned_thread(db.atomic_update, "versions.json", update, default_data={}))
|
| 179 |
except Exception as error:
|
| 180 |
+
logger.warning("预缓存失败: stage=%s | error_type=%s", stage, type(error).__name__)
|
| 181 |
return False
|
| 182 |
finally:
|
| 183 |
try:
|