Spaces:
Build error
Build error
refactor logic
Browse files- src/gradio_space_ci/webhook.py +34 -77
src/gradio_space_ci/webhook.py
CHANGED
|
@@ -213,7 +213,7 @@ def recover_after_restart(space_id: str) -> None:
|
|
| 213 |
if repo_exists(repo_id=ci_space_id, repo_type="space"):
|
| 214 |
# Found a PR for which the CI space has not been deleted
|
| 215 |
print(f"Recovery. Found a closed PR with an active CI space: {discussion.url}. Deleting it.")
|
| 216 |
-
background_pool.submit(delete_ci_space, space_id=space_id, pr_num=discussion.num)
|
| 217 |
|
| 218 |
|
| 219 |
###
|
|
@@ -258,7 +258,7 @@ async def trigger_ci_on_pr(payload: WebhookPayload, task_queue: BackgroundTasks)
|
|
| 258 |
and payload.discussion.isPullRequest
|
| 259 |
and payload.discussion.status in ACTIVE_PR_STATUS
|
| 260 |
):
|
| 261 |
-
# A comment, is it
|
| 262 |
if payload.event.scope == "discussion.comment":
|
| 263 |
handle_command(space_id=space_id, payload=payload)
|
| 264 |
# Always sync (in case the space was sleeping or building)
|
|
@@ -274,11 +274,7 @@ async def trigger_ci_on_pr(payload: WebhookPayload, task_queue: BackgroundTasks)
|
|
| 274 |
and payload.discussion.isPullRequest
|
| 275 |
and (payload.discussion.status == "merged" or payload.discussion.status == "closed")
|
| 276 |
):
|
| 277 |
-
task_queue.add_task(
|
| 278 |
-
delete_ci_space,
|
| 279 |
-
space_id=space_id,
|
| 280 |
-
pr_num=payload.discussion.num,
|
| 281 |
-
)
|
| 282 |
has_task = True
|
| 283 |
elif (
|
| 284 |
# Means "some content has been pushed to the Space" (any branch)
|
|
@@ -390,7 +386,7 @@ def create_ephemeral_space(space_id: str, pr_num: int) -> bool:
|
|
| 390 |
raise
|
| 391 |
|
| 392 |
|
| 393 |
-
def configure_ephemeral_space(space_id: str, pr_num: int) -> bool:
|
| 394 |
# Config values
|
| 395 |
ci_space_id = _get_ci_space_id(space_id=space_id, pr_num=pr_num)
|
| 396 |
trusted_authors: List[str] = EPHEMERAL_SPACES_CONFIG["trusted_authors"]
|
|
@@ -401,25 +397,25 @@ def configure_ephemeral_space(space_id: str, pr_num: int) -> bool:
|
|
| 401 |
|
| 402 |
# Check if trusted author
|
| 403 |
details = get_discussion_details(repo_id=space_id, repo_type="space", discussion_num=pr_num)
|
| 404 |
-
if details.author
|
| 405 |
-
|
| 406 |
-
|
| 407 |
-
|
| 408 |
-
|
| 409 |
-
|
| 410 |
-
|
| 411 |
-
|
| 412 |
-
|
| 413 |
-
|
| 414 |
-
|
| 415 |
-
|
| 416 |
-
if storage is not None:
|
| 417 |
-
request_space_storage(ci_space_id, storage)
|
| 418 |
|
| 419 |
-
|
|
|
|
|
|
|
| 420 |
|
| 421 |
|
| 422 |
-
def delete_ci_space(space_id: str, pr_num: int) -> None:
|
| 423 |
print(f"New task: delete ephemeral env for {space_id} (PR {pr_num})")
|
| 424 |
|
| 425 |
# Delete
|
|
@@ -431,7 +427,9 @@ def delete_ci_space(space_id: str, pr_num: int) -> None:
|
|
| 431 |
return
|
| 432 |
|
| 433 |
# Notify about deletion
|
| 434 |
-
|
|
|
|
|
|
|
| 435 |
|
| 436 |
|
| 437 |
def notify_pr(
|
|
@@ -464,57 +462,13 @@ def _get_ci_space_id(space_id: str, pr_num: int) -> str:
|
|
| 464 |
return f"{space_id}-ci-pr-{pr_num}"
|
| 465 |
|
| 466 |
|
| 467 |
-
def set_config(space_id: str, pr_num: str) -> None:
|
| 468 |
-
"""a function to set the ephemerial space config"""
|
| 469 |
-
variables: Dict[str, str] = EPHEMERAL_SPACES_CONFIG["variables"]
|
| 470 |
-
secrets: Dict[str, str] = EPHEMERAL_SPACES_CONFIG["secrets"]
|
| 471 |
-
hardware: Optional[SpaceHardware] = EPHEMERAL_SPACES_CONFIG["hardware"]
|
| 472 |
-
storage: Optional[SpaceHardware] = EPHEMERAL_SPACES_CONFIG["storage"]
|
| 473 |
-
ci_space_id = _get_ci_space_id(space_id=space_id, pr_num=pr_num)
|
| 474 |
-
# Configure space
|
| 475 |
-
for key, value in variables.items():
|
| 476 |
-
add_space_variable(ci_space_id, key, value)
|
| 477 |
-
for key, value in secrets.items():
|
| 478 |
-
add_space_secret(ci_space_id, key, value)
|
| 479 |
-
|
| 480 |
-
# Request hardware/storage for space
|
| 481 |
-
if hardware is not None and hardware != SpaceHardware.CPU_BASIC:
|
| 482 |
-
request_space_hardware(ci_space_id, hardware, sleep_time=5 * 60) # sleep after 5min on PR Spaces with GPU
|
| 483 |
-
if storage is not None:
|
| 484 |
-
request_space_storage(ci_space_id, storage)
|
| 485 |
-
|
| 486 |
-
|
| 487 |
def rebuild_space(space_id: str, pr_num: int) -> None:
|
| 488 |
"a function to rebuild the ephemeral space without config"
|
| 489 |
# This is useful to cut down on resource usage and to remove tokens from
|
| 490 |
# the ephemeral space
|
| 491 |
-
|
| 492 |
-
|
| 493 |
-
|
| 494 |
-
except RepositoryNotFoundError:
|
| 495 |
-
pass
|
| 496 |
-
create_ephemeral_space(space_id=space_id, pr_num=pr_num)
|
| 497 |
-
# Download space codebase from PR revision
|
| 498 |
-
snapshot_path = Path(snapshot_download(repo_id=space_id, revision=f"refs/pr/{pr_num}", repo_type="space"))
|
| 499 |
-
|
| 500 |
-
# Overwrite README file in cache (/!\)
|
| 501 |
-
readme_path = snapshot_path / "README.md"
|
| 502 |
-
card = RepoCard.load(readme_path)
|
| 503 |
-
setattr(card.data, "synced_sha", snapshot_path.name) # latest sha
|
| 504 |
-
card.data.title = f"{card.data.title} (ephemeral #{pr_num})"
|
| 505 |
-
card.save(readme_path)
|
| 506 |
-
|
| 507 |
-
# Sync space codebase with PR revision
|
| 508 |
-
upload_folder(
|
| 509 |
-
repo_id=ci_space_id,
|
| 510 |
-
repo_type="space",
|
| 511 |
-
commit_message=f"Sync CI Space with PR {pr_num}.",
|
| 512 |
-
folder_path=snapshot_path,
|
| 513 |
-
delete_patterns="*",
|
| 514 |
-
)
|
| 515 |
-
|
| 516 |
-
# Delete readme file from cache (just in case)
|
| 517 |
-
readme_path.unlink(missing_ok=True)
|
| 518 |
|
| 519 |
|
| 520 |
def handle_modification(space_id: str, discussion: Any) -> None:
|
|
@@ -522,10 +476,13 @@ def handle_modification(space_id: str, discussion: Any) -> None:
|
|
| 522 |
if not repo_exists(ci_space_id):
|
| 523 |
return
|
| 524 |
details = get_discussion_details(repo_id=space_id, repo_type="space", discussion_num=discussion.num)
|
| 525 |
-
|
| 526 |
-
|
| 527 |
-
|
| 528 |
-
|
|
|
|
|
|
|
|
|
|
| 529 |
|
| 530 |
|
| 531 |
def handle_command(space_id: str, payload: WebhookPayload) -> None:
|
|
@@ -535,7 +492,7 @@ def handle_command(space_id: str, payload: WebhookPayload) -> None:
|
|
| 535 |
event_author = details.events[-1]._event["author"]["name"] # username of that event
|
| 536 |
if event_author in EPHEMERAL_SPACES_CONFIG["trusted_authors"]:
|
| 537 |
if payload.comment.content == "/trust_pr":
|
| 538 |
-
|
| 539 |
notify_pr(space_id=space_id, pr_num=pr_num, action="trusted_pr")
|
| 540 |
elif payload.comment.content == "/untrust_pr":
|
| 541 |
rebuild_space(space_id=space_id, pr_num=pr_num)
|
|
|
|
| 213 |
if repo_exists(repo_id=ci_space_id, repo_type="space"):
|
| 214 |
# Found a PR for which the CI space has not been deleted
|
| 215 |
print(f"Recovery. Found a closed PR with an active CI space: {discussion.url}. Deleting it.")
|
| 216 |
+
background_pool.submit(delete_ci_space, space_id=space_id, pr_num=discussion.num, notify=True)
|
| 217 |
|
| 218 |
|
| 219 |
###
|
|
|
|
| 258 |
and payload.discussion.isPullRequest
|
| 259 |
and payload.discussion.status in ACTIVE_PR_STATUS
|
| 260 |
):
|
| 261 |
+
# A comment, is it a command ?
|
| 262 |
if payload.event.scope == "discussion.comment":
|
| 263 |
handle_command(space_id=space_id, payload=payload)
|
| 264 |
# Always sync (in case the space was sleeping or building)
|
|
|
|
| 274 |
and payload.discussion.isPullRequest
|
| 275 |
and (payload.discussion.status == "merged" or payload.discussion.status == "closed")
|
| 276 |
):
|
| 277 |
+
task_queue.add_task(delete_ci_space, space_id=space_id, pr_num=payload.discussion.num, notify=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 278 |
has_task = True
|
| 279 |
elif (
|
| 280 |
# Means "some content has been pushed to the Space" (any branch)
|
|
|
|
| 386 |
raise
|
| 387 |
|
| 388 |
|
| 389 |
+
def configure_ephemeral_space(space_id: str, pr_num: int, trusted_pr=False) -> bool:
|
| 390 |
# Config values
|
| 391 |
ci_space_id = _get_ci_space_id(space_id=space_id, pr_num=pr_num)
|
| 392 |
trusted_authors: List[str] = EPHEMERAL_SPACES_CONFIG["trusted_authors"]
|
|
|
|
| 397 |
|
| 398 |
# Check if trusted author
|
| 399 |
details = get_discussion_details(repo_id=space_id, repo_type="space", discussion_num=pr_num)
|
| 400 |
+
if details.author in trusted_authors or trusted_pr is True:
|
| 401 |
+
# Configure space
|
| 402 |
+
for key, value in variables.items():
|
| 403 |
+
add_space_variable(ci_space_id, key, value)
|
| 404 |
+
for key, value in secrets.items():
|
| 405 |
+
add_space_secret(ci_space_id, key, value)
|
| 406 |
+
|
| 407 |
+
# Request hardware/storage for space
|
| 408 |
+
if hardware is not None and hardware != SpaceHardware.CPU_BASIC:
|
| 409 |
+
request_space_hardware(ci_space_id, hardware, sleep_time=5 * 60) # sleep after 5min on PR Spaces with GPU
|
| 410 |
+
if storage is not None:
|
| 411 |
+
request_space_storage(ci_space_id, storage)
|
|
|
|
|
|
|
| 412 |
|
| 413 |
+
return True
|
| 414 |
+
else:
|
| 415 |
+
return False
|
| 416 |
|
| 417 |
|
| 418 |
+
def delete_ci_space(space_id: str, pr_num: int, notify: bool = True) -> None:
|
| 419 |
print(f"New task: delete ephemeral env for {space_id} (PR {pr_num})")
|
| 420 |
|
| 421 |
# Delete
|
|
|
|
| 427 |
return
|
| 428 |
|
| 429 |
# Notify about deletion
|
| 430 |
+
if notify is True:
|
| 431 |
+
# This logic is adaped across multiple functions so we will not always notify the pr
|
| 432 |
+
notify_pr(space_id=space_id, pr_num=pr_num, action="deleted")
|
| 433 |
|
| 434 |
|
| 435 |
def notify_pr(
|
|
|
|
| 462 |
return f"{space_id}-ci-pr-{pr_num}"
|
| 463 |
|
| 464 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 465 |
def rebuild_space(space_id: str, pr_num: int) -> None:
|
| 466 |
"a function to rebuild the ephemeral space without config"
|
| 467 |
# This is useful to cut down on resource usage and to remove tokens from
|
| 468 |
# the ephemeral space
|
| 469 |
+
delete_ci_space(space_id=space_id, pr_num=pr_num, notify=False)
|
| 470 |
+
# create a new synced ephemeral space
|
| 471 |
+
sync_ci_space(space_id=space_id, pr_num=pr_num)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 472 |
|
| 473 |
|
| 474 |
def handle_modification(space_id: str, discussion: Any) -> None:
|
|
|
|
| 476 |
if not repo_exists(ci_space_id):
|
| 477 |
return
|
| 478 |
details = get_discussion_details(repo_id=space_id, repo_type="space", discussion_num=discussion.num)
|
| 479 |
+
# If last commit is not by a trusted author we rebuild it
|
| 480 |
+
for event in details.events[::-1]:
|
| 481 |
+
if event.type == "commit":
|
| 482 |
+
if event._event["author"]["name"] not in EPHEMERAL_SPACES_CONFIG["trusted_authors"]:
|
| 483 |
+
rebuild_space(space_id=space_id, pr_num=discussion.num)
|
| 484 |
+
else:
|
| 485 |
+
return
|
| 486 |
|
| 487 |
|
| 488 |
def handle_command(space_id: str, payload: WebhookPayload) -> None:
|
|
|
|
| 492 |
event_author = details.events[-1]._event["author"]["name"] # username of that event
|
| 493 |
if event_author in EPHEMERAL_SPACES_CONFIG["trusted_authors"]:
|
| 494 |
if payload.comment.content == "/trust_pr":
|
| 495 |
+
configure_ephemeral_space(space_id=space_id, pr_num=pr_num, trusted_pr=True)
|
| 496 |
notify_pr(space_id=space_id, pr_num=pr_num, action="trusted_pr")
|
| 497 |
elif payload.comment.content == "/untrust_pr":
|
| 498 |
rebuild_space(space_id=space_id, pr_num=pr_num)
|