minhluanquach's picture
download
raw
3.05 kB
"""
Training completion notification utility.
Sends a notification when training finishes so you can stop the pod
and minimize cloud costs. Supports multiple notification methods.
Usage:
# Set webhook URL as environment variable before training:
export NOTIFY_WEBHOOK="https://discord.com/api/webhooks/YOUR/WEBHOOK"
# or
export NOTIFY_WEBHOOK="https://hooks.slack.com/services/YOUR/WEBHOOK"
# The training scripts call this automatically on completion.
"""
import json
import os
import subprocess
from urllib import request, error
def send_notification(
title: str,
message: str,
webhook_url: str | None = None,
):
"""Send a training completion notification.
Tries in order:
1. Discord/Slack webhook (if NOTIFY_WEBHOOK env var or webhook_url is set)
2. System beep (terminal bell — works over SSH)
Args:
title: Short subject line (e.g., "SyncNet Training Complete")
message: Details (e.g., "Epoch 50, val_loss=0.1234, 4.2 hours")
webhook_url: Override for NOTIFY_WEBHOOK env var
"""
url = webhook_url or os.environ.get("NOTIFY_WEBHOOK", "https://discordapp.com/api/webhooks/1480062943403180062/i_hdOQhtXDeSihk72j8rs_XGnctJn6UKNaX5Vr6Dnu5Cwh4mSpWqvnJDzLNq-Huzg7XH")
# Always print to terminal
print(f"\n{'=' * 50}")
print(f" {title}")
print(f" {message}")
print(f"{'=' * 50}\n")
if url:
_send_webhook(url, title, message)
else:
print("Tip: Set NOTIFY_WEBHOOK env var to get Discord/Slack notifications.")
print(' export NOTIFY_WEBHOOK="https://discord.com/api/webhooks/TrainingPipeline"')
# Terminal bell — audible if SSH'd in
print("\a", end="", flush=True)
def _send_webhook(url: str, title: str, message: str):
"""Send notification via Discord or Slack webhook."""
try:
if "discord.com" in url:
payload = json.dumps({
"content": f"**{title}**\n{message}"
}).encode()
else:
# Slack-compatible format
payload = json.dumps({
"text": f"*{title}*\n{message}"
}).encode()
req = request.Request(
url,
data=payload,
headers={"Content-Type": "application/json"},
method="POST",
)
request.urlopen(req, timeout=10)
print(" Notification sent successfully.")
except Exception as e:
print(f" Warning: Failed to send notification: {e}")
def send_training_complete(
model_name: str,
epoch: int,
best_val_loss: float,
total_hours: float,
extra: str = "",
):
"""Convenience wrapper for training completion."""
title = f"{model_name} Training Complete"
message = (
f"Epoch: {epoch} | Best val loss: {best_val_loss:.4f} | "
f"Time: {total_hours:.2f}h"
)
if extra:
message += f" | {extra}"
send_notification(title, message)

Xet Storage Details

Size:
3.05 kB
·
Xet hash:
3406c3d0565bbdfa3b069bf3201910483f55f4d703dd02da0e4dd997e78d84b3

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.