File size: 4,288 Bytes
03388e8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 | # Phase 6 launcher with a heartbeat. Kaggle kernel body: 2xT4, billed 1x wall clock.
#
# Why this exists: run_benchmarks.py is silent until its first push (the smoke pass writes smoke.json only
# after all eight smoke runs), and on this account `kernels.get` is denied, so a kernel that is queued,
# booting, installing or dead all look identical from outside. A heartbeat in the report repo makes those
# states distinguishable and costs a few kilobytes per push.
#
# Before launching: book the QUOTA.md row, then set REV/PUB/SHA pins to the files actually published at that
# revision, and read the pins back after fetch -- a kernel that runs code it did not verify is not reproducible.
import hashlib, os, runpy, subprocess, sys, threading, time
REV = "55f23ef5d0e8e6a60adad23bf8a2ffffc6bc7580"
RB_SHA = "8196d853f3e3d8f1c729ff0d76517ddc7814b9d460a09497e1f289b03ca8da69"
CRED_SHA = "6525f62f03f2d73650a1eb4f70fcb52d1194caad4ca88b2d8bd8fd54f88339b6"
BASE = "https://huggingface.co/Cion-lab/ounce100m-code/resolve/" + REV
REPORT_REPO = "Cion-lab/ounce100m-p5-report" # private scratch; keeps the deliverable repo clean
LOG = "/kaggle/working/p6.log"
STAGE = {"name": "submit", "detail": ""}
def log(msg):
line = "%s %s\n" % (time.strftime("%H:%M:%S", time.gmtime()), msg)
with open(LOG, "a") as fh:
fh.write(line)
sys.__stdout__.write(line)
sys.__stdout__.flush()
def push_heartbeat(api, tok):
try:
body = open(LOG, "rb").read()[-60000:]
api.upload_file(path_or_fileobj=body, path_in_repo="p6_heartbeat.log",
repo_id=REPORT_REPO, repo_type="dataset",
commit_message="heartbeat %s" % STAGE["name"], token=tok)
except Exception as e: # a lost beat must never kill the eval
log("heartbeat push failed %r" % (e,))
def beat(api, tok):
while True:
time.sleep(120)
push_heartbeat(api, tok)
os.chdir("/kaggle/working")
os.makedirs("/kaggle/working/lib/eval", exist_ok=True)
for rel, want in (("eval/run_benchmarks.py", RB_SHA), ("ounce100m_credentials.py", CRED_SHA)):
dest = "/kaggle/working/lib/" + rel
assert subprocess.run(["curl", "-sfL", BASE + "/" + rel, "-o", dest]).returncode == 0, rel
got = hashlib.sha256(open(dest, "rb").read()).hexdigest()
assert got == want, "%s: %s != %s" % (rel, got[:16], want[:16])
log("PINNED %s %s" % (rel, got[:16]))
sys.path.insert(0, "/kaggle/working/lib")
import ounce100m_credentials
STAGE["name"] = "creds"
log("creds " + str(ounce100m_credentials.install(verify=True)))
tok = os.environ["HF_TOKEN"]
from huggingface_hub import HfApi
api = HfApi(token=tok)
threading.Thread(target=beat, args=(api, tok), daemon=True).start()
push_heartbeat(api, tok) # first beat immediately: the kernel is alive
import site
up = site.getusersitepackages() # E-047: ~/.local is not on sys.path when created
if up and up not in sys.path:
sys.path.append(up) # mid-process, so add it explicitly
log("user_site " + str(up))
# tee: the runner streams its own progress, and that stream is the only status a denied-kernels.get account
# can still see once it is being shipped somewhere.
real = sys.stdout
class Tee:
def write(self, s):
real.write(s)
real.flush()
with open(LOG, "a") as fh:
fh.write(s)
return len(s)
def flush(self):
real.flush()
def isatty(self): # transformers' progress bars ask
return False
STAGE["name"] = "run_benchmarks"
sys.argv = ["run_benchmarks.py", "--model", "Cion-lab/ounce100m-v1",
"--repo", "Cion-lab/ounce100m-v1", "--full-gpu-hours", "0.3"]
log("ARGV " + " ".join(sys.argv))
sys.stdout = Tee()
err = None
try:
runpy.run_path("/kaggle/working/lib/eval/run_benchmarks.py", run_name="__main__")
except SystemExit as e:
err = None if e.code in (0, None) else "SystemExit(%r)" % (e.code,)
except BaseException as e:
err = repr(e)
finally:
sys.stdout = real
STAGE["name"] = "done"
log("P6_DONE err=%s" % (err,))
push_heartbeat(api, tok)
print("VERDICT P6 err=%s" % (bool(err),), flush=True)
|