Arshit Malik commited on
Commit
bf0b578
·
1 Parent(s): aeb694b

fix syntax, remove 60s, poll 15s, kernel logs, suppress 404 noise

Browse files
Files changed (2) hide show
  1. automation.py +3 -6
  2. sync_hub.py +1 -1
automation.py CHANGED
@@ -1,6 +1,7 @@
1
  #!/usr/bin/env python3
2
  import os, json, time, subprocess, pickle, sys, shutil, re, random, traceback, logging
3
  from pathlib import Path
 
4
  from datetime import datetime, timezone
5
 
6
  BASE_DIR = Path(__file__).resolve().parent
@@ -169,10 +170,7 @@ def step_content(run: Path):
169
  set_kaggle_creds(acc); break
170
 
171
  log.info(f"Waiting for LLM kernel {info['slug']} ...")
172
- log.info(" Sleeping 60s for kernel to register...")
173
- # time.sleep(60) removed - kernel registers instantly
174
- log.info(" (Dataset mount ~0min + inference ~3-5min)")
175
- wait_kaggle(info["slug"])
176
 
177
  log.info("Downloading content.json ...")
178
  subprocess.run(["kaggle", "kernels", "output", info["slug"], "-p", str(run)],
@@ -415,8 +413,7 @@ def main():
415
  TRIGGER_SKIP_FILE.unlink(missing_ok=True)
416
  NEXT_RUN_AFTER = time.time() + 86400
417
  log.info("Skipped. Next run in 24h.")
418
- # time.sleep(60) removed - kernel registers instantly
419
-
420
  if TRIGGER_RUN_FILE.exists():
421
  TRIGGER_RUN_FILE.unlink(missing_ok=True)
422
  NEXT_RUN_AFTER = 0
 
1
  #!/usr/bin/env python3
2
  import os, json, time, subprocess, pickle, sys, shutil, re, random, traceback, logging
3
  from pathlib import Path
4
+ import tempfile, shutil
5
  from datetime import datetime, timezone
6
 
7
  BASE_DIR = Path(__file__).resolve().parent
 
170
  set_kaggle_creds(acc); break
171
 
172
  log.info(f"Waiting for LLM kernel {info['slug']} ...")
173
+ # wait_kaggle(info["slug"])
 
 
 
174
 
175
  log.info("Downloading content.json ...")
176
  subprocess.run(["kaggle", "kernels", "output", info["slug"], "-p", str(run)],
 
413
  TRIGGER_SKIP_FILE.unlink(missing_ok=True)
414
  NEXT_RUN_AFTER = time.time() + 86400
415
  log.info("Skipped. Next run in 24h.")
416
+ #
 
417
  if TRIGGER_RUN_FILE.exists():
418
  TRIGGER_RUN_FILE.unlink(missing_ok=True)
419
  NEXT_RUN_AFTER = 0
sync_hub.py CHANGED
@@ -13,7 +13,7 @@ def pull_state(base_dir):
13
  try:
14
  hf_hub_download(repo_id=DATASET_REPO, filename=fname,
15
  repo_type="dataset", token=HF_TOKEN,
16
- local_dir=str(base_dir), local_dir_use_symlinks=False)
17
  print(f" pulled {fname}")
18
  except Exception as e:
19
  print(f" skipped {fname}: {e}")
 
13
  try:
14
  hf_hub_download(repo_id=DATASET_REPO, filename=fname,
15
  repo_type="dataset", token=HF_TOKEN,
16
+ local_dir=str(base_dir))
17
  print(f" pulled {fname}")
18
  except Exception as e:
19
  print(f" skipped {fname}: {e}")