Buckets:
| """Poll the V2 logs every 20 s and print new lines worth acting on (stage events, errors, OOM redos, every 250 steps).""" | |
| import glob, os, re, sys, time | |
| D = "/workspace/decision-reasoner/logs" | |
| PAT = re.compile(r'Traceback|Error|OutOfMemory|Killed|oom_retry|"step": \d*(250|500|750|000),|"dev_acc"|PIPELINE_EXIT|DRIVER_EXIT|chosen|"saved"|FAILED|S1MB_DONE|NIGHT_DONE') | |
| pos = {} | |
| def scan(first): | |
| for p in [f"{D}/v2_timeline.jsonl", f"{D}/v2_pipeline_exit.log"] + sorted(glob.glob(f"{D}/v2_*.log") + glob.glob(f"{D}/v2b_*.log") + glob.glob(f"{D}/rl_eval_driver*.log") + glob.glob(f"{D}/s1mb_run.log")): | |
| if not os.path.exists(p) or p.endswith("v2_sync.log") or p.endswith("v2b_sync.log"): | |
| continue | |
| size = os.path.getsize(p) | |
| if first or p not in pos: | |
| pos[p] = size if first else 0 | |
| if first: continue | |
| if size < pos[p]: pos[p] = 0 | |
| if size == pos[p]: continue | |
| with open(p, errors="replace") as f: | |
| f.seek(pos[p]); chunk = f.read(); pos[p] = f.tell() | |
| for l in chunk.splitlines(): | |
| if "UserWarning" in l: continue | |
| if p.endswith("v2_timeline.jsonl") or p.endswith("pipeline_exit.log") or PAT.search(l): | |
| print(os.path.basename(p) + ": " + l[:280], flush=True) | |
| scan(True) | |
| while True: | |
| time.sleep(20); scan(False) | |
Xet Storage Details
- Size:
- 1.35 kB
- Xet hash:
- 8a685227833f048d7e6f3805c93594ff4c787c75319d9c736ba9043aa339bd09
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.