Buckets:
| """Poll logs/rl_grpo.log every 30 s: print a compact line every 5th logged step with reward, plus errors and exit.""" | |
| import ast, os, re, time | |
| import sys | |
| P = sys.argv[1] if len(sys.argv) > 1 else "/workspace/decision-reasoner/logs/rl_grpo.log" | |
| pos, n_steps, first = 0, 0, True | |
| ERR = re.compile(r"Traceback|OutOfMemory|CUDA error|Killed|RL_EXIT|Error:") | |
| while True: | |
| if os.path.exists(P): | |
| pass | |
| if os.path.exists(P): | |
| with open(P, errors="replace") as f: | |
| if os.path.getsize(P) < pos: pos = 0 # log replaced by a restart | |
| f.seek(pos); chunk = f.read(); pos = f.tell() | |
| for l in chunk.replace("\r", "\n").splitlines(): | |
| if ERR.search(l) and "Warning" not in l and not first: | |
| print(l[:300], flush=True) | |
| for m in re.finditer(r"\{'loss'[^{}]*\}", l): | |
| try: d = ast.literal_eval(m.group(0)) | |
| except Exception: continue | |
| if "reward" not in d: continue | |
| n_steps += 1 | |
| if not first: | |
| keys = ["reward", "reward_std", "completions/mean_length", "completions/clipped_ratio", "frac_reward_zero_std", "grad_norm", "entropy", "step_time"] | |
| print(f"round {n_steps} (step ~{4 * n_steps - 3}): " + " ".join(f"{k.split('/')[-1]}={d[k]}" for k in keys if k in d), flush=True) | |
| first = False | |
| time.sleep(30) | |
Xet Storage Details
- Size:
- 1.4 kB
- Xet hash:
- 2c686e90fcae32cafc29b7867dc4bda891422e9dae880519b5c26305f7bdf3fc
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.