safe_v2_2nodes/code/wandb/debug-internal.log

{"time":"2026-05-05T06:45:20.953820962Z","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/fsx/craffel/lingua_logs/safe_v2_2nodes/code/wandb/run-20260505_064520-udrqjcql/logs/debug-core.log"}
{"time":"2026-05-05T06:45:21.298763895Z","level":"INFO","msg":"created new stream","id":"udrqjcql"}
{"time":"2026-05-05T06:45:21.298807136Z","level":"INFO","msg":"stream: started","id":"udrqjcql"}
{"time":"2026-05-05T06:45:21.298822847Z","level":"INFO","msg":"writer: Do: started","stream_id":"udrqjcql"}
{"time":"2026-05-05T06:45:21.298836597Z","level":"INFO","msg":"sender: started","stream_id":"udrqjcql"}
{"time":"2026-05-05T06:45:21.298859708Z","level":"INFO","msg":"handler: started","stream_id":"udrqjcql"}
{"time":"2026-05-05T06:45:22.152118471Z","level":"INFO","msg":"Starting system monitor"}
{"time":"2026-05-05T10:46:51.357295241Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2026-05-05T11:59:06.670586455Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2026-05-05T11:59:39.096364471Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
{"time":"2026-05-05T12:02:21.693691681Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2026-05-05T12:25:51.795102952Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": read tcp 26.0.160.192:50294->34.8.250.101:443: read: connection reset by peer"}
{"time":"2026-05-05T18:18:30.849687225Z","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"context deadline exceeded\",\"path\":[\"project\",\"run\"]}],\"data\":{\"project\":{\"run\":null}}}"}
{"time":"2026-05-05T20:22:24.229809742Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2026-05-05T20:23:24.236799124Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2026-05-05T20:24:24.238355492Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
{"time":"2026-05-07T13:36:48.691484652Z","level":"INFO","msg":"stream: closing","id":"udrqjcql"}
{"time":"2026-05-07T13:36:48.691543303Z","level":"INFO","msg":"Stopping system monitor"}
{"time":"2026-05-07T13:36:48.692662245Z","level":"INFO","msg":"Stopped system monitor"}
{"time":"2026-05-07T13:36:49.157164022Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
{"time":"2026-05-07T13:36:49.351261466Z","level":"INFO","msg":"handler: closed","stream_id":"udrqjcql"}
{"time":"2026-05-07T13:36:49.351274067Z","level":"INFO","msg":"writer: Close: closed","stream_id":"udrqjcql"}
{"time":"2026-05-07T13:36:49.351279947Z","level":"INFO","msg":"sender: closed","stream_id":"udrqjcql"}
{"time":"2026-05-07T13:36:49.351327418Z","level":"INFO","msg":"stream: closed","id":"udrqjcql"}