| {"time":"2025-07-29T19:06:42.599309982+08:00","level":"INFO","msg":"stream: starting","core version":"0.20.1","symlink path":"outputs/train/act_record_chess_pieces/wandb/run-20250729_190642-7pxzahhl/logs/debug-core.log"} | |
| {"time":"2025-07-29T19:06:43.068230122+08:00","level":"INFO","msg":"stream: created new stream","id":"7pxzahhl"} | |
| {"time":"2025-07-29T19:06:43.068247787+08:00","level":"INFO","msg":"stream: started","id":"7pxzahhl"} | |
| {"time":"2025-07-29T19:06:43.068254725+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"7pxzahhl"} | |
| {"time":"2025-07-29T19:06:43.06828547+08:00","level":"INFO","msg":"sender: started","stream_id":"7pxzahhl"} | |
| {"time":"2025-07-29T19:06:43.068308515+08:00","level":"INFO","msg":"handler: started","stream_id":"7pxzahhl"} | |
| {"time":"2025-07-29T19:06:43.59327593+08:00","level":"INFO","msg":"Starting system monitor"} | |
| {"time":"2025-07-30T00:51:44.256881191+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"} | |
| {"time":"2025-07-30T00:52:16.489618377+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"} | |
| {"time":"2025-07-30T00:52:51.039331424+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"} | |
| {"time":"2025-07-30T00:53:29.501630563+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"} | |
| {"time":"2025-07-30T00:54:16.390834964+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"} | |
| {"time":"2025-07-30T00:55:24.014718572+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"} | |
| {"time":"2025-07-30T00:56:54.016194024+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"} | |
| {"time":"2025-07-30T00:58:24.017607837+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"} | |
| {"time":"2025-07-30T00:59:54.020216665+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"} | |
| {"time":"2025-07-30T01:01:14.258233536+08:00","level":"WARN","msg":"sender: taking a long time","seconds":600.000049675,"work":"WorkRecord(*service_go_proto.Request_StopStatus); Control(local:true mailbox_slot:\"8xrgiyx6nyv8\" connection_id:\"127.0.0.1:54224\")"} | |
| {"time":"2025-07-30T01:01:24.021417616+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"} | |
| {"time":"2025-07-30T01:02:54.02228871+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"} | |
| {"time":"2025-07-30T01:04:24.024274762+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"} | |
| {"time":"2025-07-30T01:05:54.025438361+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"} | |
| {"time":"2025-07-30T01:07:05.349485763+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": read tcp 10.101.118.49:36572->35.186.228.49:443: read: connection timed out"} | |
| {"time":"2025-07-30T01:08:05.878463846+08:00","level":"INFO","msg":"sender: succeeded after taking longer than expected","seconds":1011.625598297,"work":"WorkRecord(*service_go_proto.Request_StopStatus); Control(local:true mailbox_slot:\"8xrgiyx6nyv8\" connection_id:\"127.0.0.1:54224\")"} | |
| {"time":"2025-07-30T01:08:06.494372711+08:00","level":"INFO","msg":"api: retrying HTTP error","status":503,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"The service is currently unavailable. Please try again later.\",\"path\":[\"createArtifact\"]}],\"data\":{\"createArtifact\":null}}"} | |
| {"time":"2025-07-30T01:08:06.53524368+08:00","level":"INFO","msg":"api: retrying HTTP error","status":503,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"The service is currently unavailable. Please try again later.\",\"path\":[\"createArtifact\"]}],\"data\":{\"createArtifact\":null}}"} | |
| {"time":"2025-07-30T01:08:06.535269244+08:00","level":"INFO","msg":"api: retrying HTTP error","status":503,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"The service is currently unavailable. Please try again later.\",\"path\":[\"createArtifact\"]}],\"data\":{\"createArtifact\":null}}"} | |
| {"time":"2025-07-30T01:08:08.86812743+08:00","level":"INFO","msg":"api: retrying HTTP error","status":503,"url":"https://api.wandb.ai/graphql","body":"{\"errors\":[{\"message\":\"The service is currently unavailable. Please try again later.\",\"path\":[\"createArtifact\"]}],\"data\":{\"createArtifact\":null}}"} | |
| {"time":"2025-07-30T04:06:35.898340109+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/moizbinjaafar-asia-pacific-university-of-technology-inno/lerobot/7pxzahhl/file_stream\": read tcp 10.101.118.49:60600->35.186.228.49:443: read: connection reset by peer"} | |
| {"time":"2025-07-30T06:41:10.605831038+08:00","level":"INFO","msg":"stream: closing","id":"7pxzahhl"} | |
| {"time":"2025-07-30T06:41:10.606684492+08:00","level":"INFO","msg":"Stopping system monitor"} | |
| {"time":"2025-07-30T06:41:10.60743285+08:00","level":"INFO","msg":"Stopped system monitor"} | |
| {"time":"2025-07-30T06:41:10.610007369+08:00","level":"ERROR","msg":"sender: failed to upload run summary: write outputs/train/act_record_chess_pieces/wandb/run-20250729_190642-7pxzahhl/files/wandb-summary.json: no space left on device"} | |
| {"time":"2025-07-30T06:41:10.61946706+08:00","level":"ERROR","msg":"sender: failed to upload run config: write outputs/train/act_record_chess_pieces/wandb/run-20250729_190642-7pxzahhl/files/config.yaml: no space left on device"} | |
| {"time":"2025-07-30T06:41:11.760413707+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} | |
| {"time":"2025-07-30T06:41:12.100409048+08:00","level":"INFO","msg":"handler: closed","stream_id":"7pxzahhl"} | |
| {"time":"2025-07-30T06:41:12.100461118+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"7pxzahhl"} | |
| {"time":"2025-07-30T06:41:12.100555129+08:00","level":"ERROR","msg":"writer: startStore: error closing store: store: failed closing writer: write outputs/train/act_record_chess_pieces/wandb/run-20250729_190642-7pxzahhl/run-7pxzahhl.wandb: no space left on device"} | |
| {"time":"2025-07-30T06:41:12.101299793+08:00","level":"INFO","msg":"sender: closed","stream_id":"7pxzahhl"} | |
| {"time":"2025-07-30T06:41:12.101757074+08:00","level":"INFO","msg":"stream: closed","id":"7pxzahhl"} | |