diff --git a/wandb/run-20240804_021032-cd2cg2ui/files/output.log b/wandb/run-20240804_021032-cd2cg2ui/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..8d2961d37c554125b707d2f39bf88948c8412160 --- /dev/null +++ b/wandb/run-20240804_021032-cd2cg2ui/files/output.log @@ -0,0 +1,11 @@ +Created Hugging Face repository with ID koichi12/tiny-mistral-sample. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +Traceback (most recent call last): + File "/project/examples/finetuning.py", line 13, in + main() + File "/project/src/llama_recipes/finetuning.py", line 103, in main + model = get_model( + File "/project/src/llama_recipes/get_models.py", line 71, in get_model + assert sliding_window == 4096 +AssertionError \ No newline at end of file diff --git a/wandb/run-20240804_021032-cd2cg2ui/files/wandb-metadata.json b/wandb/run-20240804_021032-cd2cg2ui/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..462a9fef676e88ca76d765517aeb8b56a2cf7336 --- /dev/null +++ b/wandb/run-20240804_021032-cd2cg2ui/files/wandb-metadata.json @@ -0,0 +1,215 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-03T17:10:33.458421", + "startedAt": "2024-08-03T17:10:32.395506", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "1024", + "--sliding-window-size", + "8192", + "--micro-batch-size", + "8", + "--global-batch-size", + "320", + "--train-iters", + "20000", + "--tokenizer-type", + "Llama2Tokenizer", + "--tokenizer-model", + "/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3", + "--train-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--valid-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--test-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--lr", + "2e-5", + "--min-lr", + "1e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "20000", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "adam", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-6", + "--save-interval", + "200", + "--eval-interval", + "200", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/custom/tiny-mistral", + "--save", + "/work/llm_recipes/models/tiny-mistral-sample", + "--load", + "/work/llm_recipes/models/tiny-mistral-sample", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "FULL_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/tiny-mistral-sample", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "llm_tutorial", + "--wandb-name", + "tiny-mistral-sample_train_2024-08-04-02:10:14" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "3be5353210a678dc7008f237fa16b99f2bdf36ea" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.48782730102539 + } +} diff --git a/wandb/run-20240804_021032-cd2cg2ui/files/wandb-summary.json b/wandb/run-20240804_021032-cd2cg2ui/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..fe9e22cf416d08fa3b43b9f0c278f2270e8d86cd --- /dev/null +++ b/wandb/run-20240804_021032-cd2cg2ui/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 0}} \ No newline at end of file diff --git a/wandb/run-20240804_021444-pk5j08lr/files/config.yaml b/wandb/run-20240804_021444-pk5j08lr/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f0ee7a3e2af403897f802a970b0ca6f29515c5e --- /dev/null +++ b/wandb/run-20240804_021444-pk5j08lr/files/config.yaml @@ -0,0 +1,335 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: FULL_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +valid_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +test_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 1024 +num_workers: + desc: null + value: 2 +tokenizer_type: + desc: null + value: Llama2Tokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3 +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: tiny-mistral-sample_train_2024-08-04-02:14:34 +wandb_project: + desc: null + value: llm_tutorial +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/tiny-mistral-sample +save: + desc: null + value: /work/llm_recipes/models/tiny-mistral-sample +base_model: + desc: null + value: /share/pretrained_lm/custom/tiny-mistral +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 200 +save_interval: + desc: null + value: 200 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: adam +lr: + desc: null + value: 2.0e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 20000 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 1.0e-06 +train_iters: + desc: null + value: 20000 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 320 +micro_batch_size: + desc: null + value: 8 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 4096 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-06 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/tiny-mistral-sample +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: false +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 32768 +gradient_accumulation_steps: + desc: null + value: 40 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1722705284.714592 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +activation_function: + desc: null + value: silu +hidden_size: + desc: null + value: 256 +model_type: + desc: null + value: mistral +max_position_embeddings: + desc: null + value: 1024 +num_attention_heads: + desc: null + value: 4 +num_hidden_layers: + desc: null + value: 4 +model_architecture: + desc: null + value: MistralForCausalLM diff --git a/wandb/run-20240804_021444-pk5j08lr/files/output.log b/wandb/run-20240804_021444-pk5j08lr/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..beafad7691dca3a2ed4e03b3fbcf59afbb5157db --- /dev/null +++ b/wandb/run-20240804_021444-pk5j08lr/files/output.log @@ -0,0 +1,103 @@ +Created Hugging Face repository with ID koichi12/tiny-mistral-sample. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +Loading model state dict from /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/model.pt +Loaded model state dict from /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/model.pt +--> Model /share/pretrained_lm/custom/tiny-mistral +--> /share/pretrained_lm/custom/tiny-mistral has 19.925248 Million params +BFloat16 enabled for mixed precision - using bfSixteen policy +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 6400000 + validation: 323200 + test: 3200 +> building train, validation, and test datasets for GPT ... +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1. + warnings.warn( +Let split = None +> finished creating GPT datasets ... +Loading optimizer state dict from /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/optimizer.pt +Loaded optimizer state dict from /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/optimizer.pt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): MistralForCausalLM( + (model): MistralModel( + (embed_tokens): Embedding(32768, 256) + (layers): ModuleList( + (0-3): 4 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): MistralDecoderLayer( + (self_attn): MistralFlashAttention2( + (q_proj): Linear(in_features=256, out_features=512, bias=False) + (k_proj): Linear(in_features=256, out_features=256, bias=False) + (v_proj): Linear(in_features=256, out_features=256, bias=False) + (o_proj): Linear(in_features=512, out_features=256, bias=False) + (rotary_emb): MistralRotaryEmbedding() + ) + (mlp): MistralMLP( + (gate_proj): Linear(in_features=256, out_features=512, bias=False) + (up_proj): Linear(in_features=256, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=256, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): MistralRMSNorm() + (post_attention_layernorm): MistralRMSNorm() + ) + ) + ) + ) + (norm): MistralRMSNorm() + ) + (lm_head): Linear(in_features=256, out_features=32768, bias=False) + ) +) +model config: MistralConfig { + "_name_or_path": "/share/pretrained_lm/custom/tiny-mistral", + "architectures": [ + "MistralForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 1, + "eos_token_id": 2, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 256, + "initializer_range": 0.02, + "intermediate_size": 512, + "label_smoothing": 0.0, + "max_position_embeddings": 1024, + "model_type": "mistral", + "num_attention_heads": 4, + "num_hidden_layers": 4, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-05, + "rope_theta": 1000000.0, + "sliding_window": 4096, + "tie_word_embeddings": false, + "torch_dtype": "float32", + "transformers_version": "4.43.3", + "use_cache": false, + "vocab_size": 32768 +} +Saving checkpoint to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000 +Saving model state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/model.pt +Saved model state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/rng.pt +Saved RNG states to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/rng.pt +Saved checkpoint to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000, took 0.17s +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +[rank0]:[2024-08-04 02:14:50,842] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _flatten_optim_state_dict() profiling: defaultdict(, {}) +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +[rank0]:[2024-08-04 02:14:50,959] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0010300300018570852, 'preprocessing_with_comm': 0.0005270100009511225, 'state_converting': 0.021121047997439746, : 0.022993901999143418}) \ No newline at end of file diff --git a/wandb/run-20240804_021444-pk5j08lr/files/requirements.txt b/wandb/run-20240804_021444-pk5j08lr/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7c90fd1c54d0a9881f6b4c6465b2a4fa88c9056c --- /dev/null +++ b/wandb/run-20240804_021444-pk5j08lr/files/requirements.txt @@ -0,0 +1,271 @@ +absl-py==2.1.0 +accelerate==0.33.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bleach==6.1.0 +blis==0.7.11 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +distributed==2023.11.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +einops==0.7.0 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +grpcio==1.60.1 +huggingface-hub==0.24.5 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +jedi==0.19.1 +jinja2==3.1.3 +joblib==1.3.2 +json5==0.9.14 +jsonnet==0.19.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +kiwisolver==1.4.5 +langcodes==3.3.0 +lazy-loader==0.3 +librosa==0.10.1 +llvmlite==0.40.1 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +murmurhash==1.0.10 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +onnx==1.15.0rc2 +opencv==4.7.0 +optree==0.10.0 +packaging==23.2 +pandas==1.5.3 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +peft==0.11.1 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==14.0.1.dev0+gba5374836.d20240125 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pynvml==11.4.1 +pyparsing==3.1.1 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests==2.31.0 +rich==13.7.0 +rmm==23.12.0 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.0 +safetensors==0.4.3 +scikit-learn==1.2.0 +scipy==1.12.0 +send2trash==1.8.2 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +srsly==2.4.8 +stack-data==0.6.3 +sympy==1.12 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm==4.66.1 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.9.0 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +urllib3==1.26.18 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +xdoctest==1.0.2 +xgboost==1.7.6 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 \ No newline at end of file diff --git a/wandb/run-20240804_021444-pk5j08lr/files/wandb-metadata.json b/wandb/run-20240804_021444-pk5j08lr/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..79b349410a3fb29404d7558fa646f0dbf8d91a92 --- /dev/null +++ b/wandb/run-20240804_021444-pk5j08lr/files/wandb-metadata.json @@ -0,0 +1,215 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-03T17:14:45.302596", + "startedAt": "2024-08-03T17:14:44.702200", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "1024", + "--sliding-window-size", + "4096", + "--micro-batch-size", + "8", + "--global-batch-size", + "320", + "--train-iters", + "20000", + "--tokenizer-type", + "Llama2Tokenizer", + "--tokenizer-model", + "/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3", + "--train-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--valid-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--test-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--lr", + "2e-5", + "--min-lr", + "1e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "20000", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "adam", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-6", + "--save-interval", + "200", + "--eval-interval", + "200", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/custom/tiny-mistral", + "--save", + "/work/llm_recipes/models/tiny-mistral-sample", + "--load", + "/work/llm_recipes/models/tiny-mistral-sample", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "FULL_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/tiny-mistral-sample", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "llm_tutorial", + "--wandb-name", + "tiny-mistral-sample_train_2024-08-04-02:14:34" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "3be5353210a678dc7008f237fa16b99f2bdf36ea" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.48782730102539 + } +} diff --git a/wandb/run-20240804_021444-pk5j08lr/files/wandb-summary.json b/wandb/run-20240804_021444-pk5j08lr/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e682bae6b5eaeba8295fd0fffdc51474a259249e --- /dev/null +++ b/wandb/run-20240804_021444-pk5j08lr/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 5}} \ No newline at end of file diff --git a/wandb/run-20240804_021444-pk5j08lr/logs/debug-internal.log b/wandb/run-20240804_021444-pk5j08lr/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..74a3d2efafda2fc982e1d71183575617ba1ff0be --- /dev/null +++ b/wandb/run-20240804_021444-pk5j08lr/logs/debug-internal.log @@ -0,0 +1,191 @@ +2024-08-04 02:14:44,716 INFO StreamThr :11553 [internal.py:wandb_internal():86] W&B internal server running at pid: 11553, started at: 2024-08-04 02:14:44.715209 +2024-08-04 02:14:44,717 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: status +2024-08-04 02:14:44,719 INFO WriterThread:11553 [datastore.py:open_for_write():87] open: /project/wandb/run-20240804_021444-pk5j08lr/run-pk5j08lr.wandb +2024-08-04 02:14:44,720 DEBUG SenderThread:11553 [sender.py:send():382] send: header +2024-08-04 02:14:44,733 DEBUG SenderThread:11553 [sender.py:send():382] send: run +2024-08-04 02:14:45,190 INFO SenderThread:11553 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240804_021444-pk5j08lr/files +2024-08-04 02:14:45,190 INFO SenderThread:11553 [sender.py:_start_run_threads():1136] run started: pk5j08lr with start time 1722705284.714592 +2024-08-04 02:14:45,195 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: check_version +2024-08-04 02:14:45,195 DEBUG SenderThread:11553 [sender.py:send_request():409] send_request: check_version +2024-08-04 02:14:45,280 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: run_start +2024-08-04 02:14:45,286 DEBUG HandlerThread:11553 [system_info.py:__init__():27] System info init +2024-08-04 02:14:45,286 DEBUG HandlerThread:11553 [system_info.py:__init__():42] System info init done +2024-08-04 02:14:45,286 INFO HandlerThread:11553 [system_monitor.py:start():194] Starting system monitor +2024-08-04 02:14:45,286 INFO SystemMonitor:11553 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-04 02:14:45,287 INFO HandlerThread:11553 [system_monitor.py:probe():214] Collecting system info +2024-08-04 02:14:45,287 INFO SystemMonitor:11553 [interfaces.py:start():190] Started cpu monitoring +2024-08-04 02:14:45,288 INFO SystemMonitor:11553 [interfaces.py:start():190] Started disk monitoring +2024-08-04 02:14:45,289 INFO SystemMonitor:11553 [interfaces.py:start():190] Started gpu monitoring +2024-08-04 02:14:45,289 INFO SystemMonitor:11553 [interfaces.py:start():190] Started memory monitoring +2024-08-04 02:14:45,290 INFO SystemMonitor:11553 [interfaces.py:start():190] Started network monitoring +2024-08-04 02:14:45,302 DEBUG HandlerThread:11553 [system_info.py:probe():151] Probing system +2024-08-04 02:14:45,304 DEBUG HandlerThread:11553 [system_info.py:_probe_git():136] Probing git +2024-08-04 02:14:45,315 DEBUG HandlerThread:11553 [system_info.py:_probe_git():144] Probing git done +2024-08-04 02:14:45,315 DEBUG HandlerThread:11553 [system_info.py:probe():199] Probing system done +2024-08-04 02:14:45,315 DEBUG HandlerThread:11553 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-03T17:14:45.302596', 'startedAt': '2024-08-03T17:14:44.702200', 'docker': None, 'cuda': None, 'args': ('--seq-length', '1024', '--sliding-window-size', '4096', '--micro-batch-size', '8', '--global-batch-size', '320', '--train-iters', '20000', '--tokenizer-type', 'Llama2Tokenizer', '--tokenizer-model', '/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3', '--train-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--valid-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--test-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '20000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'adam', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '200', '--eval-interval', '200', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/custom/tiny-mistral', '--save', '/work/llm_recipes/models/tiny-mistral-sample', '--load', '/work/llm_recipes/models/tiny-mistral-sample', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/tiny-mistral-sample', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial', '--wandb-name', 'tiny-mistral-sample_train_2024-08-04-02:14:34'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '3be5353210a678dc7008f237fa16b99f2bdf36ea'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.034, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.48782730102539}} +2024-08-04 02:14:45,315 INFO HandlerThread:11553 [system_monitor.py:probe():224] Finished collecting system info +2024-08-04 02:14:45,316 INFO HandlerThread:11553 [system_monitor.py:probe():227] Publishing system info +2024-08-04 02:14:45,317 INFO HandlerThread:11553 [system_monitor.py:probe():229] Finished publishing system info +2024-08-04 02:14:45,323 DEBUG SenderThread:11553 [sender.py:send():382] send: files +2024-08-04 02:14:45,323 INFO SenderThread:11553 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-04 02:14:45,332 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: python_packages +2024-08-04 02:14:45,332 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 02:14:45,332 DEBUG SenderThread:11553 [sender.py:send_request():409] send_request: python_packages +2024-08-04 02:14:45,333 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 02:14:45,334 DEBUG SenderThread:11553 [sender.py:send_request():409] send_request: stop_status +2024-08-04 02:14:45,580 DEBUG SenderThread:11553 [sender.py:send():382] send: telemetry +2024-08-04 02:14:46,067 INFO wandb-upload_0:11553 [upload_job.py:push():131] Uploaded file /tmp/tmp8oqwu4dewandb/gzg3ga4a-wandb-metadata.json +2024-08-04 02:14:46,191 INFO Thread-12 :11553 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_021444-pk5j08lr/files/wandb-metadata.json +2024-08-04 02:14:46,192 INFO Thread-12 :11553 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_021444-pk5j08lr/files/output.log +2024-08-04 02:14:46,192 INFO Thread-12 :11553 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_021444-pk5j08lr/files/requirements.txt +2024-08-04 02:14:48,192 INFO Thread-12 :11553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_021444-pk5j08lr/files/output.log +2024-08-04 02:14:49,192 INFO Thread-12 :11553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_021444-pk5j08lr/files/output.log +2024-08-04 02:14:50,179 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 02:14:50,193 INFO Thread-12 :11553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_021444-pk5j08lr/files/output.log +2024-08-04 02:14:50,882 DEBUG SenderThread:11553 [sender.py:send():382] send: config +2024-08-04 02:14:50,882 DEBUG SenderThread:11553 [sender.py:send():382] send: config +2024-08-04 02:14:51,067 DEBUG SenderThread:11553 [sender.py:send():382] send: exit +2024-08-04 02:14:51,067 INFO SenderThread:11553 [sender.py:send_exit():589] handling exit code: 0 +2024-08-04 02:14:51,067 INFO SenderThread:11553 [sender.py:send_exit():591] handling runtime: 5 +2024-08-04 02:14:51,068 INFO SenderThread:11553 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 02:14:51,068 INFO SenderThread:11553 [sender.py:send_exit():597] send defer +2024-08-04 02:14:51,068 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:14:51,068 INFO HandlerThread:11553 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-04 02:14:51,069 DEBUG SenderThread:11553 [sender.py:send_request():409] send_request: defer +2024-08-04 02:14:51,069 INFO SenderThread:11553 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-04 02:14:51,069 INFO SenderThread:11553 [sender.py:transition_state():617] send defer: 1 +2024-08-04 02:14:51,069 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:14:51,069 INFO HandlerThread:11553 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-04 02:14:51,069 DEBUG SenderThread:11553 [sender.py:send_request():409] send_request: defer +2024-08-04 02:14:51,069 INFO SenderThread:11553 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-04 02:14:51,069 INFO SenderThread:11553 [sender.py:transition_state():617] send defer: 2 +2024-08-04 02:14:51,069 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:14:51,069 INFO HandlerThread:11553 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-04 02:14:51,069 INFO HandlerThread:11553 [system_monitor.py:finish():203] Stopping system monitor +2024-08-04 02:14:51,069 DEBUG SystemMonitor:11553 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-04 02:14:51,070 INFO HandlerThread:11553 [interfaces.py:finish():202] Joined cpu monitor +2024-08-04 02:14:51,070 DEBUG SystemMonitor:11553 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-04 02:14:51,070 INFO HandlerThread:11553 [interfaces.py:finish():202] Joined disk monitor +2024-08-04 02:14:51,070 DEBUG SystemMonitor:11553 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-04 02:14:51,103 INFO HandlerThread:11553 [interfaces.py:finish():202] Joined gpu monitor +2024-08-04 02:14:51,103 INFO HandlerThread:11553 [interfaces.py:finish():202] Joined memory monitor +2024-08-04 02:14:51,103 INFO HandlerThread:11553 [interfaces.py:finish():202] Joined network monitor +2024-08-04 02:14:51,104 DEBUG SenderThread:11553 [sender.py:send_request():409] send_request: defer +2024-08-04 02:14:51,104 INFO SenderThread:11553 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-04 02:14:51,104 INFO SenderThread:11553 [sender.py:transition_state():617] send defer: 3 +2024-08-04 02:14:51,104 DEBUG SenderThread:11553 [sender.py:send():382] send: stats +2024-08-04 02:14:51,104 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:14:51,104 INFO HandlerThread:11553 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-04 02:14:51,104 DEBUG SenderThread:11553 [sender.py:send_request():409] send_request: defer +2024-08-04 02:14:51,104 INFO SenderThread:11553 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-04 02:14:51,105 INFO SenderThread:11553 [sender.py:transition_state():617] send defer: 4 +2024-08-04 02:14:51,105 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:14:51,105 INFO HandlerThread:11553 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-04 02:14:51,105 DEBUG SenderThread:11553 [sender.py:send_request():409] send_request: defer +2024-08-04 02:14:51,105 INFO SenderThread:11553 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-04 02:14:51,105 INFO SenderThread:11553 [sender.py:transition_state():617] send defer: 5 +2024-08-04 02:14:51,105 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:14:51,105 INFO HandlerThread:11553 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-04 02:14:51,105 DEBUG SenderThread:11553 [sender.py:send():382] send: summary +2024-08-04 02:14:51,106 INFO SenderThread:11553 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 02:14:51,106 DEBUG SenderThread:11553 [sender.py:send_request():409] send_request: defer +2024-08-04 02:14:51,106 INFO SenderThread:11553 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-04 02:14:51,106 INFO SenderThread:11553 [sender.py:transition_state():617] send defer: 6 +2024-08-04 02:14:51,106 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:14:51,106 INFO HandlerThread:11553 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-04 02:14:51,107 DEBUG SenderThread:11553 [sender.py:send_request():409] send_request: defer +2024-08-04 02:14:51,107 INFO SenderThread:11553 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-04 02:14:51,109 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 02:14:51,194 INFO Thread-12 :11553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_021444-pk5j08lr/files/output.log +2024-08-04 02:14:51,194 INFO Thread-12 :11553 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_021444-pk5j08lr/files/wandb-summary.json +2024-08-04 02:14:51,396 INFO SenderThread:11553 [sender.py:transition_state():617] send defer: 7 +2024-08-04 02:14:51,396 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:14:51,396 INFO HandlerThread:11553 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-04 02:14:51,396 DEBUG SenderThread:11553 [sender.py:send_request():409] send_request: defer +2024-08-04 02:14:51,396 INFO SenderThread:11553 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-04 02:14:52,066 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 02:14:52,195 INFO Thread-12 :11553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_021444-pk5j08lr/files/config.yaml +2024-08-04 02:14:52,195 INFO Thread-12 :11553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_021444-pk5j08lr/files/output.log +2024-08-04 02:14:52,692 INFO SenderThread:11553 [sender.py:transition_state():617] send defer: 8 +2024-08-04 02:14:52,692 DEBUG SenderThread:11553 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 02:14:52,692 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:14:52,692 INFO HandlerThread:11553 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-04 02:14:52,692 DEBUG SenderThread:11553 [sender.py:send_request():409] send_request: defer +2024-08-04 02:14:52,693 INFO SenderThread:11553 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-04 02:14:52,693 INFO SenderThread:11553 [job_builder.py:build():296] Attempting to build job artifact +2024-08-04 02:14:52,693 INFO SenderThread:11553 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-04 02:14:52,707 INFO SenderThread:11553 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-04 02:14:52,715 INFO SenderThread:11553 [sender.py:transition_state():617] send defer: 9 +2024-08-04 02:14:52,716 DEBUG SenderThread:11553 [sender.py:send():382] send: artifact +2024-08-04 02:14:52,716 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:14:52,717 INFO HandlerThread:11553 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-04 02:14:53,067 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 02:14:53,195 INFO Thread-12 :11553 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_021444-pk5j08lr/files/output.log +2024-08-04 02:14:53,655 INFO SenderThread:11553 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTA5MTk2NTkzOA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjM2MjY3MjMzNA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTA5MzUzODM4NQ==', 'versionIndex': 3}}} +2024-08-04 02:14:53,655 DEBUG SenderThread:11553 [sender.py:send_request():409] send_request: defer +2024-08-04 02:14:53,655 INFO SenderThread:11553 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-04 02:14:53,655 INFO SenderThread:11553 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-04 02:14:54,196 INFO SenderThread:11553 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240804_021444-pk5j08lr/files +2024-08-04 02:14:54,197 INFO SenderThread:11553 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_021444-pk5j08lr/files/requirements.txt requirements.txt +2024-08-04 02:14:54,197 INFO SenderThread:11553 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_021444-pk5j08lr/files/config.yaml config.yaml +2024-08-04 02:14:54,198 INFO SenderThread:11553 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_021444-pk5j08lr/files/wandb-metadata.json wandb-metadata.json +2024-08-04 02:14:54,198 INFO SenderThread:11553 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_021444-pk5j08lr/files/wandb-summary.json wandb-summary.json +2024-08-04 02:14:54,200 INFO SenderThread:11553 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_021444-pk5j08lr/files/output.log output.log +2024-08-04 02:14:54,200 INFO SenderThread:11553 [sender.py:transition_state():617] send defer: 10 +2024-08-04 02:14:54,202 DEBUG SenderThread:11553 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 02:14:54,202 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:14:54,205 INFO HandlerThread:11553 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-04 02:14:54,206 DEBUG SenderThread:11553 [sender.py:send_request():409] send_request: defer +2024-08-04 02:14:54,206 INFO SenderThread:11553 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-04 02:14:54,206 INFO SenderThread:11553 [file_pusher.py:finish():172] shutting down file pusher +2024-08-04 02:14:54,605 INFO wandb-upload_1:11553 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_021444-pk5j08lr/files/config.yaml +2024-08-04 02:14:54,711 INFO wandb-upload_0:11553 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_021444-pk5j08lr/files/requirements.txt +2024-08-04 02:14:54,762 INFO wandb-upload_2:11553 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_021444-pk5j08lr/files/wandb-summary.json +2024-08-04 02:14:54,792 INFO wandb-upload_3:11553 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_021444-pk5j08lr/files/output.log +2024-08-04 02:14:54,992 INFO Thread-11 (_thread_body):11553 [sender.py:transition_state():617] send defer: 11 +2024-08-04 02:14:54,992 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:14:54,992 INFO HandlerThread:11553 [handler.py:handle_request_defer():172] handle defer: 11 +2024-08-04 02:14:54,992 DEBUG SenderThread:11553 [sender.py:send_request():409] send_request: defer +2024-08-04 02:14:54,992 INFO SenderThread:11553 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-08-04 02:14:54,993 INFO SenderThread:11553 [file_pusher.py:join():178] waiting for file pusher +2024-08-04 02:14:54,993 INFO SenderThread:11553 [sender.py:transition_state():617] send defer: 12 +2024-08-04 02:14:54,993 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:14:54,993 INFO HandlerThread:11553 [handler.py:handle_request_defer():172] handle defer: 12 +2024-08-04 02:14:54,993 DEBUG SenderThread:11553 [sender.py:send_request():409] send_request: defer +2024-08-04 02:14:54,993 INFO SenderThread:11553 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-08-04 02:14:54,993 INFO SenderThread:11553 [file_stream.py:finish():595] file stream finish called +2024-08-04 02:14:55,067 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 02:14:55,176 INFO SenderThread:11553 [file_stream.py:finish():599] file stream finish is done +2024-08-04 02:14:55,176 INFO SenderThread:11553 [sender.py:transition_state():617] send defer: 13 +2024-08-04 02:14:55,176 DEBUG SenderThread:11553 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 02:14:55,176 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:14:55,177 INFO HandlerThread:11553 [handler.py:handle_request_defer():172] handle defer: 13 +2024-08-04 02:14:55,177 DEBUG SenderThread:11553 [sender.py:send_request():409] send_request: defer +2024-08-04 02:14:55,177 INFO SenderThread:11553 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-08-04 02:14:55,177 INFO SenderThread:11553 [sender.py:transition_state():617] send defer: 14 +2024-08-04 02:14:55,177 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: defer +2024-08-04 02:14:55,177 DEBUG SenderThread:11553 [sender.py:send():382] send: final +2024-08-04 02:14:55,177 INFO HandlerThread:11553 [handler.py:handle_request_defer():172] handle defer: 14 +2024-08-04 02:14:55,177 DEBUG SenderThread:11553 [sender.py:send():382] send: footer +2024-08-04 02:14:55,178 DEBUG SenderThread:11553 [sender.py:send_request():409] send_request: defer +2024-08-04 02:14:55,178 INFO SenderThread:11553 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-08-04 02:14:55,178 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 02:14:55,178 DEBUG SenderThread:11553 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 02:14:55,178 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 02:14:55,179 DEBUG SenderThread:11553 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 02:14:55,179 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: server_info +2024-08-04 02:14:55,179 DEBUG SenderThread:11553 [sender.py:send_request():409] send_request: server_info +2024-08-04 02:14:55,180 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: get_summary +2024-08-04 02:14:55,181 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: sampled_history +2024-08-04 02:14:55,181 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 02:14:55,181 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: job_info +2024-08-04 02:14:55,346 DEBUG SenderThread:11553 [sender.py:send_request():409] send_request: job_info +2024-08-04 02:14:55,346 INFO MainThread:11553 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-08-04 02:14:55,346 INFO MainThread:11553 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-08-04 02:14:55,346 INFO MainThread:11553 [wandb_run.py:_footer_sync_info():3825] logging synced files +2024-08-04 02:14:55,346 DEBUG HandlerThread:11553 [handler.py:handle_request():146] handle_request: shutdown +2024-08-04 02:14:55,346 INFO HandlerThread:11553 [handler.py:finish():869] shutting down handler +2024-08-04 02:14:56,181 INFO WriterThread:11553 [datastore.py:close():296] close: /project/wandb/run-20240804_021444-pk5j08lr/run-pk5j08lr.wandb +2024-08-04 02:14:56,346 INFO SenderThread:11553 [sender.py:finish():1572] shutting down sender +2024-08-04 02:14:56,346 INFO SenderThread:11553 [file_pusher.py:finish():172] shutting down file pusher +2024-08-04 02:14:56,346 INFO SenderThread:11553 [file_pusher.py:join():178] waiting for file pusher diff --git a/wandb/run-20240804_021444-pk5j08lr/logs/debug.log b/wandb/run-20240804_021444-pk5j08lr/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b39037fae993867f61df66c38bf9f9f548e3f6f5 --- /dev/null +++ b/wandb/run-20240804_021444-pk5j08lr/logs/debug.log @@ -0,0 +1,30 @@ +2024-08-04 02:14:44,708 INFO MainThread:11482 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-04 02:14:44,708 INFO MainThread:11482 [wandb_setup.py:_flush():76] Configure stats pid to 11482 +2024-08-04 02:14:44,708 INFO MainThread:11482 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-04 02:14:44,708 INFO MainThread:11482 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-04 02:14:44,708 INFO MainThread:11482 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train tuny llama sample'} +2024-08-04 02:14:44,708 INFO MainThread:11482 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-04 02:14:44,708 INFO MainThread:11482 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-04 02:14:44,708 INFO MainThread:11482 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240804_021444-pk5j08lr/logs/debug.log +2024-08-04 02:14:44,708 INFO MainThread:11482 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240804_021444-pk5j08lr/logs/debug-internal.log +2024-08-04 02:14:44,708 INFO MainThread:11482 [wandb_init.py:init():566] calling init triggers +2024-08-04 02:14:44,708 INFO MainThread:11482 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'valid_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'test_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 1024, 'num_workers': 2, 'tokenizer_type': 'Llama2Tokenizer', 'tokenizer_model': '/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'tiny-mistral-sample_train_2024-08-04-02:14:34', 'wandb_project': 'llm_tutorial', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/tiny-mistral-sample', 'save': '/work/llm_recipes/models/tiny-mistral-sample', 'base_model': '/share/pretrained_lm/custom/tiny-mistral', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 200, 'save_interval': 200, 'eval_iters': 10, 'optimizer': 'adam', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 20000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 20000, 'train_samples': None, 'global_batch_size': 320, 'micro_batch_size': 8, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 4096, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/tiny-mistral-sample', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 32768, 'gradient_accumulation_steps': 40} +2024-08-04 02:14:44,708 INFO MainThread:11482 [wandb_init.py:init():616] starting backend +2024-08-04 02:14:44,708 INFO MainThread:11482 [wandb_init.py:init():620] setting up manager +2024-08-04 02:14:44,713 INFO MainThread:11482 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-04 02:14:44,714 INFO MainThread:11482 [wandb_init.py:init():628] backend started and connected +2024-08-04 02:14:44,719 INFO MainThread:11482 [wandb_init.py:init():720] updated telemetry +2024-08-04 02:14:44,729 INFO MainThread:11482 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-04 02:14:45,194 INFO MainThread:11482 [wandb_run.py:_on_init():2262] communicating current version +2024-08-04 02:14:45,273 INFO MainThread:11482 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.5 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-04 02:14:45,274 INFO MainThread:11482 [wandb_init.py:init():804] starting run threads in backend +2024-08-04 02:14:45,331 INFO MainThread:11482 [wandb_run.py:_console_start():2241] atexit reg +2024-08-04 02:14:45,331 INFO MainThread:11482 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-04 02:14:45,332 INFO MainThread:11482 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-04 02:14:45,332 INFO MainThread:11482 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-04 02:14:45,333 INFO MainThread:11482 [wandb_init.py:init():847] run started, returning control to user process +2024-08-04 02:14:50,881 INFO MainThread:11482 [wandb_run.py:_config_callback():1343] config_cb None None {'activation_function': 'silu', 'hidden_size': 256, 'model_type': 'mistral', 'max_position_embeddings': 1024, 'num_attention_heads': 4, 'num_hidden_layers': 4, 'model_architecture': 'MistralForCausalLM'} +2024-08-04 02:14:50,881 INFO MainThread:11482 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1} +2024-08-04 02:14:56,347 WARNING MsgRouterThr:11482 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20240804_021444-pk5j08lr/run-pk5j08lr.wandb b/wandb/run-20240804_021444-pk5j08lr/run-pk5j08lr.wandb new file mode 100644 index 0000000000000000000000000000000000000000..ed00c83a495318ecbea176c8563b271e0d4c7dc5 Binary files /dev/null and b/wandb/run-20240804_021444-pk5j08lr/run-pk5j08lr.wandb differ diff --git a/wandb/run-20240804_144007-dds6qqbt/files/config.yaml b/wandb/run-20240804_144007-dds6qqbt/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97581daca9a197b4556b2e0444b776ac580cc007 --- /dev/null +++ b/wandb/run-20240804_144007-dds6qqbt/files/config.yaml @@ -0,0 +1,335 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: FULL_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +valid_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +test_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 512 +num_workers: + desc: null + value: 2 +tokenizer_type: + desc: null + value: Llama2Tokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/meta-llama/TinyLlama_v1.1/tokenizer.model +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: tiny-llama_train_2024-08-04-14:39:57 +wandb_project: + desc: null + value: llm_tutorial +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/tiny-llama +save: + desc: null + value: /work/llm_recipes/models/tiny-llama +base_model: + desc: null + value: /share/pretrained_lm/meta-llama/TinyLlama_v1.1 +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 200 +save_interval: + desc: null + value: 200 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: adam +lr: + desc: null + value: 2.0e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 2000 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 1.0e-06 +train_iters: + desc: null + value: 2000 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 320 +micro_batch_size: + desc: null + value: 8 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 4096 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-06 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/tiny-llama +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: false +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 32000 +gradient_accumulation_steps: + desc: null + value: 40 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1722750007.607754 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +activation_function: + desc: null + value: silu +hidden_size: + desc: null + value: 2048 +model_type: + desc: null + value: llama +max_position_embeddings: + desc: null + value: 2048 +num_attention_heads: + desc: null + value: 32 +num_hidden_layers: + desc: null + value: 22 +model_architecture: + desc: null + value: LlamaForCausalLM diff --git a/wandb/run-20240804_144007-dds6qqbt/files/output.log b/wandb/run-20240804_144007-dds6qqbt/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..9abb53ace2c37c8a4d95ebea31daf244e7fa2440 --- /dev/null +++ b/wandb/run-20240804_144007-dds6qqbt/files/output.log @@ -0,0 +1,135 @@ +Created Hugging Face repository with ID koichi12/tiny-llama. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +File not found: /work/llm_recipes/models/tiny-llama/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-llama/latest_iteration.txt +File not found: /work/llm_recipes/models/tiny-llama/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-llama/latest_iteration.txt +File not found: /work/llm_recipes/models/tiny-llama/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-llama/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/tiny-llama, skipping model loading +--> Model /share/pretrained_lm/meta-llama/TinyLlama_v1.1 +--> /share/pretrained_lm/meta-llama/TinyLlama_v1.1 has 1100.048384 Million params +You are attempting to use Flash Attention 2.0 without specifying a torch dtype. This might lead to unexpected behaviour +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +Flash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in LlamaForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `torch_dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", torch_dtype=torch.float16)` +Flash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in LlamaModel is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `torch_dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", torch_dtype=torch.float16)` +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1. + warnings.warn( +BFloat16 enabled for mixed precision - using bfSixteen policy +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 640000 + validation: 35200 + test: 3200 +> building train, validation, and test datasets for GPT ... +> finished creating GPT datasets ... +File not found: /work/llm_recipes/models/tiny-llama/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-llama/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/tiny-llama, skipping optimizer loading +File not found: /work/llm_recipes/models/tiny-llama/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-llama/latest_iteration.txt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): LlamaForCausalLM( + (model): LlamaModel( + (embed_tokens): Embedding(32000, 2048) + (layers): ModuleList( + (0-21): 22 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): LlamaDecoderLayer( + (self_attn): LlamaFlashAttention2( + (q_proj): Linear(in_features=2048, out_features=2048, bias=False) + (k_proj): Linear(in_features=2048, out_features=256, bias=False) + (v_proj): Linear(in_features=2048, out_features=256, bias=False) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + (rotary_emb): LlamaRotaryEmbedding() + ) + (mlp): LlamaMLP( + (gate_proj): Linear(in_features=2048, out_features=5632, bias=False) + (up_proj): Linear(in_features=2048, out_features=5632, bias=False) + (down_proj): Linear(in_features=5632, out_features=2048, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): LlamaRMSNorm() + (post_attention_layernorm): LlamaRMSNorm() + ) + ) + ) + ) + (norm): LlamaRMSNorm() + (rotary_emb): LlamaRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=32000, bias=False) + ) +) +model config: LlamaConfig { + "_name_or_path": "/share/pretrained_lm/meta-llama/TinyLlama_v1.1", + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 1, + "eos_token_id": 2, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 5632, + "label_smoothing": 0.0, + "max_position_embeddings": 2048, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 22, + "num_key_value_heads": 4, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 10000.0, + "tie_word_embeddings": false, + "torch_dtype": "float32", + "transformers_version": "4.43.3", + "use_cache": false, + "vocab_size": 32000 +} +Let split = None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Traceback (most recent call last): + File "/project/examples/finetuning.py", line 13, in + main() + File "/project/src/llama_recipes/finetuning.py", line 281, in main + train( + File "/project/src/llama_recipes/utils/train_utils.py", line 104, in train + batch = next(train_dataloader) + File "/project/src/llama_recipes/utils/train_utils.py", line 24, in cyclic_iter + for x in iter: + File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py", line 631, in __next__ + data = self._next_data() + File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py", line 1346, in _next_data + return self._process_data(data) + File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py", line 1372, in _process_data + data.reraise() + File "/usr/local/lib/python3.10/dist-packages/torch/_utils.py", line 705, in reraise + raise exception +RuntimeError: Caught RuntimeError in DataLoader worker process 0. +Original Traceback (most recent call last): + File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/worker.py", line 308, in _worker_loop + data = fetcher.fetch(index) + File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/fetch.py", line 54, in fetch + return self.collate_fn(data) + File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/collate.py", line 277, in default_collate + return collate(batch, collate_fn_map=default_collate_fn_map) + File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/collate.py", line 129, in collate + return elem_type({key: collate([d[key] for d in batch], collate_fn_map=collate_fn_map) for key in elem}) + File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/collate.py", line 129, in + return elem_type({key: collate([d[key] for d in batch], collate_fn_map=collate_fn_map) for key in elem}) + File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/collate.py", line 121, in collate + return collate_fn_map[elem_type](batch, collate_fn_map=collate_fn_map) + File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/collate.py", line 174, in collate_tensor_fn + return torch.stack(batch, 0, out=out) +RuntimeError: stack expects each tensor to be equal size, but got [513] at entry 0 and [543] at entry 1 \ No newline at end of file diff --git a/wandb/run-20240804_144007-dds6qqbt/files/requirements.txt b/wandb/run-20240804_144007-dds6qqbt/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7c90fd1c54d0a9881f6b4c6465b2a4fa88c9056c --- /dev/null +++ b/wandb/run-20240804_144007-dds6qqbt/files/requirements.txt @@ -0,0 +1,271 @@ +absl-py==2.1.0 +accelerate==0.33.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bleach==6.1.0 +blis==0.7.11 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +distributed==2023.11.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +einops==0.7.0 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +grpcio==1.60.1 +huggingface-hub==0.24.5 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +jedi==0.19.1 +jinja2==3.1.3 +joblib==1.3.2 +json5==0.9.14 +jsonnet==0.19.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +kiwisolver==1.4.5 +langcodes==3.3.0 +lazy-loader==0.3 +librosa==0.10.1 +llvmlite==0.40.1 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +murmurhash==1.0.10 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +onnx==1.15.0rc2 +opencv==4.7.0 +optree==0.10.0 +packaging==23.2 +pandas==1.5.3 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +peft==0.11.1 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==14.0.1.dev0+gba5374836.d20240125 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pynvml==11.4.1 +pyparsing==3.1.1 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests==2.31.0 +rich==13.7.0 +rmm==23.12.0 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.0 +safetensors==0.4.3 +scikit-learn==1.2.0 +scipy==1.12.0 +send2trash==1.8.2 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +srsly==2.4.8 +stack-data==0.6.3 +sympy==1.12 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm==4.66.1 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.9.0 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +urllib3==1.26.18 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +xdoctest==1.0.2 +xgboost==1.7.6 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 \ No newline at end of file diff --git a/wandb/run-20240804_144007-dds6qqbt/files/wandb-metadata.json b/wandb/run-20240804_144007-dds6qqbt/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..571e57191002af329e26e892599bae2dbdaf5bb1 --- /dev/null +++ b/wandb/run-20240804_144007-dds6qqbt/files/wandb-metadata.json @@ -0,0 +1,215 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-04T05:40:08.224323", + "startedAt": "2024-08-04T05:40:07.595226", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "512", + "--sliding-window-size", + "4096", + "--micro-batch-size", + "8", + "--global-batch-size", + "320", + "--train-iters", + "2000", + "--tokenizer-type", + "Llama2Tokenizer", + "--tokenizer-model", + "/share/pretrained_lm/meta-llama/TinyLlama_v1.1/tokenizer.model", + "--train-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--valid-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--test-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--lr", + "2e-5", + "--min-lr", + "1e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "2000", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "adam", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-6", + "--save-interval", + "200", + "--eval-interval", + "200", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/meta-llama/TinyLlama_v1.1", + "--save", + "/work/llm_recipes/models/tiny-llama", + "--load", + "/work/llm_recipes/models/tiny-llama", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "FULL_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/tiny-llama", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "llm_tutorial", + "--wandb-name", + "tiny-llama_train_2024-08-04-14:39:57" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "3be5353210a678dc7008f237fa16b99f2bdf36ea" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.0389999999993, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.48781967163086 + } +} diff --git a/wandb/run-20240804_144007-dds6qqbt/files/wandb-summary.json b/wandb/run-20240804_144007-dds6qqbt/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6a2353df9a39aec28b5e444685dc5d7223bc37fd --- /dev/null +++ b/wandb/run-20240804_144007-dds6qqbt/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 2}} \ No newline at end of file diff --git a/wandb/run-20240804_144007-dds6qqbt/logs/debug-internal.log b/wandb/run-20240804_144007-dds6qqbt/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..91b96fae03bf11247eec249b95ed72d50b9898f9 --- /dev/null +++ b/wandb/run-20240804_144007-dds6qqbt/logs/debug-internal.log @@ -0,0 +1,186 @@ +2024-08-04 14:40:07,609 INFO StreamThr :11999 [internal.py:wandb_internal():86] W&B internal server running at pid: 11999, started at: 2024-08-04 14:40:07.608480 +2024-08-04 14:40:07,610 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: status +2024-08-04 14:40:07,612 INFO WriterThread:11999 [datastore.py:open_for_write():87] open: /project/wandb/run-20240804_144007-dds6qqbt/run-dds6qqbt.wandb +2024-08-04 14:40:07,613 DEBUG SenderThread:11999 [sender.py:send():382] send: header +2024-08-04 14:40:07,627 DEBUG SenderThread:11999 [sender.py:send():382] send: run +2024-08-04 14:40:08,110 INFO SenderThread:11999 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240804_144007-dds6qqbt/files +2024-08-04 14:40:08,111 INFO SenderThread:11999 [sender.py:_start_run_threads():1136] run started: dds6qqbt with start time 1722750007.607754 +2024-08-04 14:40:08,116 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: check_version +2024-08-04 14:40:08,116 DEBUG SenderThread:11999 [sender.py:send_request():409] send_request: check_version +2024-08-04 14:40:08,204 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: run_start +2024-08-04 14:40:08,210 DEBUG HandlerThread:11999 [system_info.py:__init__():27] System info init +2024-08-04 14:40:08,210 DEBUG HandlerThread:11999 [system_info.py:__init__():42] System info init done +2024-08-04 14:40:08,211 INFO HandlerThread:11999 [system_monitor.py:start():194] Starting system monitor +2024-08-04 14:40:08,211 INFO SystemMonitor:11999 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-04 14:40:08,211 INFO HandlerThread:11999 [system_monitor.py:probe():214] Collecting system info +2024-08-04 14:40:08,211 INFO SystemMonitor:11999 [interfaces.py:start():190] Started cpu monitoring +2024-08-04 14:40:08,212 INFO SystemMonitor:11999 [interfaces.py:start():190] Started disk monitoring +2024-08-04 14:40:08,213 INFO SystemMonitor:11999 [interfaces.py:start():190] Started gpu monitoring +2024-08-04 14:40:08,214 INFO SystemMonitor:11999 [interfaces.py:start():190] Started memory monitoring +2024-08-04 14:40:08,214 INFO SystemMonitor:11999 [interfaces.py:start():190] Started network monitoring +2024-08-04 14:40:08,224 DEBUG HandlerThread:11999 [system_info.py:probe():151] Probing system +2024-08-04 14:40:08,226 DEBUG HandlerThread:11999 [system_info.py:_probe_git():136] Probing git +2024-08-04 14:40:08,238 DEBUG HandlerThread:11999 [system_info.py:_probe_git():144] Probing git done +2024-08-04 14:40:08,238 DEBUG HandlerThread:11999 [system_info.py:probe():199] Probing system done +2024-08-04 14:40:08,238 DEBUG HandlerThread:11999 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-04T05:40:08.224323', 'startedAt': '2024-08-04T05:40:07.595226', 'docker': None, 'cuda': None, 'args': ('--seq-length', '512', '--sliding-window-size', '4096', '--micro-batch-size', '8', '--global-batch-size', '320', '--train-iters', '2000', '--tokenizer-type', 'Llama2Tokenizer', '--tokenizer-model', '/share/pretrained_lm/meta-llama/TinyLlama_v1.1/tokenizer.model', '--train-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--valid-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--test-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '2000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'adam', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '200', '--eval-interval', '200', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/meta-llama/TinyLlama_v1.1', '--save', '/work/llm_recipes/models/tiny-llama', '--load', '/work/llm_recipes/models/tiny-llama', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/tiny-llama', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial', '--wandb-name', 'tiny-llama_train_2024-08-04-14:39:57'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '3be5353210a678dc7008f237fa16b99f2bdf36ea'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.0389999999993, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.48781967163086}} +2024-08-04 14:40:08,238 INFO HandlerThread:11999 [system_monitor.py:probe():224] Finished collecting system info +2024-08-04 14:40:08,238 INFO HandlerThread:11999 [system_monitor.py:probe():227] Publishing system info +2024-08-04 14:40:08,239 INFO HandlerThread:11999 [system_monitor.py:probe():229] Finished publishing system info +2024-08-04 14:40:08,245 DEBUG SenderThread:11999 [sender.py:send():382] send: files +2024-08-04 14:40:08,246 INFO SenderThread:11999 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-04 14:40:08,255 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: python_packages +2024-08-04 14:40:08,255 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 14:40:08,255 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 14:40:08,255 DEBUG SenderThread:11999 [sender.py:send_request():409] send_request: python_packages +2024-08-04 14:40:08,257 DEBUG SenderThread:11999 [sender.py:send_request():409] send_request: stop_status +2024-08-04 14:40:08,521 DEBUG SenderThread:11999 [sender.py:send():382] send: telemetry +2024-08-04 14:40:08,889 INFO wandb-upload_0:11999 [upload_job.py:push():131] Uploaded file /tmp/tmp5bbx13axwandb/8bl0rtdu-wandb-metadata.json +2024-08-04 14:40:09,112 INFO Thread-12 :11999 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_144007-dds6qqbt/files/requirements.txt +2024-08-04 14:40:09,113 INFO Thread-12 :11999 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_144007-dds6qqbt/files/output.log +2024-08-04 14:40:09,113 INFO Thread-12 :11999 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_144007-dds6qqbt/files/wandb-metadata.json +2024-08-04 14:40:10,756 DEBUG SenderThread:11999 [sender.py:send():382] send: config +2024-08-04 14:40:10,756 DEBUG SenderThread:11999 [sender.py:send():382] send: config +2024-08-04 14:40:10,842 DEBUG SenderThread:11999 [sender.py:send():382] send: exit +2024-08-04 14:40:10,842 INFO SenderThread:11999 [sender.py:send_exit():589] handling exit code: 1 +2024-08-04 14:40:10,842 INFO SenderThread:11999 [sender.py:send_exit():591] handling runtime: 2 +2024-08-04 14:40:10,843 INFO SenderThread:11999 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 14:40:10,843 INFO SenderThread:11999 [sender.py:send_exit():597] send defer +2024-08-04 14:40:10,844 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: defer +2024-08-04 14:40:10,844 INFO HandlerThread:11999 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-04 14:40:10,844 DEBUG SenderThread:11999 [sender.py:send_request():409] send_request: defer +2024-08-04 14:40:10,844 INFO SenderThread:11999 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-04 14:40:10,844 INFO SenderThread:11999 [sender.py:transition_state():617] send defer: 1 +2024-08-04 14:40:10,844 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: defer +2024-08-04 14:40:10,844 INFO HandlerThread:11999 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-04 14:40:10,844 DEBUG SenderThread:11999 [sender.py:send_request():409] send_request: defer +2024-08-04 14:40:10,844 INFO SenderThread:11999 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-04 14:40:10,844 INFO SenderThread:11999 [sender.py:transition_state():617] send defer: 2 +2024-08-04 14:40:10,844 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: defer +2024-08-04 14:40:10,844 INFO HandlerThread:11999 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-04 14:40:10,844 INFO HandlerThread:11999 [system_monitor.py:finish():203] Stopping system monitor +2024-08-04 14:40:10,845 DEBUG SystemMonitor:11999 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-04 14:40:10,845 INFO HandlerThread:11999 [interfaces.py:finish():202] Joined cpu monitor +2024-08-04 14:40:10,845 DEBUG SystemMonitor:11999 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-04 14:40:10,845 INFO HandlerThread:11999 [interfaces.py:finish():202] Joined disk monitor +2024-08-04 14:40:10,845 DEBUG SystemMonitor:11999 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-04 14:40:10,878 INFO HandlerThread:11999 [interfaces.py:finish():202] Joined gpu monitor +2024-08-04 14:40:10,878 INFO HandlerThread:11999 [interfaces.py:finish():202] Joined memory monitor +2024-08-04 14:40:10,878 INFO HandlerThread:11999 [interfaces.py:finish():202] Joined network monitor +2024-08-04 14:40:10,878 DEBUG SenderThread:11999 [sender.py:send_request():409] send_request: defer +2024-08-04 14:40:10,878 INFO SenderThread:11999 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-04 14:40:10,878 INFO SenderThread:11999 [sender.py:transition_state():617] send defer: 3 +2024-08-04 14:40:10,879 DEBUG SenderThread:11999 [sender.py:send():382] send: stats +2024-08-04 14:40:10,879 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: defer +2024-08-04 14:40:10,879 INFO HandlerThread:11999 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-04 14:40:10,879 DEBUG SenderThread:11999 [sender.py:send_request():409] send_request: defer +2024-08-04 14:40:10,879 INFO SenderThread:11999 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-04 14:40:10,879 INFO SenderThread:11999 [sender.py:transition_state():617] send defer: 4 +2024-08-04 14:40:10,879 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: defer +2024-08-04 14:40:10,879 INFO HandlerThread:11999 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-04 14:40:10,879 DEBUG SenderThread:11999 [sender.py:send_request():409] send_request: defer +2024-08-04 14:40:10,879 INFO SenderThread:11999 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-04 14:40:10,879 INFO SenderThread:11999 [sender.py:transition_state():617] send defer: 5 +2024-08-04 14:40:10,879 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: defer +2024-08-04 14:40:10,880 INFO HandlerThread:11999 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-04 14:40:10,880 DEBUG SenderThread:11999 [sender.py:send():382] send: summary +2024-08-04 14:40:10,881 INFO SenderThread:11999 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 14:40:10,881 DEBUG SenderThread:11999 [sender.py:send_request():409] send_request: defer +2024-08-04 14:40:10,881 INFO SenderThread:11999 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-04 14:40:10,881 INFO SenderThread:11999 [sender.py:transition_state():617] send defer: 6 +2024-08-04 14:40:10,881 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: defer +2024-08-04 14:40:10,881 INFO HandlerThread:11999 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-04 14:40:10,881 DEBUG SenderThread:11999 [sender.py:send_request():409] send_request: defer +2024-08-04 14:40:10,881 INFO SenderThread:11999 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-04 14:40:10,884 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 14:40:11,083 INFO SenderThread:11999 [sender.py:transition_state():617] send defer: 7 +2024-08-04 14:40:11,083 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: defer +2024-08-04 14:40:11,083 INFO HandlerThread:11999 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-04 14:40:11,083 DEBUG SenderThread:11999 [sender.py:send_request():409] send_request: defer +2024-08-04 14:40:11,083 INFO SenderThread:11999 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-04 14:40:11,113 INFO Thread-12 :11999 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_144007-dds6qqbt/files/output.log +2024-08-04 14:40:11,114 INFO Thread-12 :11999 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_144007-dds6qqbt/files/config.yaml +2024-08-04 14:40:11,114 INFO Thread-12 :11999 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_144007-dds6qqbt/files/wandb-summary.json +2024-08-04 14:40:11,842 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 14:40:12,953 INFO SenderThread:11999 [sender.py:transition_state():617] send defer: 8 +2024-08-04 14:40:12,953 DEBUG SenderThread:11999 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 14:40:12,953 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: defer +2024-08-04 14:40:12,954 INFO HandlerThread:11999 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-04 14:40:12,954 DEBUG SenderThread:11999 [sender.py:send_request():409] send_request: defer +2024-08-04 14:40:12,954 INFO SenderThread:11999 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-04 14:40:12,954 INFO SenderThread:11999 [job_builder.py:build():296] Attempting to build job artifact +2024-08-04 14:40:12,955 INFO SenderThread:11999 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-04 14:40:12,969 INFO SenderThread:11999 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-04 14:40:12,987 INFO SenderThread:11999 [sender.py:transition_state():617] send defer: 9 +2024-08-04 14:40:12,987 DEBUG SenderThread:11999 [sender.py:send():382] send: artifact +2024-08-04 14:40:12,988 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: defer +2024-08-04 14:40:12,989 INFO HandlerThread:11999 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-04 14:40:13,115 INFO Thread-12 :11999 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_144007-dds6qqbt/files/output.log +2024-08-04 14:40:13,842 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 14:40:13,848 INFO SenderThread:11999 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTA5MTk2NTkzOA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjM2MjY3MjMzNA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTA5MzUzODM4NQ==', 'versionIndex': 3}}} +2024-08-04 14:40:13,848 DEBUG SenderThread:11999 [sender.py:send_request():409] send_request: defer +2024-08-04 14:40:13,848 INFO SenderThread:11999 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-04 14:40:13,848 INFO SenderThread:11999 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-04 14:40:14,116 INFO SenderThread:11999 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240804_144007-dds6qqbt/files +2024-08-04 14:40:14,116 INFO SenderThread:11999 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_144007-dds6qqbt/files/requirements.txt requirements.txt +2024-08-04 14:40:14,116 INFO SenderThread:11999 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_144007-dds6qqbt/files/config.yaml config.yaml +2024-08-04 14:40:14,118 INFO SenderThread:11999 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_144007-dds6qqbt/files/wandb-metadata.json wandb-metadata.json +2024-08-04 14:40:14,118 INFO SenderThread:11999 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_144007-dds6qqbt/files/wandb-summary.json wandb-summary.json +2024-08-04 14:40:14,119 INFO SenderThread:11999 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_144007-dds6qqbt/files/output.log output.log +2024-08-04 14:40:14,121 INFO SenderThread:11999 [sender.py:transition_state():617] send defer: 10 +2024-08-04 14:40:14,121 DEBUG SenderThread:11999 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 14:40:14,121 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: defer +2024-08-04 14:40:14,121 INFO HandlerThread:11999 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-04 14:40:14,123 DEBUG SenderThread:11999 [sender.py:send_request():409] send_request: defer +2024-08-04 14:40:14,123 INFO SenderThread:11999 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-04 14:40:14,123 INFO SenderThread:11999 [file_pusher.py:finish():172] shutting down file pusher +2024-08-04 14:40:14,515 INFO wandb-upload_0:11999 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_144007-dds6qqbt/files/requirements.txt +2024-08-04 14:40:14,617 INFO wandb-upload_1:11999 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_144007-dds6qqbt/files/config.yaml +2024-08-04 14:40:14,698 INFO wandb-upload_2:11999 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_144007-dds6qqbt/files/wandb-summary.json +2024-08-04 14:40:14,843 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 14:40:14,843 DEBUG SenderThread:11999 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 14:40:15,184 INFO wandb-upload_3:11999 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_144007-dds6qqbt/files/output.log +2024-08-04 14:40:15,384 INFO Thread-11 (_thread_body):11999 [sender.py:transition_state():617] send defer: 11 +2024-08-04 14:40:15,384 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: defer +2024-08-04 14:40:15,385 INFO HandlerThread:11999 [handler.py:handle_request_defer():172] handle defer: 11 +2024-08-04 14:40:15,385 DEBUG SenderThread:11999 [sender.py:send_request():409] send_request: defer +2024-08-04 14:40:15,385 INFO SenderThread:11999 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-08-04 14:40:15,385 INFO SenderThread:11999 [file_pusher.py:join():178] waiting for file pusher +2024-08-04 14:40:15,385 INFO SenderThread:11999 [sender.py:transition_state():617] send defer: 12 +2024-08-04 14:40:15,385 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: defer +2024-08-04 14:40:15,385 INFO HandlerThread:11999 [handler.py:handle_request_defer():172] handle defer: 12 +2024-08-04 14:40:15,385 DEBUG SenderThread:11999 [sender.py:send_request():409] send_request: defer +2024-08-04 14:40:15,386 INFO SenderThread:11999 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-08-04 14:40:15,386 INFO SenderThread:11999 [file_stream.py:finish():595] file stream finish called +2024-08-04 14:40:15,573 INFO SenderThread:11999 [file_stream.py:finish():599] file stream finish is done +2024-08-04 14:40:15,573 INFO SenderThread:11999 [sender.py:transition_state():617] send defer: 13 +2024-08-04 14:40:15,573 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: defer +2024-08-04 14:40:15,573 INFO HandlerThread:11999 [handler.py:handle_request_defer():172] handle defer: 13 +2024-08-04 14:40:15,573 DEBUG SenderThread:11999 [sender.py:send_request():409] send_request: defer +2024-08-04 14:40:15,574 INFO SenderThread:11999 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-08-04 14:40:15,574 INFO SenderThread:11999 [sender.py:transition_state():617] send defer: 14 +2024-08-04 14:40:15,574 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: defer +2024-08-04 14:40:15,574 DEBUG SenderThread:11999 [sender.py:send():382] send: final +2024-08-04 14:40:15,574 INFO HandlerThread:11999 [handler.py:handle_request_defer():172] handle defer: 14 +2024-08-04 14:40:15,574 DEBUG SenderThread:11999 [sender.py:send():382] send: footer +2024-08-04 14:40:15,574 DEBUG SenderThread:11999 [sender.py:send_request():409] send_request: defer +2024-08-04 14:40:15,574 INFO SenderThread:11999 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-08-04 14:40:15,575 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 14:40:15,575 DEBUG SenderThread:11999 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 14:40:15,575 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 14:40:15,576 DEBUG SenderThread:11999 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 14:40:15,576 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: server_info +2024-08-04 14:40:15,576 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: get_summary +2024-08-04 14:40:15,576 DEBUG SenderThread:11999 [sender.py:send_request():409] send_request: server_info +2024-08-04 14:40:15,578 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: sampled_history +2024-08-04 14:40:15,578 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 14:40:15,578 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: job_info +2024-08-04 14:40:15,734 DEBUG SenderThread:11999 [sender.py:send_request():409] send_request: job_info +2024-08-04 14:40:15,735 INFO MainThread:11999 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-08-04 14:40:15,735 INFO MainThread:11999 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-08-04 14:40:15,735 INFO MainThread:11999 [wandb_run.py:_footer_sync_info():3825] logging synced files +2024-08-04 14:40:15,735 DEBUG HandlerThread:11999 [handler.py:handle_request():146] handle_request: shutdown +2024-08-04 14:40:15,735 INFO HandlerThread:11999 [handler.py:finish():869] shutting down handler +2024-08-04 14:40:16,578 INFO WriterThread:11999 [datastore.py:close():296] close: /project/wandb/run-20240804_144007-dds6qqbt/run-dds6qqbt.wandb +2024-08-04 14:40:16,735 INFO SenderThread:11999 [sender.py:finish():1572] shutting down sender +2024-08-04 14:40:16,735 INFO SenderThread:11999 [file_pusher.py:finish():172] shutting down file pusher +2024-08-04 14:40:16,735 INFO SenderThread:11999 [file_pusher.py:join():178] waiting for file pusher diff --git a/wandb/run-20240804_144007-dds6qqbt/logs/debug.log b/wandb/run-20240804_144007-dds6qqbt/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e8c917bd91ba547064142d9ec960ef0bbd88b71d --- /dev/null +++ b/wandb/run-20240804_144007-dds6qqbt/logs/debug.log @@ -0,0 +1,30 @@ +2024-08-04 14:40:07,600 INFO MainThread:11928 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-04 14:40:07,601 INFO MainThread:11928 [wandb_setup.py:_flush():76] Configure stats pid to 11928 +2024-08-04 14:40:07,601 INFO MainThread:11928 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-04 14:40:07,601 INFO MainThread:11928 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-04 14:40:07,601 INFO MainThread:11928 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train tiny llama sample'} +2024-08-04 14:40:07,601 INFO MainThread:11928 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-04 14:40:07,601 INFO MainThread:11928 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-04 14:40:07,601 INFO MainThread:11928 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240804_144007-dds6qqbt/logs/debug.log +2024-08-04 14:40:07,601 INFO MainThread:11928 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240804_144007-dds6qqbt/logs/debug-internal.log +2024-08-04 14:40:07,601 INFO MainThread:11928 [wandb_init.py:init():566] calling init triggers +2024-08-04 14:40:07,601 INFO MainThread:11928 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'valid_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'test_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 512, 'num_workers': 2, 'tokenizer_type': 'Llama2Tokenizer', 'tokenizer_model': '/share/pretrained_lm/meta-llama/TinyLlama_v1.1/tokenizer.model', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'tiny-llama_train_2024-08-04-14:39:57', 'wandb_project': 'llm_tutorial', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/tiny-llama', 'save': '/work/llm_recipes/models/tiny-llama', 'base_model': '/share/pretrained_lm/meta-llama/TinyLlama_v1.1', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 200, 'save_interval': 200, 'eval_iters': 10, 'optimizer': 'adam', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 2000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 2000, 'train_samples': None, 'global_batch_size': 320, 'micro_batch_size': 8, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 4096, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/tiny-llama', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 32000, 'gradient_accumulation_steps': 40} +2024-08-04 14:40:07,602 INFO MainThread:11928 [wandb_init.py:init():616] starting backend +2024-08-04 14:40:07,602 INFO MainThread:11928 [wandb_init.py:init():620] setting up manager +2024-08-04 14:40:07,606 INFO MainThread:11928 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-04 14:40:07,607 INFO MainThread:11928 [wandb_init.py:init():628] backend started and connected +2024-08-04 14:40:07,612 INFO MainThread:11928 [wandb_init.py:init():720] updated telemetry +2024-08-04 14:40:07,623 INFO MainThread:11928 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-04 14:40:08,115 INFO MainThread:11928 [wandb_run.py:_on_init():2262] communicating current version +2024-08-04 14:40:08,197 INFO MainThread:11928 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.5 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-04 14:40:08,197 INFO MainThread:11928 [wandb_init.py:init():804] starting run threads in backend +2024-08-04 14:40:08,254 INFO MainThread:11928 [wandb_run.py:_console_start():2241] atexit reg +2024-08-04 14:40:08,254 INFO MainThread:11928 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-04 14:40:08,254 INFO MainThread:11928 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-04 14:40:08,255 INFO MainThread:11928 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-04 14:40:08,255 INFO MainThread:11928 [wandb_init.py:init():847] run started, returning control to user process +2024-08-04 14:40:10,755 INFO MainThread:11928 [wandb_run.py:_config_callback():1343] config_cb None None {'activation_function': 'silu', 'hidden_size': 2048, 'model_type': 'llama', 'max_position_embeddings': 2048, 'num_attention_heads': 32, 'num_hidden_layers': 22, 'model_architecture': 'LlamaForCausalLM'} +2024-08-04 14:40:10,755 INFO MainThread:11928 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1} +2024-08-04 14:40:16,736 WARNING MsgRouterThr:11928 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20240804_144007-dds6qqbt/run-dds6qqbt.wandb b/wandb/run-20240804_144007-dds6qqbt/run-dds6qqbt.wandb new file mode 100644 index 0000000000000000000000000000000000000000..a3ec300c9404cedc12541b7344bed4fdbaaefa0e Binary files /dev/null and b/wandb/run-20240804_144007-dds6qqbt/run-dds6qqbt.wandb differ diff --git a/wandb/run-20240804_222226-kh5katc1/files/config.yaml b/wandb/run-20240804_222226-kh5katc1/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..717b9dfa818b739b61b1f4e94a2c48b5cd855c44 --- /dev/null +++ b/wandb/run-20240804_222226-kh5katc1/files/config.yaml @@ -0,0 +1,335 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: FULL_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '235289369' + - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document +valid_data_path: + desc: null + value: + - '235289369' + - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document +test_data_path: + desc: null + value: + - '235289369' + - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 4096 +num_workers: + desc: null + value: 2 +tokenizer_type: + desc: null + value: HFPreTrainedTokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/google/gemma-2-2b +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: yans-sample-gemma-2-2b_train_2024-08-04-22:22:15 +wandb_project: + desc: null + value: llm_tutorial +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/yans-sample-gemma-2-2b +save: + desc: null + value: /work/llm_recipes/models/yans-sample-gemma-2-2b +base_model: + desc: null + value: /share/pretrained_lm/google/gemma-2-2b +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 200 +save_interval: + desc: null + value: 200 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: anyprecision +lr: + desc: null + value: 2.0e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 20000 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 1.0e-06 +train_iters: + desc: null + value: 20000 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 320 +micro_batch_size: + desc: null + value: 1 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 4096 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-06 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/yans-sample-gemma-2-2b +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: false +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 256000 +gradient_accumulation_steps: + desc: null + value: 320 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1722777746.267116 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +activation_function: + desc: null + value: gelu_pytorch_tanh +hidden_size: + desc: null + value: 2304 +model_type: + desc: null + value: gemma2 +max_position_embeddings: + desc: null + value: 4096 +num_attention_heads: + desc: null + value: 8 +num_hidden_layers: + desc: null + value: 26 +model_architecture: + desc: null + value: Gemma2ForCausalLM diff --git a/wandb/run-20240804_222226-kh5katc1/files/output.log b/wandb/run-20240804_222226-kh5katc1/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..675fff82ef8343b27733c8a7667a1a38777c3bc6 --- /dev/null +++ b/wandb/run-20240804_222226-kh5katc1/files/output.log @@ -0,0 +1,468 @@ +Created Hugging Face repository with ID koichi12/yans-sample-gemma-2-2b. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +File not found: /work/llm_recipes/models/yans-sample-gemma-2-2b/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-sample-gemma-2-2b/latest_iteration.txt +File not found: /work/llm_recipes/models/yans-sample-gemma-2-2b/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-sample-gemma-2-2b/latest_iteration.txt +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. + +Loading checkpoint shards: 67%|██████▋ | 2/3 [00:03<00:01, 1.62s/it] +File not found: /work/llm_recipes/models/yans-sample-gemma-2-2b/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-sample-gemma-2-2b/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/yans-sample-gemma-2-2b, skipping model loading +--> Model /share/pretrained_lm/google/gemma-2-2b +--> /share/pretrained_lm/google/gemma-2-2b has 2614.341888 Million params +BFloat16 enabled for mixed precision - using bfSixteen policy +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 6400000 + validation: 323200 + test: 3200 +> building train, validation, and test datasets for GPT ... +> finished creating GPT datasets ... +File not found: /work/llm_recipes/models/yans-sample-gemma-2-2b/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-sample-gemma-2-2b/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/yans-sample-gemma-2-2b, skipping optimizer loading +File not found: /work/llm_recipes/models/yans-sample-gemma-2-2b/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-sample-gemma-2-2b/latest_iteration.txt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): Gemma2ForCausalLM( + (model): Gemma2Model( + (embed_tokens): Embedding(256000, 2304, padding_idx=0) + (layers): ModuleList( + (0-25): 26 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): Gemma2DecoderLayer( + (self_attn): Gemma2FlashAttention2( + (q_proj): Linear(in_features=2304, out_features=2048, bias=False) + (k_proj): Linear(in_features=2304, out_features=1024, bias=False) + (v_proj): Linear(in_features=2304, out_features=1024, bias=False) + (o_proj): Linear(in_features=2048, out_features=2304, bias=False) + (rotary_emb): Gemma2RotaryEmbedding() + ) + (mlp): Gemma2MLP( + (gate_proj): Linear(in_features=2304, out_features=9216, bias=False) + (up_proj): Linear(in_features=2304, out_features=9216, bias=False) + (down_proj): Linear(in_features=9216, out_features=2304, bias=False) + (act_fn): PytorchGELUTanh() + ) + (input_layernorm): Gemma2RMSNorm() + (post_attention_layernorm): Gemma2RMSNorm() + (pre_feedforward_layernorm): Gemma2RMSNorm() + (post_feedforward_layernorm): Gemma2RMSNorm() + ) + ) + ) + ) + (norm): Gemma2RMSNorm() + ) + (lm_head): Linear(in_features=2304, out_features=256000, bias=False) + ) +) +model config: Gemma2Config { + "_name_or_path": "/share/pretrained_lm/google/gemma-2-2b", + "architectures": [ + "Gemma2ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": 50.0, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "eos_token_id": 1, + "final_logit_softcapping": 30.0, + "head_dim": 256, + "hidden_act": "gelu_pytorch_tanh", + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 2304, + "initializer_range": 0.02, + "intermediate_size": 9216, + "label_smoothing": 0.0, + "max_position_embeddings": 4096, + "model_type": "gemma2", + "num_attention_heads": 8, + "num_hidden_layers": 26, + "num_key_value_heads": 4, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_theta": 10000.0, + "sliding_window": 4096, + "torch_dtype": "float32", + "transformers_version": "4.43.3", + "use_cache": false, + "vocab_size": 256000 +Loading checkpoint shards: 100%|██████████| 3/3 [00:03<00:00, 1.16s/it] +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1. + warnings.warn( +Let split = None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +It is strongly recommended to train Gemma2 models with the `eager` attention implementation instead of `flash_attention_2`. Use `eager` with `AutoModelForCausalLM.from_pretrained('', attn_implementation='eager')`. +------------------------------------------------------------------ +iteration: 1 , TFLOPS: 86.75197547568487, Tokens per sec: 5563.411303067021, Loss: 4.171908378601074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2 , TFLOPS: 66.89933870537911, Tokens per sec: 4290.26007857923, Loss: 4.01677942276001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3 , TFLOPS: 67.16726893555325, Tokens per sec: 4307.442466217215, Loss: 3.9401252269744873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4 , TFLOPS: 67.25290490347041, Tokens per sec: 4312.934307864013, Loss: 3.754024028778076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5 , TFLOPS: 67.291445985822, Tokens per sec: 4315.405950636545, Loss: 3.8183631896972656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6 , TFLOPS: 67.19993814817916, Tokens per sec: 4309.537545502599, Loss: 3.913503408432007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7 , TFLOPS: 67.30122810400093, Tokens per sec: 4316.033278677735, Loss: 3.851064682006836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8 , TFLOPS: 67.16795653479824, Tokens per sec: 4307.486562013197, Loss: 3.6646127700805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9 , TFLOPS: 67.23016958415664, Tokens per sec: 4311.4762899715615, Loss: 3.7966654300689697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10 , TFLOPS: 67.23271391538408, Tokens per sec: 4311.639458141876, Loss: 3.5526936054229736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11 , TFLOPS: 67.17798338980677, Tokens per sec: 4308.129585047344, Loss: 3.6002132892608643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12 , TFLOPS: 67.30360350705875, Tokens per sec: 4316.185613470676, Loss: 3.5705204010009766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13 , TFLOPS: 67.13811947997524, Tokens per sec: 4305.573109240019, Loss: 3.5447990894317627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14 , TFLOPS: 67.15854019228757, Tokens per sec: 4306.882691195614, Loss: 3.450416088104248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15 , TFLOPS: 67.19845754951105, Tokens per sec: 4309.442594588568, Loss: 3.443570613861084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16 , TFLOPS: 67.23455541812397, Tokens per sec: 4311.757553863634, Loss: 3.3366641998291016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17 , TFLOPS: 67.30688001895524, Tokens per sec: 4316.395736447352, Loss: 3.332282066345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18 , TFLOPS: 67.36120902746241, Tokens per sec: 4319.879860219242, Loss: 3.34403395652771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19 , TFLOPS: 67.26840440584516, Tokens per sec: 4313.928292222649, Loss: 3.256293773651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20 , TFLOPS: 67.17348341042366, Tokens per sec: 4307.8410010003945, Loss: 3.3122451305389404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21 , TFLOPS: 67.2001168793811, Tokens per sec: 4309.54900754924, Loss: 3.2204227447509766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22 , TFLOPS: 67.23699865533753, Tokens per sec: 4311.914238866545, Loss: 3.2488620281219482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23 , TFLOPS: 67.2425865851171, Tokens per sec: 4312.272593261658, Loss: 3.163287401199341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 24 , TFLOPS: 67.21941753377986, Tokens per sec: 4310.786760098965, Loss: 3.2160401344299316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 25 , TFLOPS: 67.09871135713247, Tokens per sec: 4303.04586308967, Loss: 3.0935139656066895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 26 , TFLOPS: 67.20080576079224, Tokens per sec: 4309.593185570642, Loss: 3.047175168991089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 27 , TFLOPS: 67.27441115034365, Tokens per sec: 4314.313505240039, Loss: 3.0304696559906006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 28 , TFLOPS: 67.26365793583362, Tokens per sec: 4313.623900711482, Loss: 3.0319135189056396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 29 , TFLOPS: 67.16464708688589, Tokens per sec: 4307.27432684712, Loss: 2.959254264831543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 30 , TFLOPS: 67.3000542568793, Tokens per sec: 4315.957999765541, Loss: 2.913499116897583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 31 , TFLOPS: 67.18211917043104, Tokens per sec: 4308.3948129980145, Loss: 2.940014362335205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 32 , TFLOPS: 67.25841762372463, Tokens per sec: 4313.287839066096, Loss: 2.8469998836517334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 33 , TFLOPS: 67.33731321073192, Tokens per sec: 4318.347419532266, Loss: 2.829812526702881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 34 , TFLOPS: 67.24161982046462, Tokens per sec: 4312.210594565195, Loss: 2.8521993160247803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 35 , TFLOPS: 67.24248740627992, Tokens per sec: 4312.266232914695, Loss: 2.8338708877563477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 36 , TFLOPS: 67.24777489174788, Tokens per sec: 4312.60531979146, Loss: 2.787545680999756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 37 , TFLOPS: 67.30205154448893, Tokens per sec: 4316.086085983773, Loss: 2.81471848487854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 38 , TFLOPS: 67.13737290861587, Tokens per sec: 4305.525231557506, Loss: 2.7764387130737305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 39 , TFLOPS: 67.22735358248879, Tokens per sec: 4311.295699553621, Loss: 2.7642412185668945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 40 , TFLOPS: 67.26715109677696, Tokens per sec: 4313.847917409303, Loss: 2.7132599353790283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 41 , TFLOPS: 67.23918606123682, Tokens per sec: 4312.054517386288, Loss: 2.668989896774292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 42 , TFLOPS: 67.13128246048267, Tokens per sec: 4305.134650619155, Loss: 2.6973328590393066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 43 , TFLOPS: 67.23091373690416, Tokens per sec: 4311.524012548299, Loss: 2.685912609100342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 44 , TFLOPS: 67.27693115124784, Tokens per sec: 4314.475113104727, Loss: 2.662001371383667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 45 , TFLOPS: 67.27965002709941, Tokens per sec: 4314.649474836105, Loss: 2.6665873527526855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 46 , TFLOPS: 67.15514015419501, Tokens per sec: 4306.664646473851, Loss: 2.6501307487487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 47 , TFLOPS: 67.2760527329066, Tokens per sec: 4314.418780064453, Loss: 2.6316823959350586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 48 , TFLOPS: 67.25548187637087, Tokens per sec: 4313.099569347494, Loss: 2.6278648376464844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 49 , TFLOPS: 67.35263957774154, Tokens per sec: 4319.330300705736, Loss: 2.6157166957855225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 50 , TFLOPS: 67.32408825677271, Tokens per sec: 4317.499302150089, Loss: 2.5965774059295654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 51 , TFLOPS: 67.1953666892378, Tokens per sec: 4309.244377465717, Loss: 2.578054904937744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 52 , TFLOPS: 67.25156682148656, Tokens per sec: 4312.848496556634, Loss: 2.5468966960906982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 53 , TFLOPS: 67.32404734871982, Tokens per sec: 4317.496678713301, Loss: 2.53428316116333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 54 , TFLOPS: 67.15867426285547, Tokens per sec: 4306.89128915213, Loss: 2.545722246170044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 55 , TFLOPS: 67.27601676163123, Tokens per sec: 4314.416473223611, Loss: 2.5279200077056885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 56 , TFLOPS: 67.19740155918589, Tokens per sec: 4309.374873842397, Loss: 2.534917116165161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 57 , TFLOPS: 67.2461120484207, Tokens per sec: 4312.498681512492, Loss: 2.5658233165740967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 58 , TFLOPS: 67.2920938769174, Tokens per sec: 4315.447499945635, Loss: 2.5472288131713867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 59 , TFLOPS: 67.27804058384706, Tokens per sec: 4314.546261108317, Loss: 2.4994900226593018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 60 , TFLOPS: 67.28150855801171, Tokens per sec: 4314.768662575956, Loss: 2.502976417541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 61 , TFLOPS: 67.3506410671317, Tokens per sec: 4319.2021360563995, Loss: 2.5281176567077637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 62 , TFLOPS: 67.23894764547772, Tokens per sec: 4312.039227764101, Loss: 2.514285087585449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 63 , TFLOPS: 67.26110814707724, Tokens per sec: 4313.460382549388, Loss: 2.482907772064209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 64 , TFLOPS: 67.16648997644158, Tokens per sec: 4307.39251150549, Loss: 2.4810938835144043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 65 , TFLOPS: 67.13380749324574, Tokens per sec: 4305.2965811773665, Loss: 2.4889049530029297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 66 , TFLOPS: 67.29568135916668, Tokens per sec: 4315.677565476544, Loss: 2.4739832878112793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 67 , TFLOPS: 67.2353824902874, Tokens per sec: 4311.810594069316, Loss: 2.4979248046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 68 , TFLOPS: 67.16737608801321, Tokens per sec: 4307.449337913261, Loss: 2.4705636501312256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 69 , TFLOPS: 67.17368447741053, Tokens per sec: 4307.853895442756, Loss: 2.431494951248169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 70 , TFLOPS: 67.27513003078525, Tokens per sec: 4314.3596071017255, Loss: 2.4638864994049072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 71 , TFLOPS: 67.13314091760232, Tokens per sec: 4305.253833626679, Loss: 2.4194881916046143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 72 , TFLOPS: 67.35945536468331, Tokens per sec: 4319.767397681375, Loss: 2.4741766452789307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 73 , TFLOPS: 67.22132247798172, Tokens per sec: 4310.908924326882, Loss: 2.438474416732788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 74 , TFLOPS: 67.20619442505729, Tokens per sec: 4309.9387610519625, Loss: 2.466714859008789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 75 , TFLOPS: 67.2254479385552, Tokens per sec: 4311.17349045185, Loss: 2.4174747467041016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 76 , TFLOPS: 67.24521841222351, Tokens per sec: 4312.441372549867, Loss: 2.424267053604126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 77 , TFLOPS: 67.22922395995721, Tokens per sec: 4311.415647014088, Loss: 2.404212474822998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 78 , TFLOPS: 67.23452652330809, Tokens per sec: 4311.755700836721, Loss: 2.450658082962036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 79 , TFLOPS: 67.0846114872016, Tokens per sec: 4302.141637274464, Loss: 2.4231417179107666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 80 , TFLOPS: 67.17704276320255, Tokens per sec: 4308.069262586061, Loss: 2.413994312286377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 81 , TFLOPS: 67.2345689529718, Tokens per sec: 4311.758421854535, Loss: 2.4133667945861816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 82 , TFLOPS: 67.18505033340458, Tokens per sec: 4308.582788719936, Loss: 2.389362335205078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 83 , TFLOPS: 67.28162310992364, Tokens per sec: 4314.776008799464, Loss: 2.4374401569366455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 84 , TFLOPS: 67.2334157092426, Tokens per sec: 4311.684464239587, Loss: 2.3909661769866943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 85 , TFLOPS: 67.31368056601009, Tokens per sec: 4316.831856087792, Loss: 2.411787748336792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 86 , TFLOPS: 67.11865914241415, Tokens per sec: 4304.325116195997, Loss: 2.4398515224456787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 87 , TFLOPS: 67.24083693352927, Tokens per sec: 4312.160387961816, Loss: 2.3902275562286377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 88 , TFLOPS: 67.3222851144248, Tokens per sec: 4317.383666483415, Loss: 2.3877973556518555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 89 , TFLOPS: 67.14511488288893, Tokens per sec: 4306.021725002672, Loss: 2.376176357269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 90 , TFLOPS: 67.29125521000229, Tokens per sec: 4315.3937161675785, Loss: 2.3973848819732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 91 , TFLOPS: 67.1356528047859, Tokens per sec: 4305.414921157799, Loss: 2.388991355895996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 92 , TFLOPS: 67.25754211457983, Tokens per sec: 4313.231692592827, Loss: 2.383312463760376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 93 , TFLOPS: 67.15498729921683, Tokens per sec: 4306.654843871562, Loss: 2.3923604488372803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 94 , TFLOPS: 67.32478814446085, Tokens per sec: 4317.544186004938, Loss: 2.3716728687286377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 95 , TFLOPS: 67.3161465459375, Tokens per sec: 4316.989999582809, Loss: 2.405150890350342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 96 , TFLOPS: 67.20162737067454, Tokens per sec: 4309.645875479786, Loss: 2.365361213684082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 97 , TFLOPS: 67.17173577081181, Tokens per sec: 4307.728924728738, Loss: 2.3839645385742188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 98 , TFLOPS: 67.20004987934048, Tokens per sec: 4309.544710831139, Loss: 2.3723373413085938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 99 , TFLOPS: 67.30991336059388, Tokens per sec: 4316.590264895447, Loss: 2.3913819789886475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 100 , TFLOPS: 67.23987549288418, Tokens per sec: 4312.098730694383, Loss: 2.3768458366394043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 101 , TFLOPS: 67.33907694033823, Tokens per sec: 4318.460527656589, Loss: 2.3836305141448975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 102 , TFLOPS: 67.30975607840512, Tokens per sec: 4316.580178375781, Loss: 2.3950178623199463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 103 , TFLOPS: 67.1982354002556, Tokens per sec: 4309.428348138593, Loss: 2.361278772354126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 104 , TFLOPS: 67.20376894334782, Tokens per sec: 4309.783214710986, Loss: 2.3559556007385254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 105 , TFLOPS: 67.23013357946196, Tokens per sec: 4311.47398098754, Loss: 2.349632740020752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 106 , TFLOPS: 67.23129147862021, Tokens per sec: 4311.548237155534, Loss: 2.379448652267456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 107 , TFLOPS: 67.16429762559119, Tokens per sec: 4307.251915865627, Loss: 2.4072415828704834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 108 , TFLOPS: 67.26025670890765, Tokens per sec: 4313.405779749734, Loss: 2.3945987224578857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 109 , TFLOPS: 67.13558642664209, Tokens per sec: 4305.410664321992, Loss: 2.3535115718841553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 110 , TFLOPS: 67.27982379366702, Tokens per sec: 4314.660618500338, Loss: 2.3627665042877197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 111 , TFLOPS: 67.26391532288811, Tokens per sec: 4313.640406964398, Loss: 2.3859591484069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 112 , TFLOPS: 67.27053505647855, Tokens per sec: 4314.064931022535, Loss: 2.3465442657470703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 113 , TFLOPS: 67.22654753561278, Tokens per sec: 4311.244007701346, Loss: 2.396284818649292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 114 , TFLOPS: 67.12289176484347, Tokens per sec: 4304.596554619569, Loss: 2.3716585636138916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 115 , TFLOPS: 67.13262769476694, Tokens per sec: 4305.220920604149, Loss: 2.3369154930114746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 116 , TFLOPS: 67.17146478693049, Tokens per sec: 4307.711546510201, Loss: 2.302396535873413 +------------------------------------------------------------------ +Traceback (most recent call last): + File "/project/examples/finetuning.py", line 13, in + main() + File "/project/src/llama_recipes/finetuning.py", line 281, in main + train( + File "/project/src/llama_recipes/utils/train_utils.py", line 118, in train + loss.backward() + File "/usr/local/lib/python3.10/dist-packages/torch/_tensor.py", line 522, in backward + torch.autograd.backward( + File "/usr/local/lib/python3.10/dist-packages/torch/autograd/__init__.py", line 267, in backward + _engine_run_backward( + File "/usr/local/lib/python3.10/dist-packages/torch/autograd/graph.py", line 681, in _engine_run_backward + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +KeyboardInterrupt \ No newline at end of file diff --git a/wandb/run-20240804_222226-kh5katc1/files/requirements.txt b/wandb/run-20240804_222226-kh5katc1/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7c90fd1c54d0a9881f6b4c6465b2a4fa88c9056c --- /dev/null +++ b/wandb/run-20240804_222226-kh5katc1/files/requirements.txt @@ -0,0 +1,271 @@ +absl-py==2.1.0 +accelerate==0.33.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bleach==6.1.0 +blis==0.7.11 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +distributed==2023.11.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +einops==0.7.0 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +grpcio==1.60.1 +huggingface-hub==0.24.5 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +jedi==0.19.1 +jinja2==3.1.3 +joblib==1.3.2 +json5==0.9.14 +jsonnet==0.19.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +kiwisolver==1.4.5 +langcodes==3.3.0 +lazy-loader==0.3 +librosa==0.10.1 +llvmlite==0.40.1 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +murmurhash==1.0.10 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +onnx==1.15.0rc2 +opencv==4.7.0 +optree==0.10.0 +packaging==23.2 +pandas==1.5.3 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +peft==0.11.1 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==14.0.1.dev0+gba5374836.d20240125 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pynvml==11.4.1 +pyparsing==3.1.1 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests==2.31.0 +rich==13.7.0 +rmm==23.12.0 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.0 +safetensors==0.4.3 +scikit-learn==1.2.0 +scipy==1.12.0 +send2trash==1.8.2 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +srsly==2.4.8 +stack-data==0.6.3 +sympy==1.12 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm==4.66.1 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.9.0 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +urllib3==1.26.18 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +xdoctest==1.0.2 +xgboost==1.7.6 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 \ No newline at end of file diff --git a/wandb/run-20240804_222226-kh5katc1/files/wandb-metadata.json b/wandb/run-20240804_222226-kh5katc1/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..2bedf2b72e55da243beb0015962302e30cce6c65 --- /dev/null +++ b/wandb/run-20240804_222226-kh5katc1/files/wandb-metadata.json @@ -0,0 +1,215 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-04T13:22:26.872566", + "startedAt": "2024-08-04T13:22:26.250232", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "4096", + "--sliding-window-size", + "4096", + "--micro-batch-size", + "1", + "--global-batch-size", + "320", + "--train-iters", + "20000", + "--tokenizer-type", + "HFPreTrainedTokenizer", + "--tokenizer-model", + "/share/pretrained_lm/google/gemma-2-2b", + "--train-data-path", + "235289369", + "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document", + "--valid-data-path", + "235289369", + "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document", + "--test-data-path", + "235289369", + "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document", + "--lr", + "2e-5", + "--min-lr", + "1e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "20000", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "anyprecision", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-6", + "--save-interval", + "200", + "--eval-interval", + "200", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/google/gemma-2-2b", + "--save", + "/work/llm_recipes/models/yans-sample-gemma-2-2b", + "--load", + "/work/llm_recipes/models/yans-sample-gemma-2-2b", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "FULL_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/yans-sample-gemma-2-2b", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "llm_tutorial", + "--wandb-name", + "yans-sample-gemma-2-2b_train_2024-08-04-22:22:15" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "0336bd6c20fe25d78eda1d14afa66c1ae2e6d687" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.044999999999, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.045, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.48782730102539 + } +} diff --git a/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json b/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..284773fc1cdc6a0b7620f94e6fea33675636f9db --- /dev/null +++ b/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json @@ -0,0 +1 @@ +{"training/loss": 2.302396535873413, "training/perplexity": 9.99811460655144, "utils/batch_size": 1, "utils/global_batch_size": 320, "utils/seq_len": 4097, "utils/gradient_accumulation_steps": 320, "utils/iteration": 116, "optimizer/lr": 5.4080000000000006e-06, "optimizer/variance_l2": 0.0030219239359304895, "optimizer/variance_sqrt_l2": 0.8405880490942215, "optimizer/momentum_l2": 0.36270596473675665, "optimizer/weight_l2": 1167.8420269882395, "optimizer/variance_l1": 0.70648193359375, "optimizer/variance_sqrt_l1": 19948.0, "optimizer/momentum_l1": 5862.0, "optimizer/weight_l1": 29775872.0, "optimizer/variance_abs_max": 0.001068115234375, "optimizer/variance_sqrt_abs_max": 0.03271484375, "optimizer/momentum_abs_max": 0.0250244140625, "optimizer/weight_abs_max": 12.9375, "stats/1_iteration_time": 304.34721216700564, "stats/tokens_per_sec": 4307.711546510201, "stats/tokens_per_sec_per_gpu": 4307.711546510201, "stats/tflops": 67.17146478693049, "_timestamp": 1722812960.6351748, "_runtime": 35214.36805868149, "_step": 116, "_wandb": {"runtime": 35371}} \ No newline at end of file diff --git a/wandb/run-20240804_222226-kh5katc1/logs/debug-internal.log b/wandb/run-20240804_222226-kh5katc1/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a6f48da10206da3effdfd4ce9c8a9cfc7c12ccde --- /dev/null +++ b/wandb/run-20240804_222226-kh5katc1/logs/debug-internal.log @@ -0,0 +1,16462 @@ +2024-08-04 22:22:26,287 INFO StreamThr :12967 [internal.py:wandb_internal():86] W&B internal server running at pid: 12967, started at: 2024-08-04 22:22:26.267702 +2024-08-04 22:22:26,289 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status +2024-08-04 22:22:26,291 INFO WriterThread:12967 [datastore.py:open_for_write():87] open: /project/wandb/run-20240804_222226-kh5katc1/run-kh5katc1.wandb +2024-08-04 22:22:26,308 DEBUG SenderThread:12967 [sender.py:send():382] send: header +2024-08-04 22:22:26,308 DEBUG SenderThread:12967 [sender.py:send():382] send: run +2024-08-04 22:22:26,761 INFO SenderThread:12967 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240804_222226-kh5katc1/files +2024-08-04 22:22:26,761 INFO SenderThread:12967 [sender.py:_start_run_threads():1136] run started: kh5katc1 with start time 1722777746.267116 +2024-08-04 22:22:26,767 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: check_version +2024-08-04 22:22:26,767 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: check_version +2024-08-04 22:22:26,854 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: run_start +2024-08-04 22:22:26,860 DEBUG HandlerThread:12967 [system_info.py:__init__():27] System info init +2024-08-04 22:22:26,860 DEBUG HandlerThread:12967 [system_info.py:__init__():42] System info init done +2024-08-04 22:22:26,860 INFO HandlerThread:12967 [system_monitor.py:start():194] Starting system monitor +2024-08-04 22:22:26,860 INFO SystemMonitor:12967 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-04 22:22:26,860 INFO HandlerThread:12967 [system_monitor.py:probe():214] Collecting system info +2024-08-04 22:22:26,861 INFO SystemMonitor:12967 [interfaces.py:start():190] Started cpu monitoring +2024-08-04 22:22:26,861 INFO SystemMonitor:12967 [interfaces.py:start():190] Started disk monitoring +2024-08-04 22:22:26,862 INFO SystemMonitor:12967 [interfaces.py:start():190] Started gpu monitoring +2024-08-04 22:22:26,863 INFO SystemMonitor:12967 [interfaces.py:start():190] Started memory monitoring +2024-08-04 22:22:26,863 INFO SystemMonitor:12967 [interfaces.py:start():190] Started network monitoring +2024-08-04 22:22:26,872 DEBUG HandlerThread:12967 [system_info.py:probe():151] Probing system +2024-08-04 22:22:26,874 DEBUG HandlerThread:12967 [system_info.py:_probe_git():136] Probing git +2024-08-04 22:22:26,886 DEBUG HandlerThread:12967 [system_info.py:_probe_git():144] Probing git done +2024-08-04 22:22:26,886 DEBUG HandlerThread:12967 [system_info.py:probe():199] Probing system done +2024-08-04 22:22:26,886 DEBUG HandlerThread:12967 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-04T13:22:26.872566', 'startedAt': '2024-08-04T13:22:26.250232', 'docker': None, 'cuda': None, 'args': ('--seq-length', '4096', '--sliding-window-size', '4096', '--micro-batch-size', '1', '--global-batch-size', '320', '--train-iters', '20000', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/google/gemma-2-2b', '--train-data-path', '235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document', '--valid-data-path', '235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document', '--test-data-path', '235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '20000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'anyprecision', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '200', '--eval-interval', '200', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/google/gemma-2-2b', '--save', '/work/llm_recipes/models/yans-sample-gemma-2-2b', '--load', '/work/llm_recipes/models/yans-sample-gemma-2-2b', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/yans-sample-gemma-2-2b', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial', '--wandb-name', 'yans-sample-gemma-2-2b_train_2024-08-04-22:22:15'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '0336bd6c20fe25d78eda1d14afa66c1ae2e6d687'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.044999999999, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.48782730102539}} +2024-08-04 22:22:26,886 INFO HandlerThread:12967 [system_monitor.py:probe():224] Finished collecting system info +2024-08-04 22:22:26,886 INFO HandlerThread:12967 [system_monitor.py:probe():227] Publishing system info +2024-08-04 22:22:26,888 INFO HandlerThread:12967 [system_monitor.py:probe():229] Finished publishing system info +2024-08-04 22:22:26,894 DEBUG SenderThread:12967 [sender.py:send():382] send: files +2024-08-04 22:22:26,894 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-04 22:22:26,903 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: python_packages +2024-08-04 22:22:26,903 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:22:26,903 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: python_packages +2024-08-04 22:22:26,904 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:22:26,905 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:22:27,136 DEBUG SenderThread:12967 [sender.py:send():382] send: telemetry +2024-08-04 22:22:27,646 INFO wandb-upload_0:12967 [upload_job.py:push():131] Uploaded file /tmp/tmp7lhf4igewandb/olu3lfxp-wandb-metadata.json +2024-08-04 22:22:27,763 INFO Thread-12 :12967 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_222226-kh5katc1/files/requirements.txt +2024-08-04 22:22:27,764 INFO Thread-12 :12967 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-metadata.json +2024-08-04 22:22:27,764 INFO Thread-12 :12967 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-04 22:22:29,764 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-04 22:22:31,396 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:22:32,203 DEBUG SenderThread:12967 [sender.py:send():382] send: config +2024-08-04 22:22:32,203 DEBUG SenderThread:12967 [sender.py:send():382] send: config +2024-08-04 22:22:32,766 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-04 22:22:33,766 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-04 22:22:37,287 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:22:41,903 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:22:41,903 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:22:41,904 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:22:43,112 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:22:48,113 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:22:53,113 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:22:56,902 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:22:56,902 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:22:56,944 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:22:59,083 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:22:59,780 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/config.yaml +2024-08-04 22:23:04,270 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:23:09,270 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:23:11,902 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:23:11,902 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:23:11,944 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:23:15,082 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:23:20,083 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:23:25,083 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:23:26,864 DEBUG SystemMonitor:12967 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-04 22:23:26,866 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:23:26,902 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:23:26,902 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:23:26,944 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:23:30,115 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:23:35,116 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:23:40,116 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:23:41,902 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:23:41,903 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:23:41,944 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:23:45,129 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:23:50,130 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:23:55,130 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:23:56,867 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:23:56,902 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:23:56,903 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:23:56,944 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:24:01,123 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:24:06,123 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:24:11,124 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:24:11,902 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:24:11,903 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:24:11,944 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:24:17,113 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:24:22,114 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:24:26,868 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:24:26,903 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:24:26,903 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:24:26,944 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:24:27,166 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:24:32,167 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:24:37,167 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:24:41,903 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:24:41,903 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:24:41,944 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:24:43,114 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:24:48,114 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:24:53,115 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:24:56,869 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:24:56,903 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:24:56,903 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:24:56,944 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:24:58,148 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:25:03,149 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:25:08,149 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:25:11,903 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:25:11,903 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:25:11,944 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:25:14,132 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:25:19,133 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:25:24,133 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:25:26,870 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:25:26,903 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:25:26,903 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:25:26,944 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:25:29,181 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:25:34,182 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:25:39,182 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:25:41,903 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:25:41,903 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:25:41,944 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:25:45,079 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:25:50,079 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:25:55,080 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:25:56,871 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:25:56,903 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:25:56,903 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:25:56,944 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:26:00,103 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:26:05,104 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:26:10,104 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:26:11,903 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:26:11,903 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:26:11,944 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:26:15,129 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:26:20,129 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:26:25,130 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:26:26,872 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:26:26,903 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:26:26,904 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:26:26,944 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:26:30,138 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:26:30,492 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-04 22:26:31,890 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-04 22:26:35,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:26:40,535 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:26:41,903 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:26:41,904 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:26:41,904 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:26:46,085 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:26:51,085 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:26:56,086 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:26:56,873 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:26:56,903 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:26:56,904 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:26:56,944 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:27:02,084 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:27:07,085 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:27:11,904 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:27:11,904 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:27:11,944 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:27:12,168 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:27:17,169 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:27:22,169 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:27:26,874 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:27:26,950 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:27:26,951 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:27:26,992 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:27:27,177 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:27:32,178 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:27:37,178 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:27:41,950 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:27:41,951 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:27:41,951 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:27:43,137 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:27:48,138 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:27:53,139 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:27:56,875 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:27:56,950 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:27:56,950 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:27:56,950 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:27:58,176 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:28:03,176 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:28:08,177 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:28:11,950 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:28:11,950 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:28:11,992 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:28:13,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:28:18,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:28:23,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:28:26,876 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:28:26,950 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:28:26,950 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:28:26,951 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:28:29,206 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:28:34,207 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:28:39,207 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:28:41,950 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:28:41,951 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:28:41,951 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:28:45,137 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:28:50,138 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:28:55,138 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:28:56,876 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:28:56,950 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:28:56,951 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:28:56,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:29:00,154 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:29:05,154 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:29:10,155 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:29:11,951 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:29:11,951 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:29:11,951 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:29:15,157 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:29:20,158 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:29:25,159 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:29:26,879 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:29:26,951 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:29:26,951 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:29:26,992 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:29:31,137 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:29:36,138 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:29:41,138 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:29:41,951 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:29:41,951 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:29:41,992 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:29:46,211 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:29:51,211 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:29:56,212 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:29:56,878 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:29:56,951 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:29:56,951 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:29:56,992 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:30:02,156 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:30:07,157 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:30:11,951 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:30:11,951 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:30:11,992 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:30:13,136 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:30:18,137 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:30:23,137 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:30:26,880 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:30:26,951 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:30:26,951 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:30:26,992 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:30:28,174 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:30:33,175 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:30:38,175 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:30:41,951 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:30:41,951 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:30:41,992 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:30:43,597 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:30:48,598 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:30:53,599 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:30:56,880 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:30:56,951 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:30:56,951 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:30:56,992 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:30:59,220 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:31:04,220 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:31:09,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:31:11,952 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:31:11,952 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:31:11,952 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:31:15,196 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:31:20,197 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:31:25,197 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:31:26,881 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:31:26,952 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:31:26,952 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:31:26,952 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:31:31,134 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:31:36,080 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-04 22:31:36,082 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-04 22:31:36,083 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-04 22:31:36,084 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 22:31:37,055 INFO Thread-12 :12967 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-04 22:31:37,121 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:31:39,056 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-04 22:31:41,951 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:31:41,952 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:31:41,952 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:31:42,218 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:31:47,218 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:31:52,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:31:56,882 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:31:56,952 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:31:56,952 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:31:56,992 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:31:58,149 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:32:03,150 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:32:08,150 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:32:11,952 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:32:11,952 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:32:11,952 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:32:13,177 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:32:18,178 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:32:23,179 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:32:26,883 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:32:26,952 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:32:26,952 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:32:26,952 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:32:28,213 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:32:33,213 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:32:38,214 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:32:41,952 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:32:41,952 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:32:41,953 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:32:44,174 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:32:49,175 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:32:54,176 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:32:56,884 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:32:56,953 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:32:56,953 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:32:56,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:33:00,122 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:33:05,123 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:33:10,124 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:33:11,952 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:33:11,953 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:33:11,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:33:15,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:33:20,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:33:25,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:33:26,885 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:33:26,952 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:33:26,952 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:33:26,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:33:31,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:33:36,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:33:41,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:33:41,952 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:33:41,953 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:33:41,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:33:47,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:33:52,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:33:56,886 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:33:56,952 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:33:56,953 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:33:56,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:33:58,208 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:34:03,208 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:34:08,209 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:34:11,953 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:34:11,954 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:34:11,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:34:14,181 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:34:19,182 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:34:24,182 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:34:26,887 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:34:26,953 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:34:26,953 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:34:26,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:34:29,205 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:34:34,205 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:34:39,206 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:34:41,953 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:34:41,953 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:34:41,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:34:45,184 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:34:50,184 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:34:55,185 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:34:56,888 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:34:56,953 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:34:56,953 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:34:56,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:35:00,193 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:35:05,193 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:35:10,194 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:35:11,953 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:35:11,953 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:35:11,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:35:16,175 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:35:21,175 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:35:26,176 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:35:26,889 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:35:26,953 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:35:26,953 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:35:26,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:35:32,132 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:35:37,133 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:35:41,953 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:35:41,954 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:35:41,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:35:42,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:35:47,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:35:52,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:35:56,890 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:35:56,954 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:35:56,954 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:35:56,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:35:58,164 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:36:03,164 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:36:08,165 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:36:11,954 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:36:11,954 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:36:11,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:36:13,209 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:36:18,209 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:36:23,210 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:36:26,891 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:36:26,954 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:36:26,954 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:36:26,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:36:28,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:36:33,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:36:38,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:36:40,448 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-04 22:36:40,449 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-04 22:36:40,450 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-04 22:36:40,451 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 22:36:41,225 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-04 22:36:41,954 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:36:41,954 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:36:41,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:36:43,226 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-04 22:36:44,204 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:36:49,205 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:36:54,205 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:36:56,892 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:36:56,954 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:36:56,954 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:36:56,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:36:59,205 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:37:04,206 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:37:09,207 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:37:11,954 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:37:11,955 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:37:11,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:37:14,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:37:19,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:37:24,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:37:26,893 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:37:26,954 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:37:26,954 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:37:26,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:37:30,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:37:35,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:37:40,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:37:41,954 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:37:41,955 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:37:41,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:37:46,168 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:37:51,168 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:37:56,170 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:37:56,894 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:37:56,955 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:37:56,955 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:37:56,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:38:01,228 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:38:06,229 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:38:11,229 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:38:11,955 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:38:11,955 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:38:11,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:38:17,172 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:38:22,173 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:38:26,894 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:38:26,955 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:38:26,956 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:38:26,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:38:27,244 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:38:32,245 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:38:37,246 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:38:41,955 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:38:41,955 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:38:41,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:38:43,188 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:38:48,189 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:38:53,190 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:38:56,895 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:38:56,955 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:38:56,955 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:38:56,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:38:58,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:39:03,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:39:08,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:39:11,955 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:39:11,955 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:39:11,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:39:14,199 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:39:19,200 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:39:24,200 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:39:26,896 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:39:26,955 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:39:26,955 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:39:26,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:39:29,211 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:39:34,211 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:39:39,212 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:39:41,955 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:39:41,956 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:39:41,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:39:45,199 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:39:50,199 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:39:55,200 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:39:56,897 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:39:56,955 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:39:56,956 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:39:56,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:40:01,146 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:40:06,146 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:40:11,147 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:40:11,956 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:40:11,956 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:40:12,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:40:16,162 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:40:21,162 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:40:26,163 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:40:26,898 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:40:26,956 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:40:26,956 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:40:26,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:40:32,139 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:40:37,139 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:40:41,956 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:40:41,956 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:40:41,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:40:42,228 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:40:47,228 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:40:52,229 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:40:56,900 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:40:56,956 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:40:56,956 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:40:56,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:40:58,184 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:41:03,185 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:41:08,185 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:41:11,956 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:41:11,956 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:41:11,996 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:41:14,160 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:41:19,160 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:41:24,161 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:41:26,900 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:41:26,956 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:41:26,956 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:41:27,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:41:30,142 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:41:35,142 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:41:40,143 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:41:41,956 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:41:41,956 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:41:42,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:41:44,428 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-04 22:41:44,430 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-04 22:41:44,430 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-04 22:41:44,432 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 22:41:45,389 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-04 22:41:45,473 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:41:47,390 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-04 22:41:50,474 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:41:55,474 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:41:56,901 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:41:56,956 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:41:56,956 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:41:57,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:42:01,170 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:42:06,171 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:42:11,171 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:42:11,956 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:42:11,957 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:42:12,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:42:16,190 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:42:21,191 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:42:26,191 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:42:26,903 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:42:26,956 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:42:26,956 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:42:27,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:42:32,177 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:42:37,178 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:42:41,957 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:42:41,957 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:42:42,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:42:43,167 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:42:48,167 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:42:53,168 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:42:56,903 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:42:56,957 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:42:56,957 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:42:57,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:42:59,159 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:43:04,160 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:43:09,160 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:43:11,957 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:43:11,957 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:43:12,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:43:14,161 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:43:19,161 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:43:24,162 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:43:26,904 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:43:26,957 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:43:26,957 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:43:27,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:43:29,205 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:43:34,206 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:43:39,207 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:43:41,957 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:43:41,957 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:43:42,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:43:45,153 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:43:50,154 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:43:55,155 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:43:56,905 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:43:56,957 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:43:56,957 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:43:57,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:44:01,133 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:44:06,133 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:44:11,134 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:44:11,957 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:44:11,957 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:44:12,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:44:16,184 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:44:21,184 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:44:26,185 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:44:26,905 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:44:26,957 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:44:26,957 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:44:27,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:44:32,145 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:44:37,146 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:44:41,958 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:44:41,958 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:44:42,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:44:42,293 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:44:47,293 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:44:52,294 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:44:56,906 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:44:56,958 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:44:56,958 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:44:57,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:44:58,190 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:45:03,190 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:45:08,191 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:45:11,958 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:45:11,958 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:45:12,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:45:14,187 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:45:19,187 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:45:24,188 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:45:26,907 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:45:26,958 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:45:26,958 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:45:27,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:45:30,175 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:45:35,176 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:45:40,177 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:45:41,958 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:45:41,958 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:45:42,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:45:45,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:45:50,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:45:55,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:45:56,908 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:45:56,958 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:45:56,958 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:45:57,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:46:01,179 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:46:06,180 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:46:11,181 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:46:11,958 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:46:11,958 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:46:12,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:46:17,167 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:46:22,167 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:46:26,909 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:46:26,958 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:46:26,958 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:46:27,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:46:28,308 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:46:33,308 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:46:38,309 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:46:41,958 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:46:41,959 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:46:42,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:46:44,181 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:46:48,235 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-04 22:46:48,237 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-04 22:46:48,237 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-04 22:46:48,238 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 22:46:48,560 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-04 22:46:49,277 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:46:49,561 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-04 22:46:54,278 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:46:56,910 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:46:56,958 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:46:56,959 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:46:57,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:47:00,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:47:05,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:47:10,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:47:11,958 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:47:11,959 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:47:12,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:47:15,229 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:47:20,230 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:47:25,231 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:47:26,911 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:47:26,959 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:47:26,959 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:47:27,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:47:31,166 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:47:36,167 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:47:41,168 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:47:41,959 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:47:41,959 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:47:42,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:47:47,144 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:47:52,145 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:47:56,912 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:47:56,959 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:47:56,959 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:47:57,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:47:57,148 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:48:02,149 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:48:07,149 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:48:11,959 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:48:11,959 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:48:12,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:48:13,136 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:48:18,137 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:48:23,138 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:48:26,913 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:48:26,959 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:48:26,959 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:48:27,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:48:28,171 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:48:33,172 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:48:38,173 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:48:41,959 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:48:41,959 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:48:42,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:48:43,179 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:48:48,180 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:48:53,181 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:48:56,914 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:48:56,959 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:48:56,959 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:48:57,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:48:59,133 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:49:04,134 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:49:09,134 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:49:11,959 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:49:11,959 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:49:12,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:49:14,203 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:49:19,204 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:49:24,204 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:49:26,937 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:49:26,959 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:49:26,959 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:49:27,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:49:30,143 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:49:35,144 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:49:40,144 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:49:41,959 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:49:41,960 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:49:42,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:49:45,171 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:49:50,171 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:49:55,172 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:49:56,938 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:49:56,959 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:49:56,960 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:49:57,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:50:00,172 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:50:05,173 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:50:10,174 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:50:11,960 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:50:11,960 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:50:12,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:50:15,234 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:50:20,234 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:50:25,235 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:50:26,939 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:50:26,960 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:50:26,960 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:50:27,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:50:31,173 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:50:36,174 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:50:41,175 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:50:41,960 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:50:41,960 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:50:42,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:50:46,186 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:50:51,187 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:50:56,188 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:50:56,940 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:50:56,960 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:50:56,960 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:50:57,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:51:02,179 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:51:07,179 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:51:11,960 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:51:11,960 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:51:12,000 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:51:12,204 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:51:17,205 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:51:22,213 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:51:26,941 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:51:26,960 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:51:26,960 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:51:27,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:51:28,180 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:51:33,181 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:51:38,181 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:51:41,960 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:51:41,960 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:51:42,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:51:44,149 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:51:49,150 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:51:52,456 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-04 22:51:52,458 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-04 22:51:52,458 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-04 22:51:52,459 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 22:51:52,731 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-04 22:51:53,732 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-04 22:51:54,498 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:51:56,943 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:51:56,960 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:51:56,960 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:51:57,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:52:00,179 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:52:05,189 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:52:10,189 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:52:11,960 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:52:11,961 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:52:12,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:52:15,203 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:52:20,204 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:52:25,204 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:52:26,943 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:52:26,960 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:52:26,961 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:52:27,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:52:30,340 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:52:35,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:52:40,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:52:41,961 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:52:41,961 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:52:42,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:52:46,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:52:51,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:52:56,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:52:56,944 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:52:56,961 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:52:56,961 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:52:57,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:53:01,232 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:53:06,233 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:53:11,233 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:53:11,961 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:53:11,961 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:53:12,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:53:17,178 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:53:22,179 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:53:26,945 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:53:26,961 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:53:26,961 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:53:27,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:53:27,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:53:32,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:53:37,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:53:41,961 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:53:41,961 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:53:42,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:53:43,199 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:53:48,199 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:53:53,200 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:53:56,946 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:53:56,961 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:53:56,961 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:53:57,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:53:59,147 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:54:04,148 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:54:09,148 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:54:11,961 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:54:11,961 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:54:12,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:54:14,153 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:54:19,154 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:54:24,154 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:54:26,947 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:54:26,961 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:54:26,961 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:54:27,008 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:54:29,233 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:54:34,233 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:54:39,234 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:54:41,961 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:54:41,961 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:54:42,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:54:45,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:54:50,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:54:55,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:54:56,948 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:54:56,961 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:54:56,962 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:54:57,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:55:01,204 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:55:06,204 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:55:11,205 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:55:11,961 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:55:11,962 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:55:12,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:55:17,173 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:55:22,174 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:55:26,949 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:55:26,961 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:55:26,962 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:55:27,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:55:27,175 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:55:32,176 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:55:37,176 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:55:41,962 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:55:41,962 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:55:42,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:55:42,191 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:55:47,192 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:55:52,192 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:55:56,950 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:55:56,962 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:55:56,962 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:55:57,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:55:57,228 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:56:02,229 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:56:07,230 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:56:11,962 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:56:11,962 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:56:12,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:56:13,212 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:56:18,212 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:56:23,213 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:56:26,951 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:56:26,962 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:56:26,962 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:56:27,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:56:29,170 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:56:34,171 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:56:39,172 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:56:41,962 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:56:41,962 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:56:42,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:56:44,195 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:56:49,196 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:56:54,196 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:56:56,218 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-04 22:56:56,220 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-04 22:56:56,220 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-04 22:56:56,221 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 22:56:56,899 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-04 22:56:56,951 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:56:56,962 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:56:56,962 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:56:57,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:56:57,899 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-04 22:57:00,188 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:57:05,189 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:57:10,189 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:57:11,962 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:57:11,963 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:57:12,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:57:16,172 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:57:21,172 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:57:26,173 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:57:26,952 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:57:26,962 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:57:26,963 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:57:27,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:57:31,206 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:57:36,207 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:57:41,207 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:57:41,962 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:57:41,963 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:57:42,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:57:46,218 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:57:51,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:57:56,220 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:57:56,953 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:57:56,963 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:57:56,963 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:57:57,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:58:02,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:58:07,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:58:11,963 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:58:11,963 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:58:12,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:58:13,210 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:58:18,211 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:58:23,211 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:58:26,954 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:58:26,963 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:58:26,963 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:58:27,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:58:29,200 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:58:34,201 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:58:39,201 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:58:41,963 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:58:41,963 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:58:42,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:58:44,204 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:58:49,204 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:58:54,205 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:58:56,955 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:58:56,963 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:58:56,963 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:58:57,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:59:00,198 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:59:05,199 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:59:10,199 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:59:11,963 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:59:11,963 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:59:12,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:59:16,130 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:59:21,131 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:59:26,131 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:59:26,956 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:59:26,963 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:59:26,963 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:59:27,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:59:31,168 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:59:36,169 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:59:41,169 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:59:41,963 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:59:41,963 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:59:42,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 22:59:46,236 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:59:51,237 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:59:56,237 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 22:59:56,957 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 22:59:56,963 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 22:59:56,963 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 22:59:57,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:00:02,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:00:07,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:00:11,963 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:00:11,963 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:00:12,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:00:13,202 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:00:18,203 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:00:23,203 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:00:26,958 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:00:26,963 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:00:26,964 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:00:27,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:00:29,177 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:00:34,177 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:00:39,178 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:00:41,964 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:00:41,964 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:00:42,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:00:44,220 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:00:49,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:00:54,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:00:56,959 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:00:56,964 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:00:56,964 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:00:57,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:00:59,241 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:01:04,241 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:01:09,242 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:01:11,964 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:01:11,964 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:01:12,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:01:15,193 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:01:20,194 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:01:25,195 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:01:26,960 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:01:26,964 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:01:26,964 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:01:27,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:01:31,161 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:01:36,161 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:01:41,162 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:01:41,964 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:01:41,964 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:01:42,004 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:01:46,184 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:01:51,184 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:01:56,185 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:01:56,961 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:01:56,966 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:01:56,966 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:01:56,967 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:02:00,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-04 23:02:00,585 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-04 23:02:00,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-04 23:02:00,587 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 23:02:01,072 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-04 23:02:01,626 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:02:02,073 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-04 23:02:06,627 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:02:11,628 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:02:11,966 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:02:11,966 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:02:11,966 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:02:17,228 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:02:22,228 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:02:26,962 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:02:26,966 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:02:26,966 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:02:26,967 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:02:28,179 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:02:33,180 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:02:38,180 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:02:41,966 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:02:41,967 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:02:41,967 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:02:43,218 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:02:48,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:02:53,220 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:02:56,964 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:02:56,967 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:02:56,968 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:02:56,968 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:02:58,227 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:03:03,228 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:03:08,228 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:03:11,966 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:03:11,967 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:03:11,967 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:03:14,213 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:03:19,214 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:03:24,214 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:03:26,964 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:03:26,966 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:03:26,967 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:03:26,967 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:03:29,240 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:03:34,241 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:03:39,242 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:03:41,966 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:03:41,967 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:03:41,967 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:03:45,200 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:03:50,200 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:03:55,201 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:03:56,965 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:03:56,967 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:03:56,967 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:03:57,008 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:04:00,220 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:04:05,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:04:10,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:04:11,967 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:04:11,967 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:04:11,968 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:04:16,177 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:04:21,178 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:04:26,178 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:04:26,966 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:04:26,967 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:04:26,968 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:04:26,969 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:04:32,162 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:04:37,163 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:04:41,967 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:04:41,967 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:04:41,968 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:04:42,198 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:04:47,198 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:04:52,199 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:04:56,967 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:04:56,968 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:04:56,968 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:04:56,969 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:04:57,235 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:05:02,236 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:05:07,237 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:05:11,967 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:05:11,968 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:05:11,968 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:05:13,199 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:05:18,200 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:05:23,201 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:05:26,968 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:05:26,968 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:05:26,968 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:05:26,970 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:05:29,151 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:05:34,151 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:05:39,152 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:05:41,967 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:05:41,968 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:05:41,968 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:05:44,235 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:05:49,235 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:05:54,236 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:05:56,968 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:05:56,969 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:05:56,969 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:05:56,970 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:05:59,248 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:06:04,249 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:06:09,250 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:06:11,967 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:06:11,968 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:06:11,969 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:06:15,167 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:06:20,168 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:06:25,168 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:06:26,967 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:06:26,969 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:06:26,969 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:06:26,970 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:06:31,164 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:06:36,164 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:06:41,164 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:06:41,968 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:06:41,969 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:06:41,969 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:06:46,238 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:06:51,238 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:06:56,239 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:06:56,968 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:06:56,969 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:06:56,969 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:06:57,189 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:07:02,190 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:07:04,667 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-04 23:07:04,669 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-04 23:07:04,670 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-04 23:07:04,672 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 23:07:05,256 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-04 23:07:07,258 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-04 23:07:07,672 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:07:11,968 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:07:11,969 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:07:11,969 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:07:13,238 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:07:18,239 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:07:23,239 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:07:26,968 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:07:26,969 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:07:26,969 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:07:27,182 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:07:29,183 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:07:34,183 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:07:39,184 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:07:41,968 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:07:41,969 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:07:41,969 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:07:44,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:07:49,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:07:54,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:07:56,968 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:07:56,969 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:07:56,969 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:07:57,145 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:08:00,146 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:08:05,147 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:08:10,147 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:08:11,968 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:08:11,969 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:08:11,969 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:08:15,235 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:08:20,236 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:08:25,236 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:08:26,968 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:08:26,969 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:08:26,969 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:08:27,169 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:08:31,171 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:08:36,171 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:08:41,172 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:08:41,968 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:08:41,969 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:08:41,970 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:08:46,196 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:08:51,197 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:08:56,197 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:08:56,969 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:08:56,969 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:08:56,970 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:08:57,200 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:09:01,201 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:09:06,202 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:09:11,202 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:09:11,968 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:09:11,970 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:09:11,970 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:09:16,238 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:09:21,238 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:09:26,239 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:09:26,969 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:09:26,970 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:09:26,970 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:09:27,192 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:09:32,193 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:09:37,193 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:09:41,969 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:09:41,970 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:09:41,970 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:09:43,183 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:09:48,184 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:09:53,185 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:09:56,969 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:09:56,970 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:09:56,970 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:09:57,182 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:09:59,182 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:10:04,183 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:10:09,184 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:10:11,969 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:10:11,970 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:10:11,970 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:10:15,143 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:10:20,144 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:10:25,145 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:10:26,969 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:10:26,970 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:10:26,971 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:10:27,203 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:10:30,203 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:10:35,204 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:10:40,205 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:10:41,969 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:10:41,970 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:10:41,971 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:10:45,208 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:10:50,209 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:10:55,209 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:10:56,969 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:10:56,970 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:10:56,971 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:10:57,221 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:11:00,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:11:05,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:11:10,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:11:11,969 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:11:11,970 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:11:11,971 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:11:15,227 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:11:20,228 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:11:25,228 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:11:26,969 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:11:26,971 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:11:26,971 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:11:27,143 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:11:31,144 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:11:36,145 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:11:41,145 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:11:41,970 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:11:41,971 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:11:41,971 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:11:46,168 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:11:51,169 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:11:56,169 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:11:56,970 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:11:56,971 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:11:56,971 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:11:57,215 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:12:01,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:12:06,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:12:08,740 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-04 23:12:08,742 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-04 23:12:08,743 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-04 23:12:08,744 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 23:12:09,429 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-04 23:12:11,430 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-04 23:12:11,744 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:12:11,970 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:12:11,971 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:12:11,971 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:12:17,153 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:12:22,153 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:12:26,970 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:12:26,971 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:12:26,971 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:12:27,243 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:12:27,244 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:12:32,244 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:12:37,245 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:12:41,970 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:12:41,971 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:12:41,971 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:12:43,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:12:48,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:12:53,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:12:56,970 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:12:56,971 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:12:56,972 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:12:57,186 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:12:59,187 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:13:04,188 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:13:09,188 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:13:11,970 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:13:11,972 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:13:11,972 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:13:14,248 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:13:19,249 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:13:24,250 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:13:26,970 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:13:26,972 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:13:26,972 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:13:27,446 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:13:29,447 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:13:34,447 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:13:39,448 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:13:41,970 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:13:41,972 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:13:41,972 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:13:45,180 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:13:50,181 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:13:55,181 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:13:56,971 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:13:56,972 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:13:56,972 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:13:57,187 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:14:00,188 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:14:05,188 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:14:10,189 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:14:11,971 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:14:11,972 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:14:11,972 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:14:16,163 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:14:21,164 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:14:26,164 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:14:26,971 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:14:26,972 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:14:26,972 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:14:27,230 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:14:31,231 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:14:36,231 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:14:41,232 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:14:41,971 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:14:41,972 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:14:41,972 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:14:47,203 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:14:52,203 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:14:56,971 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:14:56,972 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:14:56,973 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:14:57,244 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:14:57,245 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:15:02,245 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:15:07,246 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:15:11,971 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:15:11,972 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:15:11,973 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:15:13,238 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:15:18,239 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:15:23,239 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:15:26,971 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:15:26,973 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:15:26,973 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:15:27,246 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:15:28,247 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:15:33,247 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:15:38,248 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:15:41,971 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:15:41,973 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:15:41,973 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:15:44,146 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:15:49,146 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:15:54,147 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:15:56,971 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:15:56,973 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:15:56,973 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:15:57,168 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:15:59,169 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:16:04,169 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:16:09,170 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:16:11,972 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:16:11,973 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:16:11,973 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:16:14,195 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:16:19,195 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:16:24,195 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:16:26,971 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:16:26,973 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:16:26,973 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:16:27,188 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:16:30,190 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:16:35,190 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:16:40,191 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:16:41,971 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:16:41,973 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:16:41,973 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:16:46,188 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:16:51,189 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:16:56,190 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:16:56,972 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:16:56,973 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:16:56,973 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:16:57,189 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:17:02,190 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:17:07,190 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:17:11,972 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:17:11,973 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:17:11,973 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:17:13,059 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-04 23:17:13,062 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-04 23:17:13,062 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-04 23:17:13,062 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:17:13,063 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 23:17:13,603 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-04 23:17:15,604 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-04 23:17:18,064 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:17:23,065 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:17:26,972 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:17:26,973 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:17:26,973 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:17:27,196 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:17:28,197 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:17:33,198 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:17:38,198 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:17:41,972 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:17:41,973 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:17:41,973 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:17:43,599 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:17:48,599 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:17:53,600 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:17:56,972 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:17:56,973 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:17:56,974 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:17:57,191 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:17:59,191 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:18:04,192 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:18:09,193 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:18:11,972 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:18:11,974 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:18:11,974 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:18:14,235 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:18:19,236 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:18:24,236 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:18:26,972 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:18:26,974 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:18:26,974 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:18:27,206 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:18:30,207 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:18:35,208 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:18:40,208 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:18:41,972 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:18:41,974 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:18:41,974 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:18:45,232 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:18:50,233 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:18:55,233 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:18:56,973 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:18:56,974 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:18:56,974 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:18:57,170 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:19:01,171 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:19:06,172 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:19:11,172 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:19:11,973 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:19:11,974 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:19:11,974 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:19:16,210 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:19:21,210 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:19:26,211 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:19:26,973 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:19:26,974 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:19:26,974 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:19:27,222 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:19:31,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:19:36,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:19:41,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:19:41,974 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:19:41,974 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:19:41,974 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:19:47,187 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:19:52,188 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:19:56,973 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:19:56,974 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:19:56,974 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:19:57,244 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:19:57,244 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:20:02,245 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:20:07,245 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:20:11,973 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:20:11,974 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:20:11,975 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:20:13,237 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:20:18,238 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:20:23,238 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:20:26,974 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:20:26,974 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:20:26,975 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:20:27,208 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:20:29,209 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:20:34,210 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:20:39,210 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:20:41,973 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:20:41,975 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:20:41,975 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:20:44,233 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:20:49,234 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:20:54,234 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:20:56,974 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:20:56,975 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:20:56,975 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:20:57,214 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:21:00,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:21:05,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:21:10,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:21:11,974 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:21:11,975 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:21:11,975 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:21:16,214 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:21:21,214 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:21:26,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:21:26,974 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:21:26,975 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:21:26,975 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:21:27,142 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:21:32,143 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:21:37,144 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:21:41,974 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:21:41,975 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:21:41,975 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:21:42,176 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:21:47,177 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:21:52,177 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:21:56,974 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:21:56,975 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:21:56,975 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:21:57,149 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:21:58,150 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:22:03,151 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:22:08,151 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:22:11,974 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:22:11,975 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:22:11,975 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:22:13,230 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:22:16,811 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-04 23:22:16,813 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-04 23:22:16,814 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-04 23:22:16,815 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 23:22:17,772 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-04 23:22:17,772 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-04 23:22:18,816 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:22:23,817 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:22:26,974 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:22:26,975 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:22:26,975 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:22:27,251 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:22:29,251 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:22:34,252 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:22:39,253 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:22:41,974 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:22:41,975 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:22:41,975 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:22:44,305 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:22:49,306 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:22:54,306 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:22:56,974 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:22:56,975 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:22:56,976 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:22:57,244 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:23:00,245 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:23:05,246 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:23:10,246 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:23:11,974 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:23:11,975 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:23:11,976 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:23:16,193 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:23:21,194 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:23:26,194 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:23:26,974 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:23:26,976 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:23:26,976 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:23:27,252 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:23:31,253 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:23:36,253 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:23:41,254 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:23:41,975 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:23:41,976 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:23:41,976 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:23:47,207 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:23:52,207 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:23:56,975 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:23:56,976 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:23:56,976 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:23:57,169 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:23:58,169 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:24:03,170 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:24:08,171 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:24:11,975 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:24:11,976 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:24:11,976 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:24:13,257 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:24:18,257 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:24:23,258 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:24:26,975 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:24:26,976 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:24:26,976 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:24:27,257 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:24:29,257 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:24:34,258 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:24:39,259 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:24:41,975 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:24:41,976 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:24:41,976 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:24:45,145 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:24:50,146 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:24:55,146 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:24:56,975 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:24:56,976 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:24:56,976 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:24:57,209 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:25:00,210 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:25:05,210 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:25:10,211 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:25:11,975 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:25:11,976 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:25:11,976 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:25:16,178 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:25:21,179 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:25:26,179 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:25:26,975 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:25:26,976 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:25:26,977 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:25:27,225 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:25:31,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:25:36,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:25:41,227 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:25:41,975 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:25:41,976 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:25:41,976 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:25:46,492 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:25:51,493 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:25:56,494 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:25:56,975 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:25:56,977 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:25:56,977 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:25:57,227 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:26:02,228 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:26:07,228 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:26:11,975 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:26:11,977 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:26:11,977 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:26:12,237 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:26:17,237 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:26:22,238 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:26:26,976 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:26:26,977 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:26:26,977 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:26:27,216 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:26:28,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:26:33,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:26:38,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:26:41,976 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:26:41,977 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:26:41,977 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:26:44,201 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:26:49,201 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:26:54,202 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:26:56,976 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:26:56,977 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:26:56,977 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:26:57,188 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:27:00,189 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:27:05,189 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:27:10,190 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:27:11,976 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:27:11,977 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:27:11,977 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:27:16,176 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:27:21,176 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:27:21,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-04 23:27:21,314 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-04 23:27:21,315 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-04 23:27:21,316 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 23:27:21,943 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-04 23:27:23,944 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-04 23:27:26,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:27:26,976 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:27:26,977 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:27:26,977 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:27:27,216 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:27:32,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:27:37,218 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:27:41,976 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:27:41,977 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:27:41,977 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:27:42,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:27:47,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:27:52,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:27:56,976 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:27:56,977 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:27:56,977 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:27:57,161 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:27:58,162 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:28:03,162 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:28:08,163 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:28:11,976 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:28:11,977 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:28:11,978 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:28:13,187 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:28:18,187 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:28:23,188 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:28:26,977 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:28:26,977 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:28:26,978 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:28:27,156 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:28:29,157 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:28:34,158 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:28:39,158 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:28:41,977 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:28:41,977 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:28:41,978 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:28:44,212 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:28:49,212 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:28:54,213 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:28:56,977 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:28:56,978 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:28:56,978 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:28:57,168 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:29:00,168 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:29:05,169 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:29:10,169 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:29:11,977 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:29:11,978 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:29:11,978 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:29:15,243 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:29:20,244 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:29:25,244 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:29:26,977 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:29:26,978 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:29:26,978 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:29:27,245 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:29:30,245 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:29:35,246 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:29:40,246 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:29:41,977 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:29:41,978 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:29:41,978 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:29:46,245 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:29:51,245 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:29:56,246 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:29:56,977 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:29:56,978 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:29:56,978 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:29:57,178 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:30:02,179 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:30:07,179 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:30:11,977 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:30:11,978 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:30:11,978 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:30:12,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:30:17,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:30:22,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:30:26,977 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:30:26,978 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:30:26,978 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:30:27,206 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:30:28,207 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:30:33,208 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:30:38,208 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:30:41,977 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:30:41,978 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:30:41,978 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:30:44,154 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:30:49,155 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:30:54,155 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:30:56,977 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:30:56,978 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:30:56,979 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:30:57,175 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:30:59,176 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:31:04,177 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:31:09,178 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:31:11,977 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:31:11,979 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:31:11,979 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:31:14,243 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:31:19,244 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:31:24,245 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:31:26,977 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:31:26,979 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:31:26,979 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:31:28,276 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:31:29,276 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:31:34,277 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:31:39,278 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:31:41,978 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:31:41,979 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:31:41,979 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:31:45,178 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:31:50,179 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:31:55,180 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:31:56,978 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:31:56,979 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:31:56,979 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:31:57,171 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:32:01,173 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:32:06,173 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:32:11,174 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:32:11,978 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:32:11,979 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:32:11,979 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:32:16,243 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:32:21,244 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:32:25,720 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-04 23:32:25,722 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-04 23:32:25,722 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-04 23:32:25,725 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 23:32:26,112 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-04 23:32:26,725 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:32:26,978 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:32:26,979 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:32:26,979 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:32:27,224 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:32:28,114 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-04 23:32:32,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:32:37,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:32:41,978 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:32:41,979 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:32:41,979 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:32:42,243 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:32:47,244 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:32:52,245 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:32:56,978 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:32:56,980 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:32:56,980 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:32:57,210 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:32:58,210 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:33:03,211 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:33:08,213 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:33:11,978 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:33:11,980 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:33:11,980 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:33:13,243 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:33:18,244 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:33:23,244 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:33:26,978 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:33:26,979 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:33:26,980 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:33:27,213 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:33:29,214 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:33:34,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:33:39,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:33:41,978 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:33:41,979 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:33:41,980 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:33:44,238 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:33:49,239 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:33:54,239 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:33:56,978 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:33:56,980 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:33:56,980 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:33:57,222 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:34:00,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:34:05,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:34:10,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:34:11,979 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:34:11,980 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:34:11,980 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:34:16,146 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:34:21,146 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:34:26,147 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:34:26,979 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:34:26,980 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:34:26,980 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:34:27,233 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:34:31,234 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:34:36,234 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:34:41,235 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:34:41,979 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:34:41,980 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:34:41,980 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:34:47,192 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:34:52,193 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:34:56,979 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:34:56,980 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:34:56,980 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:34:57,236 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:34:57,237 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:35:02,238 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:35:07,238 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:35:11,979 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:35:11,980 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:35:11,980 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:35:13,180 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:35:18,181 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:35:23,181 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:35:26,979 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:35:26,980 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:35:26,980 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:35:27,214 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:35:28,214 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:35:33,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:35:38,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:35:41,979 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:35:41,980 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:35:41,980 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:35:44,176 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:35:49,177 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:35:54,177 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:35:56,980 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:35:56,981 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:35:56,981 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:35:57,243 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:35:59,243 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:36:04,244 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:36:09,245 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:36:11,979 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:36:11,980 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:36:11,981 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:36:15,232 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:36:20,232 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:36:25,233 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:36:26,979 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:36:26,981 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:36:26,981 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:36:27,194 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:36:31,195 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:36:36,196 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:36:41,198 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:36:41,980 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:36:41,981 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:36:41,981 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:36:46,250 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:36:51,251 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:36:56,252 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:36:56,980 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:36:56,981 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:36:56,981 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:36:57,173 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:37:02,174 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:37:07,175 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:37:11,980 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:37:11,981 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:37:11,981 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:37:13,162 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:37:18,163 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:37:23,164 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:37:26,980 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:37:26,981 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:37:26,981 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:37:27,244 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:37:28,245 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:37:29,948 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-04 23:37:29,950 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-04 23:37:29,950 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-04 23:37:29,952 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 23:37:30,288 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-04 23:37:32,289 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-04 23:37:33,952 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:37:38,953 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:37:41,980 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:37:41,981 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:37:41,981 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:37:44,154 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:37:49,155 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:37:54,155 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:37:56,980 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:37:56,981 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:37:56,981 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:37:57,255 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:37:59,256 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:38:04,256 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:38:09,257 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:38:11,980 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:38:11,981 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:38:11,981 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:38:15,214 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:38:20,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:38:25,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:38:26,980 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:38:26,981 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:38:26,981 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:38:27,167 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:38:31,168 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:38:36,168 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:38:41,169 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:38:41,980 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:38:41,981 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:38:41,982 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:38:46,181 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:38:51,182 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:38:56,182 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:38:56,980 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:38:56,981 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:38:56,982 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:38:57,214 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:39:01,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:39:06,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:39:11,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:39:11,981 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:39:11,981 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:39:11,982 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:39:16,236 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:39:21,236 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:39:26,237 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:39:26,981 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:39:26,982 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:39:26,982 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:39:27,240 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:39:31,241 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:39:36,242 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:39:41,243 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:39:41,981 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:39:41,982 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:39:41,982 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:39:47,194 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:39:52,195 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:39:56,981 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:39:56,982 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:39:56,982 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:39:57,234 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:39:57,234 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:40:02,235 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:40:07,236 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:40:11,981 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:40:11,982 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:40:11,982 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:40:13,213 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:40:18,214 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:40:23,214 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:40:26,981 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:40:26,982 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:40:26,982 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:40:27,253 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:40:28,254 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:40:33,254 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:40:38,255 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:40:41,981 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:40:41,982 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:40:41,983 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:40:43,256 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:40:48,256 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:40:53,257 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:40:56,981 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:40:56,982 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:40:56,983 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:40:57,154 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:40:59,155 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:41:04,155 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:41:09,156 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:41:11,981 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:41:11,982 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:41:11,983 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:41:14,255 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:41:19,256 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:41:24,257 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:41:26,981 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:41:26,982 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:41:26,983 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:41:27,166 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:41:30,167 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:41:35,167 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:41:40,168 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:41:41,982 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:41:41,983 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:41:41,983 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:41:45,253 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:41:50,254 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:41:55,255 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:41:56,982 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:41:56,983 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:41:56,983 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:41:57,193 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:42:01,194 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:42:06,195 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:42:11,195 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:42:11,981 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:42:11,983 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:42:11,983 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:42:16,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:42:21,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:42:26,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:42:26,981 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:42:26,983 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:42:26,983 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:42:27,217 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:42:32,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:42:34,011 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-04 23:42:34,014 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-04 23:42:34,014 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-04 23:42:34,015 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 23:42:34,466 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-04 23:42:35,466 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-04 23:42:38,016 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:42:41,982 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:42:41,983 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:42:41,983 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:42:43,228 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:42:48,229 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:42:53,229 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:42:56,982 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:42:56,983 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:42:56,983 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:42:57,187 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:42:59,188 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:43:04,189 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:43:09,189 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:43:11,982 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:43:11,983 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:43:11,983 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:43:14,200 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:43:19,201 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:43:24,201 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:43:26,982 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:43:26,983 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:43:26,984 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:43:27,224 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:43:29,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:43:34,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:43:39,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:43:41,982 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:43:41,983 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:43:41,983 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:43:44,251 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:43:49,251 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:43:54,252 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:43:56,982 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:43:56,983 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:43:56,984 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:43:57,199 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:44:00,200 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:44:05,200 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:44:10,201 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:44:11,982 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:44:11,984 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:44:11,984 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:44:15,242 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:44:20,243 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:44:25,243 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:44:26,982 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:44:26,984 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:44:26,984 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:44:27,217 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:44:31,218 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:44:36,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:44:41,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:44:41,982 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:44:41,984 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:44:41,984 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:44:46,233 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:44:51,234 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:44:56,234 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:44:56,983 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:44:56,984 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:44:56,984 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:44:57,245 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:45:01,246 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:45:06,246 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:45:11,247 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:45:11,983 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:45:11,984 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:45:11,984 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:45:17,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:45:22,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:45:26,983 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:45:26,984 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:45:26,984 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:45:27,154 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:45:28,155 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:45:33,156 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:45:38,156 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:45:41,983 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:45:41,984 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:45:41,984 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:45:43,187 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:45:48,188 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:45:53,188 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:45:56,983 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:45:56,984 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:45:56,984 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:45:57,263 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:45:58,264 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:46:03,265 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:46:08,265 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:46:11,983 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:46:11,984 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:46:11,984 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:46:14,188 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:46:19,188 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:46:24,189 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:46:26,983 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:46:26,984 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:46:26,985 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:46:27,166 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:46:30,167 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:46:35,167 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:46:40,168 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:46:41,983 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:46:41,985 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:46:41,985 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:46:45,242 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:46:50,243 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:46:55,244 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:46:56,984 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:46:56,985 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:46:56,985 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:46:57,233 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:47:01,234 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:47:06,234 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:47:11,235 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:47:11,984 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:47:11,985 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:47:11,985 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:47:17,165 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:47:22,166 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:47:26,984 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:47:26,985 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:47:26,985 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:47:27,240 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:47:27,240 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:47:32,241 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:47:37,241 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:47:37,748 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-04 23:47:37,750 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-04 23:47:37,750 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-04 23:47:37,752 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 23:47:38,641 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-04 23:47:39,641 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-04 23:47:41,984 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:47:41,985 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:47:41,985 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:47:43,241 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:47:48,241 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:47:53,242 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:47:56,984 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:47:56,985 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:47:56,985 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:47:57,186 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:47:59,187 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:48:04,188 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:48:09,189 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:48:11,984 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:48:11,985 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:48:11,985 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:48:14,191 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:48:19,192 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:48:24,192 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:48:26,984 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:48:26,985 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:48:26,985 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:48:27,212 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:48:29,212 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:48:34,213 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:48:39,214 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:48:41,984 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:48:41,985 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:48:41,986 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:48:45,207 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:48:50,207 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:48:55,208 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:48:56,984 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:48:56,986 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:48:56,986 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:48:57,158 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:49:01,159 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:49:06,160 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:49:11,160 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:49:11,984 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:49:11,986 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:49:11,986 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:49:16,237 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:49:21,238 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:49:26,239 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:49:26,985 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:49:26,986 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:49:26,986 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:49:27,265 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:49:31,266 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:49:36,266 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:49:41,267 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:49:41,984 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:49:41,986 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:49:41,986 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:49:47,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:49:52,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:49:56,985 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:49:56,986 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:49:56,986 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:49:57,209 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:49:58,210 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:50:03,211 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:50:08,211 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:50:11,985 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:50:11,986 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:50:11,986 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:50:14,161 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:50:19,162 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:50:24,163 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:50:26,985 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:50:26,986 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:50:26,986 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:50:27,164 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:50:29,164 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:50:34,165 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:50:39,165 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:50:41,985 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:50:41,986 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:50:41,986 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:50:44,193 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:50:49,194 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:50:54,194 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:50:56,985 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:50:56,986 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:50:56,986 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:50:57,199 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:50:59,200 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:51:04,200 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:51:09,201 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:51:11,985 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:51:11,986 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:51:11,986 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:51:14,203 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:51:19,203 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:51:24,204 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:51:26,985 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:51:26,986 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:51:26,986 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:51:27,241 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:51:29,242 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:51:34,243 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:51:39,243 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:51:41,985 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:51:41,986 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:51:41,987 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:51:45,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:51:50,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:51:55,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:51:56,985 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:51:56,986 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:51:56,987 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:51:57,245 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:52:00,246 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:52:05,247 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:52:10,247 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:52:11,986 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:52:11,987 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:52:11,987 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:52:16,189 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:52:21,190 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:52:26,191 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:52:26,986 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:52:26,987 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:52:26,987 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:52:27,170 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:52:32,171 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:52:37,171 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:52:41,240 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-04 23:52:41,242 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-04 23:52:41,242 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-04 23:52:41,244 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 23:52:41,804 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-04 23:52:41,986 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:52:41,987 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:52:41,987 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:52:42,242 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:52:43,805 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-04 23:52:47,242 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:52:52,243 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:52:56,986 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:52:56,987 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:52:56,988 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:52:57,246 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:52:57,246 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:53:02,247 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:53:07,247 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:53:11,986 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:53:11,987 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:53:11,987 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:53:13,168 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:53:18,168 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:53:23,169 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:53:26,986 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:53:26,987 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:53:26,987 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:53:27,215 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:53:28,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:53:33,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:53:38,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:53:41,986 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:53:41,987 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:53:41,988 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:53:44,171 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:53:49,172 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:53:54,172 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:53:56,986 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:53:56,987 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:53:56,988 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:53:57,193 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:53:59,193 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:54:04,194 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:54:09,194 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:54:11,986 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:54:11,988 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:54:11,988 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:54:14,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:54:19,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:54:24,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:54:26,986 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:54:26,988 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:54:26,988 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:54:27,191 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:54:30,192 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:54:35,193 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:54:40,193 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:54:41,987 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:54:41,988 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:54:41,988 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:54:45,244 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:54:50,245 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:54:55,245 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:54:56,987 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:54:56,988 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:54:56,988 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:54:57,269 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:55:00,269 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:55:05,270 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:55:10,271 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:55:11,988 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:55:11,988 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:55:11,988 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:55:16,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:55:21,220 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:55:26,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:55:26,987 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:55:26,988 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:55:26,988 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:55:27,212 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:55:32,213 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:55:37,214 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:55:41,987 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:55:41,988 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:55:41,988 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:55:42,248 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:55:47,249 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:55:52,250 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:55:56,987 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:55:56,988 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:55:56,989 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:55:57,225 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:55:58,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:56:03,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:56:08,227 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:56:11,987 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:56:11,989 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:56:11,989 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:56:14,204 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:56:19,204 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:56:24,205 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:56:26,987 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:56:26,989 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:56:26,989 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:56:27,252 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:56:29,253 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:56:34,253 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:56:39,254 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:56:41,988 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:56:41,989 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:56:41,989 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:56:45,185 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:56:50,186 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:56:55,186 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:56:56,988 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:56:56,989 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:56:56,989 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:56:57,217 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:57:00,218 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:57:05,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:57:10,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:57:11,988 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:57:11,989 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:57:11,989 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:57:16,184 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:57:21,185 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:57:26,186 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:57:26,988 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:57:26,989 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:57:26,989 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:57:27,267 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:57:31,268 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:57:36,269 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:57:41,269 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:57:41,988 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:57:41,989 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:57:41,989 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:57:45,151 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-04 23:57:45,154 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-04 23:57:45,154 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-04 23:57:45,155 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 23:57:45,969 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-04 23:57:47,156 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:57:47,970 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-04 23:57:52,156 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:57:56,988 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:57:56,989 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:57:56,990 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:57:57,185 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:57:57,185 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:58:02,186 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:58:07,186 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:58:11,988 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:58:11,989 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:58:11,990 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:58:12,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:58:17,227 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:58:22,228 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:58:26,988 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:58:26,989 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:58:26,990 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:58:27,171 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:58:28,171 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:58:33,172 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:58:38,172 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:58:42,042 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:58:42,042 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:58:42,043 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:58:43,265 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:58:48,266 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:58:53,266 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:58:57,042 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:58:57,042 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:58:57,084 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:58:57,321 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:58:58,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:59:03,322 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:59:08,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:59:12,043 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:59:12,043 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:59:12,043 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:59:14,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:59:19,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:59:24,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:59:27,042 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:59:27,042 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:59:27,043 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:59:27,315 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-04 23:59:29,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:59:34,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:59:39,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:59:42,042 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:59:42,043 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:59:42,043 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:59:45,243 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:59:50,244 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:59:55,245 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 23:59:57,043 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 23:59:57,043 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-04 23:59:57,084 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 23:59:57,296 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:00:00,296 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:00:05,297 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:00:10,298 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:00:12,043 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:00:12,043 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:00:12,084 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:00:16,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:00:21,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:00:26,227 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:00:27,043 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:00:27,043 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:00:27,084 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:00:27,272 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:00:31,273 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:00:36,273 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:00:41,274 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:00:42,043 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:00:42,043 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:00:42,084 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:00:47,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:00:52,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:00:57,043 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:00:57,043 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:00:57,043 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:00:57,307 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:00:57,308 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:01:02,308 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:01:07,309 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:01:12,043 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:01:12,043 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:01:12,044 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:01:12,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:01:17,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:01:22,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:01:27,043 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:01:27,043 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:01:27,084 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:01:27,253 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:01:28,253 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:01:33,254 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:01:38,255 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:01:42,043 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:01:42,043 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:01:42,084 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:01:43,289 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:01:48,289 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:01:53,290 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:01:57,043 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:01:57,044 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:01:57,084 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:01:57,318 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:01:58,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:02:03,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:02:08,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:02:12,043 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:02:12,044 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:02:12,084 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:02:14,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:02:19,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:02:24,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:02:27,044 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:02:27,044 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:02:27,084 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:02:27,242 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:02:29,243 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:02:34,244 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:02:39,244 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:02:42,044 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:02:42,044 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:02:42,084 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:02:44,260 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:02:49,260 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:02:49,491 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 00:02:49,493 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 00:02:49,493 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 00:02:49,495 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 00:02:50,142 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 00:02:52,143 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 00:02:54,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:02:57,043 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:02:57,044 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:02:57,044 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:02:57,261 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:03:00,262 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:03:05,262 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:03:10,263 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:03:12,044 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:03:12,044 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:03:12,084 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:03:16,233 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:03:21,234 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:03:26,235 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:03:27,044 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:03:27,044 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:03:27,084 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:03:27,280 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:03:31,281 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:03:36,282 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:03:41,282 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:03:42,044 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:03:42,044 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:03:42,084 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:03:47,241 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:03:52,242 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:03:57,044 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:03:57,044 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:03:57,088 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:03:57,285 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:03:57,285 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:04:02,286 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:04:07,286 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:04:12,044 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:04:12,044 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:04:12,088 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:04:13,261 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:04:18,262 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:04:23,262 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:04:27,044 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:04:27,044 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:04:27,088 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:04:27,285 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:04:28,285 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:04:33,286 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:04:38,286 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:04:42,044 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:04:42,045 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:04:42,045 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:04:43,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:04:48,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:04:53,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:04:57,044 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:04:57,045 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:04:57,045 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:04:57,316 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:04:58,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:05:03,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:05:08,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:05:12,045 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:05:12,045 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:05:12,045 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:05:14,252 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:05:19,253 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:05:24,253 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:05:27,044 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:05:27,045 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:05:27,088 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:05:27,257 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:05:29,258 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:05:34,259 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:05:39,259 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:05:42,045 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:05:42,045 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:05:42,088 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:05:45,232 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:05:50,233 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:05:55,233 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:05:57,045 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:05:57,045 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:05:57,045 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:05:57,291 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:06:00,292 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:06:05,292 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:06:10,293 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:06:12,045 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:06:12,045 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:06:12,045 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:06:15,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:06:20,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:06:25,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:06:27,045 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:06:27,045 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:06:27,045 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:06:27,276 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:06:31,276 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:06:36,277 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:06:41,277 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:06:42,045 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:06:42,045 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:06:42,045 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:06:46,283 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:06:51,284 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:06:56,284 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:06:57,045 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:06:57,045 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:06:57,088 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:06:57,256 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:07:02,257 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:07:07,258 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:07:12,045 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:07:12,045 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:07:12,046 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:07:13,251 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:07:18,251 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:07:23,252 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:07:27,045 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:07:27,045 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:07:27,046 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:07:27,240 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:07:29,240 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:07:34,241 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:07:39,241 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:07:42,045 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:07:42,046 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:07:42,046 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:07:44,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:07:49,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:07:53,711 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 00:07:53,713 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 00:07:53,713 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 00:07:53,714 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 00:07:54,311 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 00:07:54,753 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:07:56,312 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 00:07:57,045 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:07:57,046 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:07:57,046 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:07:57,267 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:08:00,268 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:08:05,268 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:08:10,269 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:08:12,046 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:08:12,046 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:08:12,088 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:08:15,271 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:08:20,271 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:08:25,271 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:08:27,046 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:08:27,046 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:08:27,088 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:08:27,251 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:08:31,252 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:08:36,253 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:08:41,253 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:08:42,046 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:08:42,046 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:08:42,088 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:08:46,257 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:08:51,257 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:08:56,258 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:08:57,046 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:08:57,046 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:08:57,088 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:08:57,236 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:09:02,237 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:09:07,238 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:09:12,046 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:09:12,046 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:09:12,046 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:09:12,266 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:09:17,267 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:09:22,267 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:09:27,046 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:09:27,046 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:09:27,047 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:09:27,305 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:09:27,306 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:09:32,306 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:09:37,306 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:09:42,046 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:09:42,047 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:09:42,088 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:09:42,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:09:47,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:09:52,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:09:57,046 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:09:57,046 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:09:57,047 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:09:57,231 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:09:58,231 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:10:03,232 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:10:08,232 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:10:12,046 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:10:12,047 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:10:12,047 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:10:13,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:10:18,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:10:23,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:10:27,046 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:10:27,047 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:10:27,047 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:10:27,313 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:10:28,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:10:33,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:10:38,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:10:42,047 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:10:42,047 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:10:42,047 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:10:44,286 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:10:49,287 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:10:54,287 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:10:57,047 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:10:57,047 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:10:57,047 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:10:57,222 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:11:00,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:11:05,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:11:10,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:11:12,047 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:11:12,047 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:11:12,047 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:11:15,287 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:11:20,288 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:11:25,289 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:11:27,047 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:11:27,047 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:11:27,047 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:11:27,310 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:11:30,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:11:35,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:11:40,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:11:42,047 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:11:42,047 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:11:42,047 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:11:46,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:11:51,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:11:56,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:11:57,047 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:11:57,047 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:11:57,048 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:11:57,304 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:12:02,306 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:12:07,306 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:12:12,047 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:12:12,048 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:12:12,048 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:12:13,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:12:18,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:12:23,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:12:27,047 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:12:27,048 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:12:27,048 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:12:27,292 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:12:28,293 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:12:33,293 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:12:38,294 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:12:42,047 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:12:42,048 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:12:42,088 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:12:44,272 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:12:49,272 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:12:54,273 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:12:57,048 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:12:57,048 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:12:57,088 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:12:57,241 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:12:57,764 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 00:12:57,765 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 00:12:57,765 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 00:12:57,767 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 00:12:58,478 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 00:12:58,478 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 00:12:59,805 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:13:04,806 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:13:09,806 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:13:12,048 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:13:12,048 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:13:12,088 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:13:15,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:13:20,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:13:25,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:13:27,048 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:13:27,048 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:13:27,088 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:13:27,270 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:13:31,270 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:13:36,271 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:13:41,272 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:13:42,048 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:13:42,048 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:13:42,048 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:13:46,296 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:13:51,296 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:13:56,296 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:13:57,048 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:13:57,048 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:13:57,088 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:13:57,314 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:14:01,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:14:06,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:14:11,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:14:12,048 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:14:12,048 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:14:12,092 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:14:16,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:14:21,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:14:26,322 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:14:27,048 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:14:27,048 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:14:27,088 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:14:27,279 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:14:32,280 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:14:37,280 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:14:42,048 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:14:42,048 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:14:42,092 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:14:43,257 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:14:48,257 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:14:53,258 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:14:57,048 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:14:57,048 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:14:57,092 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:14:57,272 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:14:58,273 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:15:03,273 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:15:08,274 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:15:12,049 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:15:12,049 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:15:12,049 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:15:13,302 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:15:18,303 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:15:23,303 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:15:27,048 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:15:27,049 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:15:27,092 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:15:27,319 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:15:28,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:15:33,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:15:38,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:15:42,049 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:15:42,049 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:15:42,049 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:15:44,260 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:15:49,260 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:15:54,261 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:15:57,049 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:15:57,049 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:15:57,049 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:15:57,318 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:15:59,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:16:04,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:16:09,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:16:12,049 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:16:12,049 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:16:12,092 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:16:15,278 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:16:20,279 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:16:25,279 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:16:27,049 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:16:27,049 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:16:27,092 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:16:27,253 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:16:31,254 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:16:36,255 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:16:41,256 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:16:42,049 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:16:42,049 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:16:42,092 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:16:46,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:16:51,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:16:56,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:16:57,049 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:16:57,049 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:16:57,092 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:16:57,287 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:17:02,288 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:17:07,289 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:17:12,050 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:17:12,050 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:17:12,051 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:17:12,303 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:17:17,304 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:17:22,305 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:17:27,050 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:17:27,050 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:17:27,051 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:17:27,323 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:17:27,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:17:32,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:17:37,325 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:17:42,050 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:17:42,050 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:17:42,092 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:17:42,406 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:17:47,407 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:17:52,407 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:17:57,050 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:17:57,050 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:17:57,050 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:17:57,254 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:17:58,254 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:18:01,791 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 00:18:01,793 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 00:18:01,793 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 00:18:01,795 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 00:18:02,647 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 00:18:03,648 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 00:18:03,833 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:18:08,834 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:18:12,050 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:18:12,050 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:18:12,051 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:18:14,332 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:18:19,332 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:18:24,333 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:18:27,050 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:18:27,051 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:18:27,092 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:18:27,242 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:18:30,243 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:18:35,244 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:18:40,244 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:18:42,051 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:18:42,051 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:18:42,051 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:18:45,263 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:18:50,264 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:18:55,264 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:18:57,051 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:18:57,051 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:18:57,051 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:18:57,258 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:19:01,259 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:19:06,260 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:19:11,260 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:19:12,051 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:19:12,051 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:19:12,051 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:19:16,268 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:19:21,269 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:19:26,270 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:19:27,051 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:19:27,051 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:19:27,051 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:19:27,285 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:19:31,286 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:19:36,287 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:19:41,287 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:19:42,051 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:19:42,051 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:19:42,051 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:19:47,241 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:19:52,242 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:19:57,051 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:19:57,051 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:19:57,052 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:19:57,303 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:19:57,303 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:20:02,304 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:20:07,305 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:20:12,051 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:20:12,051 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:20:12,052 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:20:12,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:20:17,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:20:22,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:20:27,051 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:20:27,051 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:20:27,052 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:20:27,332 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:20:27,332 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:20:32,333 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:20:37,333 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:20:42,051 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:20:42,052 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:20:42,052 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:20:43,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:20:48,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:20:53,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:20:57,051 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:20:57,052 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:20:57,052 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:20:57,221 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:20:59,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:21:04,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:21:09,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:21:12,052 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:21:12,052 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:21:12,052 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:21:14,261 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:21:19,262 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:21:24,263 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:21:27,052 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:21:27,052 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:21:27,092 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:21:27,323 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:21:29,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:21:34,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:21:39,325 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:21:42,052 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:21:42,052 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:21:42,092 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:21:45,287 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:21:50,288 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:21:55,288 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:21:57,052 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:21:57,052 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:21:57,092 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:21:57,277 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:22:01,278 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:22:06,278 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:22:11,279 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:22:12,052 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:22:12,052 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:22:12,092 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:22:16,322 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:22:21,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:22:26,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:22:27,052 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:22:27,052 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:22:27,092 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:22:27,262 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:22:32,262 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:22:37,263 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:22:42,052 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:22:42,052 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:22:42,096 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:22:42,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:22:47,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:22:52,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:22:57,052 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:22:57,052 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:22:57,096 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:22:57,285 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:22:58,286 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:23:03,287 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:23:05,924 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 00:23:05,925 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 00:23:05,925 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 00:23:05,927 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 00:23:06,813 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 00:23:07,813 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 00:23:08,965 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:23:12,052 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:23:12,052 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:23:12,053 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:23:14,277 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:23:19,278 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:23:24,278 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:23:27,052 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:23:27,053 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:23:27,096 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:23:27,271 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:23:30,271 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:23:35,272 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:23:40,273 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:23:42,053 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:23:42,053 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:23:42,053 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:23:45,274 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:23:50,275 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:23:55,275 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:23:57,053 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:23:57,053 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:23:57,053 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:23:57,311 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:24:00,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:24:05,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:24:10,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:24:12,053 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:24:12,053 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:24:12,053 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:24:16,234 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:24:21,234 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:24:26,235 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:24:27,053 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:24:27,053 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:24:27,053 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:24:27,224 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:24:32,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:24:37,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:24:42,053 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:24:42,053 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:24:42,053 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:24:42,255 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:24:47,255 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:24:52,256 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:24:57,053 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:24:57,053 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:24:57,053 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:24:57,264 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:24:57,264 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:25:02,265 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:25:07,265 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:25:12,053 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:25:12,053 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:25:12,053 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:25:12,298 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:25:17,299 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:25:22,299 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:25:27,053 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:25:27,053 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:25:27,054 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:25:27,284 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:25:28,284 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:25:33,285 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:25:38,286 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:25:42,053 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:25:42,054 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:25:42,054 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:25:43,303 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:25:48,304 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:25:53,304 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:25:57,096 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:25:57,097 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:25:57,097 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:25:57,358 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:25:58,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:26:03,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:26:08,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:26:12,095 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:26:12,096 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:26:12,096 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:26:14,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:26:19,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:26:24,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:26:27,095 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:26:27,096 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:26:27,096 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:26:27,347 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:26:29,348 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:26:34,349 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:26:39,349 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:26:42,095 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:26:42,096 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:26:42,096 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:26:45,298 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:26:50,298 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:26:55,299 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:26:57,095 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:26:57,096 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:26:57,097 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:26:57,266 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:27:01,267 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:27:06,268 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:27:11,268 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:27:12,095 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:27:12,096 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:27:12,097 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:27:16,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:27:21,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:27:26,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:27:27,096 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:27:27,096 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:27:27,096 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:27:27,311 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:27:32,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:27:37,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:27:42,096 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:27:42,096 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:27:42,097 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:27:43,294 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:27:48,295 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:27:53,296 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:27:57,096 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:27:57,097 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:27:57,097 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:27:57,355 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:27:58,356 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:28:03,356 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:28:08,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:28:10,603 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 00:28:10,605 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 00:28:10,605 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 00:28:10,606 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 00:28:10,979 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 00:28:11,979 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 00:28:12,096 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:28:12,097 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:28:12,097 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:28:14,302 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:28:19,303 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:28:24,303 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:28:27,096 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:28:27,097 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:28:27,097 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:28:27,340 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:28:29,340 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:28:34,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:28:39,342 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:28:42,096 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:28:42,097 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:28:42,097 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:28:45,291 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:28:50,292 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:28:55,292 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:28:57,096 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:28:57,097 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:28:57,097 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:28:57,370 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:29:00,371 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:29:05,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:29:10,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:29:12,097 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:29:12,097 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:29:12,098 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:29:16,283 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:29:21,283 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:29:26,284 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:29:27,097 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:29:27,098 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:29:27,098 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:29:27,358 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:29:31,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:29:36,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:29:41,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:29:42,097 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:29:42,097 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:29:42,098 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:29:47,349 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:29:52,349 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:29:57,097 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:29:57,098 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:29:57,098 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:29:57,292 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:29:58,292 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:30:03,293 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:30:08,294 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:30:12,097 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:30:12,098 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:30:12,098 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:30:13,366 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:30:18,366 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:30:23,367 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:30:27,097 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:30:27,098 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:30:27,098 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:30:27,347 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:30:29,348 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:30:34,348 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:30:39,349 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:30:42,097 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:30:42,098 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:30:42,098 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:30:45,269 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:30:50,270 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:30:55,271 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:30:57,097 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:30:57,098 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:30:57,099 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:30:57,294 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:31:00,295 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:31:05,295 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:31:10,296 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:31:12,097 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:31:12,098 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:31:12,099 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:31:16,296 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:31:21,297 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:31:26,297 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:31:27,097 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:31:27,098 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:31:27,098 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:31:27,270 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:31:32,271 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:31:37,272 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:31:42,098 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:31:42,099 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:31:42,099 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:31:42,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:31:47,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:31:52,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:31:57,098 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:31:57,098 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:31:57,099 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:31:57,344 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:31:58,345 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:32:03,346 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:32:08,346 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:32:12,098 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:32:12,099 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:32:12,099 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:32:14,335 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:32:19,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:32:24,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:32:27,098 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:32:27,099 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:32:27,099 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:32:27,347 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:32:29,347 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:32:34,348 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:32:39,348 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:32:42,098 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:32:42,099 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:32:42,099 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:32:45,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:32:50,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:32:55,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:32:57,098 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:32:57,099 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:32:57,100 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:32:57,311 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:33:01,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:33:06,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:33:11,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:33:12,098 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:33:12,099 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:33:12,099 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:33:14,819 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 00:33:14,822 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 00:33:14,822 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 00:33:14,824 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 00:33:15,146 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 00:33:16,146 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 00:33:16,825 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:33:21,826 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:33:26,826 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:33:27,098 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:33:27,100 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:33:27,100 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:33:27,338 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:33:32,339 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:33:37,340 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:33:42,098 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:33:42,099 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:33:42,100 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:33:43,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:33:48,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:33:53,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:33:57,098 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:33:57,099 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:33:57,100 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:33:57,273 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:33:59,274 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:34:04,275 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:34:09,275 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:34:12,099 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:34:12,100 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:34:12,100 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:34:14,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:34:19,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:34:24,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:34:27,099 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:34:27,100 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:34:27,100 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:34:27,351 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:34:29,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:34:34,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:34:39,353 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:34:42,099 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:34:42,100 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:34:42,100 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:34:45,274 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:34:50,275 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:34:55,275 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:34:57,099 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:34:57,100 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:34:57,100 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:34:57,309 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:35:00,310 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:35:05,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:35:10,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:35:12,099 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:35:12,100 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:35:12,100 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:35:15,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:35:20,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:35:25,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:35:27,099 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:35:27,101 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:35:27,101 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:35:27,298 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:35:31,299 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:35:36,300 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:35:41,300 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:35:42,099 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:35:42,100 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:35:42,101 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:35:46,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:35:51,353 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:35:56,354 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:35:57,099 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:35:57,100 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:35:57,101 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:35:57,372 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:36:01,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:36:06,374 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:36:11,375 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:36:12,099 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:36:12,101 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:36:12,101 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:36:16,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:36:21,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:36:26,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:36:27,100 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:36:27,101 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:36:27,101 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:36:27,296 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:36:32,297 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:36:37,298 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:36:42,100 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:36:42,101 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:36:42,101 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:36:42,347 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:36:47,348 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:36:52,349 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:36:57,100 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:36:57,101 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:36:57,101 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:36:57,288 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:36:58,289 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:37:03,290 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:37:08,290 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:37:12,100 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:37:12,101 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:37:12,101 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:37:13,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:37:18,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:37:23,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:37:27,100 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:37:27,101 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:37:27,101 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:37:27,361 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:37:29,362 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:37:34,362 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:37:39,363 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:37:42,100 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:37:42,101 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:37:42,102 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:37:45,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:37:50,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:37:55,331 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:37:57,100 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:37:57,102 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:37:57,102 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:37:57,299 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:38:01,300 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:38:06,300 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:38:11,301 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:38:12,100 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:38:12,102 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:38:12,102 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:38:16,303 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:38:18,703 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 00:38:18,705 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 00:38:18,706 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 00:38:18,708 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 00:38:19,312 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 00:38:20,313 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 00:38:21,708 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:38:26,709 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:38:27,100 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:38:27,102 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:38:27,102 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:38:27,272 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:38:32,273 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:38:37,273 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:38:42,100 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:38:42,102 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:38:42,102 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:38:42,370 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:38:47,371 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:38:52,371 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:38:57,101 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:38:57,102 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:38:57,102 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:38:57,377 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:38:57,378 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:39:02,378 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:39:07,379 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:39:12,101 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:39:12,102 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:39:12,102 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:39:13,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:39:18,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:39:23,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:39:27,101 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:39:27,102 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:39:27,102 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:39:27,360 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:39:28,361 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:39:33,362 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:39:38,363 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:39:42,101 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:39:42,102 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:39:42,102 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:39:44,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:39:49,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:39:54,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:39:57,101 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:39:57,102 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:39:57,103 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:39:57,299 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:40:00,300 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:40:05,300 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:40:10,301 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:40:12,101 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:40:12,102 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:40:12,103 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:40:16,296 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:40:21,297 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:40:26,298 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:40:27,101 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:40:27,102 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:40:27,103 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:40:27,346 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:40:31,347 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:40:36,347 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:40:41,348 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:40:42,101 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:40:42,103 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:40:42,103 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:40:47,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:40:52,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:40:57,101 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:40:57,103 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:40:57,103 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:40:57,280 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:40:58,281 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:41:03,282 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:41:08,282 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:41:12,101 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:41:12,103 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:41:12,103 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:41:13,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:41:18,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:41:23,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:41:27,101 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:41:27,103 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:41:27,103 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:41:27,337 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:41:28,337 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:41:33,338 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:41:38,338 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:41:42,102 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:41:42,103 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:41:42,103 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:41:43,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:41:48,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:41:53,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:41:57,102 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:41:57,103 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:41:57,103 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:41:57,321 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:41:59,322 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:42:04,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:42:09,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:42:12,102 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:42:12,103 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:42:12,104 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:42:15,296 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:42:20,296 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:42:25,297 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:42:27,102 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:42:27,104 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:42:27,104 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:42:27,359 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:42:30,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:42:35,361 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:42:40,361 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:42:42,102 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:42:42,104 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:42:42,104 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:42:46,287 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:42:51,288 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:42:56,288 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:42:57,102 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:42:57,104 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:42:57,104 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:42:57,343 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:43:01,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:43:06,345 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:43:11,345 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:43:12,102 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:43:12,104 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:43:12,104 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:43:17,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:43:22,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:43:22,636 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 00:43:22,638 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 00:43:22,639 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 00:43:22,640 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 00:43:23,479 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 00:43:24,480 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 00:43:27,102 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:43:27,104 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:43:27,104 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:43:27,300 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:43:28,301 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:43:33,302 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:43:38,302 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:43:42,103 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:43:42,104 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:43:42,104 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:43:43,377 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:43:48,377 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:43:53,378 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:43:57,103 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:43:57,104 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:43:57,105 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:43:57,321 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:43:59,322 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:44:04,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:44:09,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:44:12,103 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:44:12,104 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:44:12,105 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:44:15,294 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:44:20,294 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:44:25,295 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:44:27,103 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:44:27,104 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:44:27,105 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:44:27,326 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:44:30,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:44:35,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:44:40,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:44:42,103 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:44:42,105 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:44:42,105 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:44:45,384 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:44:50,384 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:44:55,385 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:44:57,103 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:44:57,105 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:44:57,105 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:44:57,289 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:45:01,290 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:45:06,291 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:45:11,291 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:45:12,103 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:45:12,105 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:45:12,105 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:45:16,363 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:45:21,364 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:45:26,364 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:45:27,103 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:45:27,105 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:45:27,105 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:45:27,327 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:45:32,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:45:37,329 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:45:42,104 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:45:42,105 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:45:42,105 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:45:42,339 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:45:47,339 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:45:52,340 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:45:57,104 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:45:57,105 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:45:57,105 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:45:57,354 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:45:57,355 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:46:02,356 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:46:07,356 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:46:12,104 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:46:12,105 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:46:12,106 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:46:13,309 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:46:18,310 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:46:23,310 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:46:27,104 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:46:27,105 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:46:27,106 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:46:27,299 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:46:29,300 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:46:34,301 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:46:39,302 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:46:42,104 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:46:42,106 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:46:42,106 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:46:44,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:46:49,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:46:54,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:46:57,104 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:46:57,106 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:46:57,106 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:46:57,308 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:47:00,308 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:47:05,309 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:47:10,309 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:47:12,104 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:47:12,106 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:47:12,106 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:47:15,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:47:20,342 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:47:25,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:47:27,104 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:47:27,106 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:47:27,106 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:47:27,286 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:47:31,288 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:47:36,288 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:47:41,289 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:47:42,104 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:47:42,106 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:47:42,106 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:47:46,369 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:47:51,369 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:47:56,370 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:47:57,105 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:47:57,106 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:47:57,106 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:47:57,293 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:48:02,294 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:48:07,295 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:48:12,105 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:48:12,106 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:48:12,106 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:48:12,803 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:48:17,803 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:48:22,804 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:48:27,016 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 00:48:27,018 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 00:48:27,019 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 00:48:27,020 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 00:48:27,105 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:48:27,106 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:48:27,107 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:48:27,303 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:48:27,652 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 00:48:28,303 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:48:29,653 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 00:48:33,304 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:48:38,305 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:48:42,105 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:48:42,106 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:48:42,107 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:48:44,304 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:48:49,305 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:48:54,305 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:48:57,105 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:48:57,107 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:48:57,107 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:48:57,295 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:49:00,296 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:49:05,296 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:49:10,297 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:49:12,105 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:49:12,107 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:49:12,107 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:49:15,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:49:20,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:49:25,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:49:27,105 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:49:27,107 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:49:27,107 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:49:27,379 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:49:30,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:49:35,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:49:40,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:49:42,105 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:49:42,107 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:49:42,107 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:49:45,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:49:50,384 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:49:55,384 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:49:57,105 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:49:57,107 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:49:57,107 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:49:57,327 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:50:01,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:50:06,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:50:11,329 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:50:12,105 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:50:12,107 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:50:12,107 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:50:17,279 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:50:22,280 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:50:27,106 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:50:27,107 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:50:27,107 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:50:27,301 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:50:27,301 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:50:32,302 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:50:37,303 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:50:42,106 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:50:42,107 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:50:42,108 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:50:43,283 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:50:48,284 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:50:53,285 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:50:57,106 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:50:57,108 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:50:57,108 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:50:57,279 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:50:59,279 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:51:04,280 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:51:09,281 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:51:12,106 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:51:12,108 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:51:12,108 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:51:14,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:51:19,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:51:24,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:51:27,106 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:51:27,108 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:51:27,108 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:51:27,319 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:51:29,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:51:34,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:51:39,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:51:42,106 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:51:42,108 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:51:42,108 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:51:45,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:51:50,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:51:55,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:51:57,106 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:51:57,108 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:51:57,108 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:51:57,473 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:52:00,474 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:52:05,474 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:52:10,475 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:52:12,106 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:52:12,108 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:52:12,109 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:52:16,310 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:52:21,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:52:26,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:52:27,107 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:52:27,108 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:52:27,109 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:52:27,324 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:52:31,325 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:52:36,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:52:41,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:52:42,107 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:52:42,109 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:52:42,109 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:52:46,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:52:51,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:52:56,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:52:57,107 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:52:57,109 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:52:57,109 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:52:57,377 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:53:02,378 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:53:07,378 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:53:12,107 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:53:12,109 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:53:12,109 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:53:13,349 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:53:18,349 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:53:23,350 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:53:27,107 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:53:27,109 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:53:27,109 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:53:27,380 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:53:28,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:53:30,783 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 00:53:30,786 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 00:53:30,786 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 00:53:30,787 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 00:53:30,818 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 00:53:33,788 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:53:33,820 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 00:53:38,789 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:53:42,107 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:53:42,109 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:53:42,109 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:53:44,276 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:53:49,276 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:53:54,277 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:53:57,107 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:53:57,109 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:53:57,109 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:53:57,326 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:53:59,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:54:04,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:54:09,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:54:12,107 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:54:12,109 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:54:12,109 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:54:15,307 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:54:20,308 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:54:25,309 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:54:27,107 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:54:27,109 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:54:27,109 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:54:27,309 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:54:30,310 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:54:35,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:54:40,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:54:42,108 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:54:42,109 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:54:42,110 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:54:45,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:54:50,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:54:55,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:54:57,108 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:54:57,109 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:54:57,110 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:54:57,385 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:55:00,386 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:55:05,387 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:55:10,387 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:55:12,108 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:55:12,110 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:55:12,110 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:55:16,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:55:21,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:55:26,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:55:27,108 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:55:27,110 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:55:27,110 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:55:27,301 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:55:32,302 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:55:37,303 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:55:42,108 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:55:42,110 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:55:42,110 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:55:42,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:55:47,374 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:55:52,375 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:55:57,108 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:55:57,110 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:55:57,110 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:55:57,358 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:55:58,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:56:03,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:56:08,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:56:12,108 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:56:12,110 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:56:12,110 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:56:13,367 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:56:18,367 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:56:23,368 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:56:27,108 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:56:27,110 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:56:27,110 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:56:27,325 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:56:29,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:56:34,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:56:39,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:56:42,108 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:56:42,110 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:56:42,110 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:56:44,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:56:49,345 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:56:54,345 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:56:57,109 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:56:57,110 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:56:57,110 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:56:57,378 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:56:59,379 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:57:04,379 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:57:09,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:57:12,109 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:57:12,110 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:57:12,110 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:57:15,299 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:57:20,299 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:57:25,300 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:57:27,109 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:57:27,110 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:57:27,110 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:57:27,371 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:57:30,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:57:35,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:57:40,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:57:42,109 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:57:42,111 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:57:42,111 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:57:46,284 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:57:51,284 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:57:56,285 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:57:57,109 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:57:57,110 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:57:57,111 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:57:57,325 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:58:01,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:58:06,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:58:11,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:58:12,109 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:58:12,111 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:58:12,111 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:58:17,285 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:58:22,285 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:58:27,109 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:58:27,111 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:58:27,111 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:58:27,318 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:58:27,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:58:32,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:58:35,084 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 00:58:35,087 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 00:58:35,087 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 00:58:35,089 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 00:58:35,986 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 00:58:37,987 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 00:58:38,089 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:58:42,109 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:58:42,111 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:58:42,111 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:58:43,402 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:58:48,403 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:58:53,403 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:58:57,109 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:58:57,111 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:58:57,111 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:58:57,379 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:58:59,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:59:04,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:59:09,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:59:12,109 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:59:12,111 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:59:12,111 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:59:15,310 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:59:20,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:59:25,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:59:27,109 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:59:27,111 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:59:27,111 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:59:27,319 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 00:59:30,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:59:35,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:59:40,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:59:42,109 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:59:42,111 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:59:42,111 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:59:45,349 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:59:50,349 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:59:55,350 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 00:59:57,110 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 00:59:57,111 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 00:59:57,112 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 00:59:57,330 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:00:01,331 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:00:06,331 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:00:11,332 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:00:12,110 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:00:12,112 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:00:12,112 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:00:16,351 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:00:21,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:00:26,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:00:27,110 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:00:27,112 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:00:27,112 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:00:27,339 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:00:32,340 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:00:37,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:00:42,110 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:00:42,112 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:00:42,112 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:00:43,303 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:00:48,303 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:00:53,304 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:00:57,110 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:00:57,112 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:00:57,112 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:00:57,356 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:00:58,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:01:03,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:01:08,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:01:12,110 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:01:12,112 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:01:12,112 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:01:14,298 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:01:19,299 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:01:24,299 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:01:27,110 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:01:27,112 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:01:27,112 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:01:27,301 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:01:29,302 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:01:34,302 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:01:39,303 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:01:42,110 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:01:42,112 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:01:42,112 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:01:44,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:01:49,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:01:54,389 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:01:57,110 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:01:57,112 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:01:57,112 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:01:57,321 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:02:00,322 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:02:05,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:02:10,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:02:12,110 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:02:12,112 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:02:12,113 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:02:15,356 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:02:20,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:02:25,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:02:27,111 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:02:27,112 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:02:27,113 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:02:27,358 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:02:30,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:02:35,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:02:40,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:02:42,111 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:02:42,113 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:02:42,113 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:02:46,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:02:51,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:02:56,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:02:57,111 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:02:57,113 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:02:57,113 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:02:57,342 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:03:02,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:03:07,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:03:12,111 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:03:12,113 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:03:12,113 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:03:13,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:03:18,322 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:03:23,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:03:27,111 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:03:27,113 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:03:27,113 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:03:27,300 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:03:29,301 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:03:34,302 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:03:39,040 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 01:03:39,042 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 01:03:39,042 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 01:03:39,044 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 01:03:39,149 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 01:03:40,044 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:03:40,149 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 01:03:42,111 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:03:42,113 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:03:42,113 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:03:45,338 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:03:50,338 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:03:55,339 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:03:57,111 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:03:57,113 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:03:57,113 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:03:57,345 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:04:00,346 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:04:05,347 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:04:10,348 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:04:12,111 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:04:12,113 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:04:12,113 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:04:15,378 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:04:20,379 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:04:25,379 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:04:27,111 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:04:27,113 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:04:27,113 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:04:27,335 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:04:31,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:04:36,337 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:04:41,337 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:04:42,111 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:04:42,113 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:04:42,114 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:04:47,309 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:04:52,309 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:04:57,111 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:04:57,113 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:04:57,114 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:04:57,356 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:04:57,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:05:02,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:05:07,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:05:12,112 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:05:12,114 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:05:12,114 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:05:12,414 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:05:17,415 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:05:22,416 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:05:27,112 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:05:27,114 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:05:27,114 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:05:27,355 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:05:28,356 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:05:33,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:05:38,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:05:42,112 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:05:42,114 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:05:42,114 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:05:44,297 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:05:49,297 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:05:54,298 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:05:57,112 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:05:57,114 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:05:57,114 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:05:57,307 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:05:59,308 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:06:04,308 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:06:09,309 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:06:12,112 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:06:12,114 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:06:12,114 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:06:14,385 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:06:19,386 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:06:24,386 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:06:27,112 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:06:27,114 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:06:27,114 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:06:27,317 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:06:30,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:06:35,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:06:40,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:06:42,112 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:06:42,114 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:06:42,114 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:06:45,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:06:50,322 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:06:55,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:06:57,112 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:06:57,114 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:06:57,115 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:06:57,336 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:07:00,337 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:07:05,338 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:07:10,338 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:07:12,112 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:07:12,114 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:07:12,115 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:07:15,354 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:07:20,354 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:07:25,355 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:07:27,112 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:07:27,115 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:07:27,115 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:07:27,381 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:07:30,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:07:35,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:07:40,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:07:42,113 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:07:42,115 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:07:42,115 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:07:46,348 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:07:51,349 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:07:56,349 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:07:57,113 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:07:57,115 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:07:57,115 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:07:57,390 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:08:01,391 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:08:06,391 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:08:11,392 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:08:12,113 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:08:12,115 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:08:12,115 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:08:17,329 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:08:22,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:08:27,113 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:08:27,115 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:08:27,115 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:08:27,376 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:08:27,376 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:08:32,378 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:08:37,378 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:08:42,113 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:08:42,115 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:08:42,115 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:08:42,639 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 01:08:42,642 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 01:08:42,642 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 01:08:42,643 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:08:42,644 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 01:08:43,316 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 01:08:44,317 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 01:08:47,645 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:08:52,646 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:08:57,113 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:08:57,115 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:08:57,115 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:08:57,310 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:08:58,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:09:03,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:09:08,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:09:12,113 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:09:12,115 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:09:12,115 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:09:13,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:09:18,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:09:23,331 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:09:27,113 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:09:27,115 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:09:27,115 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:09:27,387 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:09:28,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:09:33,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:09:38,389 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:09:42,113 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:09:42,116 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:09:42,116 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:09:44,362 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:09:49,362 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:09:54,363 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:09:57,113 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:09:57,116 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:09:57,116 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:09:57,366 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:09:59,367 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:10:04,367 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:10:09,368 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:10:12,113 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:10:12,116 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:10:12,116 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:10:14,385 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:10:19,385 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:10:24,386 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:10:27,113 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:10:27,116 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:10:27,116 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:10:27,325 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:10:30,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:10:35,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:10:40,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:10:42,114 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:10:42,116 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:10:42,116 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:10:46,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:10:51,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:10:56,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:10:57,114 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:10:57,116 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:10:57,116 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:10:57,372 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:11:01,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:11:06,374 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:11:11,375 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:11:12,114 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:11:12,116 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:11:12,116 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:11:17,305 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:11:22,306 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:11:27,114 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:11:27,116 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:11:27,116 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:11:27,317 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:11:27,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:11:32,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:11:37,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:11:42,114 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:11:42,116 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:11:42,116 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:11:42,353 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:11:47,353 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:11:52,354 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:11:57,114 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:11:57,116 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:11:57,117 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:11:57,360 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:11:57,361 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:12:02,361 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:12:07,362 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:12:12,114 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:12:12,117 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:12:12,117 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:12:12,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:12:17,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:12:22,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:12:27,114 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:12:27,117 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:12:27,117 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:12:27,356 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:12:28,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:12:33,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:12:38,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:12:42,114 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:12:42,117 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:12:42,117 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:12:44,337 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:12:49,337 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:12:54,338 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:12:57,114 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:12:57,117 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:12:57,117 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:12:57,349 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:12:59,350 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:13:04,350 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:13:09,351 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:13:12,115 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:13:12,117 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:13:12,117 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:13:14,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:13:19,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:13:24,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:13:27,115 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:13:27,117 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:13:27,117 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:13:27,352 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:13:30,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:13:35,353 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:13:40,354 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:13:42,115 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:13:42,117 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:13:42,117 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:13:45,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:13:46,672 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 01:13:46,673 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 01:13:46,674 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 01:13:46,676 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 01:13:47,487 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 01:13:48,488 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 01:13:50,677 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:13:55,677 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:13:57,115 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:13:57,117 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:13:57,117 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:13:57,357 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:14:01,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:14:06,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:14:11,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:14:12,115 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:14:12,117 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:14:12,117 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:14:17,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:14:22,329 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:14:27,115 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:14:27,117 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:14:27,118 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:14:27,300 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:14:28,300 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:14:33,301 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:14:38,301 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:14:42,115 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:14:42,118 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:14:42,118 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:14:44,288 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:14:49,289 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:14:54,290 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:14:57,115 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:14:57,118 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:14:57,118 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:14:57,379 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:14:59,379 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:15:04,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:15:09,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:15:12,115 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:15:12,118 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:15:12,118 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:15:15,378 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:15:20,379 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:15:25,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:15:27,115 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:15:27,118 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:15:27,118 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:15:27,348 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:15:31,348 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:15:36,349 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:15:41,350 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:15:42,115 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:15:42,118 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:15:42,118 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:15:47,292 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:15:52,293 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:15:57,116 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:15:57,118 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:15:57,118 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:15:57,290 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:15:58,290 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:16:03,291 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:16:08,292 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:16:12,116 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:16:12,118 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:16:12,118 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:16:13,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:16:18,384 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:16:23,384 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:16:27,116 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:16:27,118 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:16:27,118 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:16:27,309 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:16:29,310 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:16:34,310 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:16:39,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:16:42,116 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:16:42,118 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:16:42,118 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:16:44,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:16:49,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:16:54,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:16:57,116 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:16:57,118 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:16:57,119 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:16:57,333 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:17:00,333 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:17:05,334 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:17:10,335 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:17:12,116 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:17:12,118 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:17:12,119 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:17:15,349 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:17:20,350 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:17:25,350 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:17:27,116 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:17:27,119 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:17:27,119 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:17:27,330 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:17:31,331 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:17:36,332 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:17:41,333 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:17:42,116 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:17:42,119 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:17:42,119 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:17:46,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:17:51,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:17:56,337 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:17:57,116 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:17:57,119 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:17:57,119 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:17:57,356 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:18:01,356 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:18:06,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:18:11,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:18:12,116 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:18:12,119 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:18:12,119 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:18:17,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:18:22,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:18:27,117 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:18:27,119 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:18:27,119 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:18:27,319 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:18:28,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:18:33,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:18:38,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:18:42,117 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:18:42,119 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:18:42,119 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:18:43,668 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:18:48,669 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:18:50,700 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 01:18:50,701 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 01:18:50,702 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 01:18:50,704 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 01:18:51,656 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 01:18:52,656 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 01:18:53,705 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:18:57,117 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:18:57,119 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:18:57,119 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:18:57,340 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:18:59,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:19:04,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:19:09,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:19:12,117 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:19:12,119 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:19:12,119 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:19:14,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:19:19,345 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:19:24,345 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:19:27,117 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:19:27,119 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:19:27,120 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:19:27,306 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:19:30,306 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:19:35,307 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:19:40,308 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:19:42,117 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:19:42,120 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:19:42,120 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:19:45,310 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:19:50,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:19:55,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:19:57,117 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:19:57,120 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:19:57,120 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:19:57,452 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:20:00,453 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:20:05,453 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:20:10,454 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:20:12,117 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:20:12,120 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:20:12,120 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:20:16,387 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:20:21,387 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:20:26,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:20:27,117 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:20:27,120 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:20:27,120 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:20:27,307 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:20:32,308 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:20:37,308 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:20:42,117 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:20:42,120 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:20:42,120 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:20:42,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:20:47,337 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:20:52,337 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:20:57,117 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:20:57,120 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:20:57,120 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:20:57,318 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:20:58,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:21:03,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:21:08,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:21:12,118 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:21:12,120 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:21:12,120 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:21:13,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:21:18,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:21:23,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:21:27,118 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:21:27,120 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:21:27,121 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:21:27,359 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:21:28,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:21:33,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:21:38,361 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:21:42,118 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:21:42,120 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:21:42,121 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:21:43,551 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:21:48,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:21:53,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:21:57,118 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:21:57,121 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:21:57,121 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:21:57,389 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:21:59,391 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:22:04,391 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:22:09,392 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:22:12,118 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:22:12,121 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:22:12,121 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:22:15,366 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:22:20,367 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:22:25,367 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:22:27,118 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:22:27,121 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:22:27,121 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:22:27,362 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:22:31,363 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:22:36,364 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:22:41,364 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:22:42,118 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:22:42,121 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:22:42,121 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:22:47,294 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:22:52,295 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:22:57,118 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:22:57,121 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:22:57,121 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:22:57,365 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:22:57,365 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:23:02,366 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:23:07,366 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:23:12,119 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:23:12,121 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:23:12,121 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:23:12,462 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:23:17,463 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:23:22,463 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:23:27,119 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:23:27,121 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:23:27,121 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:23:27,303 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:23:28,304 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:23:33,304 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:23:38,305 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:23:42,119 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:23:42,121 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:23:42,121 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:23:43,345 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:23:48,346 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:23:53,346 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:23:54,705 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 01:23:54,710 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 01:23:54,710 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 01:23:54,713 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 01:23:54,830 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 01:23:57,119 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:23:57,121 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:23:57,121 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:23:57,322 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:23:57,832 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 01:23:59,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:24:04,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:24:09,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:24:12,119 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:24:12,121 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:24:12,122 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:24:14,345 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:24:19,345 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:24:24,346 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:24:27,119 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:24:27,121 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:24:27,122 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:24:27,379 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:24:29,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:24:34,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:24:39,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:24:42,121 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:24:42,122 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:24:42,123 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:24:45,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:24:50,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:24:55,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:24:57,122 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:24:57,122 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:24:57,122 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:24:57,351 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:25:00,351 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:25:05,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:25:10,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:25:12,121 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:25:12,122 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:25:12,122 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:25:15,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:25:20,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:25:25,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:25:27,121 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:25:27,122 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:25:27,122 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:25:27,327 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:25:31,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:25:36,329 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:25:41,329 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:25:42,122 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:25:42,122 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:25:42,123 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:25:46,390 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:25:51,391 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:25:56,392 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:25:57,122 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:25:57,122 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:25:57,123 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:25:57,303 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:26:02,304 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:26:07,305 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:26:12,122 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:26:12,123 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:26:12,123 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:26:12,333 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:26:17,334 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:26:22,334 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:26:27,122 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:26:27,123 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:26:27,123 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:26:27,406 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:26:27,407 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:26:32,407 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:26:37,408 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:26:42,122 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:26:42,123 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:26:42,123 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:26:43,351 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:26:48,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:26:53,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:26:57,122 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:26:57,123 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:26:57,123 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:26:57,364 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:26:58,364 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:27:03,365 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:27:08,366 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:27:12,122 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:27:12,123 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:27:12,123 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:27:14,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:27:19,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:27:24,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:27:27,122 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:27:27,123 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:27:27,123 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:27:27,339 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:27:30,339 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:27:35,340 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:27:40,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:27:42,122 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:27:42,123 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:27:42,123 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:27:45,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:27:50,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:27:55,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:27:57,122 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:27:57,123 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:27:57,124 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:27:57,349 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:28:00,350 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:28:05,351 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:28:10,351 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:28:12,123 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:28:12,123 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:28:12,124 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:28:16,302 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:28:21,303 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:28:26,303 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:28:27,123 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:28:27,123 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:28:27,124 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:28:27,347 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:28:31,348 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:28:36,348 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:28:41,349 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:28:42,123 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:28:42,123 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:28:42,124 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:28:46,368 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:28:51,369 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:28:56,369 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:28:57,123 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:28:57,124 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:28:57,124 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:28:57,299 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:28:58,467 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 01:28:58,469 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 01:28:58,470 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 01:28:58,471 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 01:28:58,992 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 01:28:59,992 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 01:29:01,472 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:29:06,473 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:29:11,473 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:29:12,123 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:29:12,124 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:29:12,124 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:29:17,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:29:22,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:29:27,123 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:29:27,124 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:29:27,124 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:29:27,395 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:29:27,396 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:29:32,396 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:29:37,397 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:29:42,123 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:29:42,124 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:29:42,124 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:29:43,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:29:48,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:29:53,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:29:57,123 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:29:57,124 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:29:57,124 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:29:57,340 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:29:59,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:30:04,342 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:30:09,342 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:30:12,123 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:30:12,124 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:30:12,125 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:30:15,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:30:20,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:30:25,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:30:27,123 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:30:27,124 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:30:27,125 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:30:27,313 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:30:31,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:30:36,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:30:41,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:30:42,123 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:30:42,124 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:30:42,124 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:30:47,296 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:30:52,296 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:30:57,124 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:30:57,125 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:30:57,125 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:30:57,312 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:30:57,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:31:02,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:31:07,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:31:12,124 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:31:12,125 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:31:12,125 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:31:12,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:31:17,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:31:22,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:31:27,124 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:31:27,125 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:31:27,125 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:31:27,349 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:31:27,349 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:31:32,350 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:31:37,351 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:31:42,124 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:31:42,125 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:31:42,125 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:31:42,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:31:47,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:31:52,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:31:57,124 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:31:57,230 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:31:57,231 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:31:57,454 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:31:57,455 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:32:02,455 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:32:07,456 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:32:12,124 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:32:12,125 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:32:12,125 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:32:13,335 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:32:18,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:32:23,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:32:27,124 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:32:27,125 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:32:27,125 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:32:27,382 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:32:28,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:32:33,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:32:38,384 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:32:42,124 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:32:42,125 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:32:42,126 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:32:44,368 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:32:49,368 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:32:54,369 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:32:57,214 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:32:57,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:32:57,216 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:32:57,458 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:32:59,459 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:33:04,460 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:33:09,460 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:33:12,214 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:33:12,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:33:12,215 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:33:15,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:33:20,396 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:33:25,396 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:33:27,214 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:33:27,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:33:27,215 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:33:27,402 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:33:30,403 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:33:35,404 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:33:40,404 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:33:42,214 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:33:42,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:33:42,215 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:33:45,453 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:33:50,454 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:33:55,455 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:33:57,214 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:33:57,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:33:57,215 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:33:57,410 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:34:01,411 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:34:02,971 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 01:34:02,973 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 01:34:02,974 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 01:34:02,975 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 01:34:03,155 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 01:34:04,155 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 01:34:06,976 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:34:11,977 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:34:12,214 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:34:12,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:34:12,215 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:34:17,399 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:34:22,400 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:34:27,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:34:27,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:34:27,215 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:34:27,457 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:34:27,457 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:34:32,458 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:34:37,459 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:34:42,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:34:42,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:34:42,216 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:34:43,440 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:34:48,441 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:34:53,441 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:34:57,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:34:57,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:34:57,217 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:34:57,426 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:34:59,427 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:35:04,428 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:35:09,428 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:35:12,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:35:12,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:35:12,216 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:35:14,463 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:35:19,464 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:35:24,464 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:35:27,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:35:27,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:35:27,216 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:35:27,444 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:35:30,445 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:35:35,445 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:35:40,446 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:35:42,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:35:42,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:35:42,216 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:35:46,397 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:35:51,398 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:35:56,398 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:35:57,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:35:57,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:35:57,216 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:35:57,439 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:36:01,440 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:36:06,441 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:36:11,441 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:36:12,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:36:12,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:36:12,216 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:36:16,478 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:36:21,479 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:36:26,480 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:36:27,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:36:27,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:36:27,216 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:36:27,450 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:36:32,451 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:36:37,452 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:36:42,215 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:36:42,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:36:42,216 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:36:42,487 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:36:47,487 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:36:52,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:36:57,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:36:57,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:36:57,216 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:36:57,476 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:36:58,477 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:37:03,477 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:37:08,478 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:37:12,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:37:12,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:37:12,217 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:37:14,451 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:37:19,451 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:37:24,452 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:37:27,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:37:27,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:37:27,217 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:37:27,474 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:37:29,475 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:37:34,475 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:37:39,476 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:37:42,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:37:42,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:37:42,217 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:37:45,460 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:37:50,461 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:37:55,461 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:37:57,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:37:57,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:37:57,217 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:37:57,492 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:38:00,493 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:38:05,494 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:38:10,494 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:38:12,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:38:12,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:38:12,217 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:38:16,443 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:38:21,444 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:38:26,445 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:38:27,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:38:27,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:38:27,217 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:38:27,485 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:38:31,486 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:38:36,487 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:38:41,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:38:42,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:38:42,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:38:42,217 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:38:47,472 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:38:52,473 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:38:57,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:38:57,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:38:57,217 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:38:57,488 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:38:57,489 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:39:02,489 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:39:07,068 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 01:39:07,070 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 01:39:07,070 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 01:39:07,072 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 01:39:07,321 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 01:39:08,073 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:39:08,321 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 01:39:12,216 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:39:12,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:39:12,217 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:39:13,444 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:39:18,445 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:39:23,445 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:39:27,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:39:27,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:39:27,217 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:39:27,396 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:39:29,397 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:39:34,397 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:39:39,398 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:39:42,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:39:42,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:39:42,217 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:39:44,477 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:39:49,478 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:39:54,478 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:39:57,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:39:57,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:39:57,218 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:39:57,483 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:39:59,484 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:40:04,485 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:40:09,485 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:40:12,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:40:12,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:40:12,218 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:40:15,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:40:20,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:40:25,396 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:40:27,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:40:27,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:40:27,218 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:40:27,492 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:40:30,493 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:40:35,494 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:40:40,494 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:40:42,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:40:42,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:40:42,218 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:40:46,444 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:40:51,445 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:40:56,445 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:40:57,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:40:57,218 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:40:57,218 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:40:57,490 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:41:01,491 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:41:06,492 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:41:11,492 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:41:12,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:41:12,218 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:41:12,218 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:41:17,429 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:41:22,429 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:41:27,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:41:27,218 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:41:27,218 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:41:27,457 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:41:27,457 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:41:32,458 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:41:37,458 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:41:42,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:41:42,218 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:41:42,218 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:41:43,432 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:41:48,433 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:41:53,433 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:41:57,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:41:57,218 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:41:57,218 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:41:57,394 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:41:59,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:42:04,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:42:09,396 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:42:12,217 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:42:12,218 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:42:12,218 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:42:14,418 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:42:19,418 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:42:24,419 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:42:27,218 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:42:27,218 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:42:27,218 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:42:27,752 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:42:29,753 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:42:34,753 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:42:39,754 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:42:42,218 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:42:42,218 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:42:42,219 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:42:45,455 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:42:50,455 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:42:55,456 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:42:57,218 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:42:57,218 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:42:57,219 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:42:57,404 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:43:01,404 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:43:06,405 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:43:11,405 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:43:12,218 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:43:12,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:43:12,219 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:43:17,402 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:43:22,402 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:43:27,218 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:43:27,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:43:27,219 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:43:27,485 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:43:27,485 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:43:32,486 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:43:37,487 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:43:42,218 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:43:42,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:43:42,219 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:43:43,419 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:43:48,420 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:43:53,420 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:43:57,218 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:43:57,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:43:57,219 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:43:57,421 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:43:58,422 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:44:03,423 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:44:08,423 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:44:10,984 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 01:44:10,986 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 01:44:10,987 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 01:44:10,989 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 01:44:11,484 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 01:44:12,218 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:44:12,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:44:12,219 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:44:12,485 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 01:44:13,481 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:44:18,482 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:44:23,482 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:44:27,218 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:44:27,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:44:27,219 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:44:27,419 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:44:29,419 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:44:34,420 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:44:39,421 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:44:42,218 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:44:42,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:44:42,219 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:44:44,510 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:44:49,511 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:44:54,511 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:44:57,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:44:57,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:44:57,219 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:44:57,469 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:45:00,470 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:45:05,471 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:45:10,471 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:45:12,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:45:12,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:45:12,219 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:45:16,448 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:45:21,449 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:45:26,450 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:45:27,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:45:27,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:45:27,219 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:45:27,428 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:45:32,429 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:45:37,430 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:45:42,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:45:42,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:45:42,220 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:45:42,461 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:45:47,462 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:45:52,462 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:45:57,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:45:57,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:45:57,220 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:45:57,494 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:45:57,494 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:46:02,495 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:46:07,495 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:46:12,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:46:12,220 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:46:12,220 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:46:13,442 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:46:18,443 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:46:23,444 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:46:27,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:46:27,220 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:46:27,220 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:46:27,412 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:46:29,413 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:46:34,414 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:46:39,415 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:46:42,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:46:42,220 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:46:42,220 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:46:44,466 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:46:49,466 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:46:54,467 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:46:57,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:46:57,220 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:46:57,220 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:46:57,392 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:47:00,393 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:47:05,394 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:47:10,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:47:12,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:47:12,220 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:47:12,220 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:47:15,459 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:47:20,460 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:47:25,460 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:47:27,219 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:47:27,220 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:47:27,220 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:47:27,384 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:47:31,385 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:47:36,386 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:47:41,386 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:47:42,220 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:47:42,220 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:47:42,220 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:47:46,449 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:47:51,450 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:47:56,451 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:47:57,220 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:47:57,220 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:47:57,220 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:47:57,465 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:48:01,466 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:48:06,467 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:48:11,467 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:48:12,220 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:48:12,220 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:48:12,220 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:48:17,422 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:48:22,422 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:48:27,220 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:48:27,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:48:27,221 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:48:27,490 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:48:27,491 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:48:32,492 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:48:37,492 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:48:42,220 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:48:42,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:48:42,221 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:48:43,424 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:48:48,425 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:48:53,425 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:48:57,220 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:48:57,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:48:57,221 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:48:57,402 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:48:59,403 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:49:04,403 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:49:09,404 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:49:12,220 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:49:12,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:49:12,221 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:49:14,408 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:49:15,027 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 01:49:15,053 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 01:49:15,054 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 01:49:15,055 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 01:49:15,650 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 01:49:16,651 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 01:49:20,056 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:49:25,057 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:49:27,220 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:49:27,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:49:27,221 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:49:27,489 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:49:30,489 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:49:35,490 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:49:40,491 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:49:42,220 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:49:42,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:49:42,221 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:49:46,408 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:49:51,409 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:49:56,409 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:49:57,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:49:57,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:49:57,221 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:49:57,465 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:50:01,466 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:50:06,466 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:50:11,467 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:50:12,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:50:12,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:50:12,221 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:50:17,424 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:50:22,424 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:50:27,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:50:27,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:50:27,222 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:50:27,477 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:50:27,477 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:50:32,478 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:50:37,478 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:50:42,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:50:42,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:50:42,222 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:50:43,465 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:50:48,465 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:50:53,466 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:50:57,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:50:57,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:50:57,222 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:50:57,443 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:50:59,444 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:51:04,445 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:51:09,445 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:51:12,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:51:12,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:51:12,222 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:51:14,502 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:51:19,503 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:51:24,504 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:51:27,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:51:27,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:51:27,222 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:51:27,414 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:51:30,415 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:51:35,415 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:51:40,416 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:51:42,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:51:42,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:51:42,222 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:51:45,479 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:51:50,479 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:51:55,480 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:51:57,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:51:57,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:51:57,222 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:51:57,437 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:52:01,438 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:52:06,438 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:52:11,439 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:52:12,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:52:12,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:52:12,222 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:52:17,430 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:52:22,431 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:52:27,221 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:52:27,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:52:27,222 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:52:27,487 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:52:27,487 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:52:32,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:52:37,489 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:52:42,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:52:42,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:52:42,223 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:52:43,415 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:52:48,416 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:52:53,416 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:52:57,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:52:57,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:52:57,223 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:52:57,499 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:52:58,500 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:53:03,501 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:53:08,501 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:53:12,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:53:12,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:53:12,223 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:53:14,493 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:53:19,494 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:53:24,494 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:53:27,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:53:27,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:53:27,223 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:53:27,454 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:53:30,455 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:53:35,456 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:53:40,456 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:53:42,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:53:42,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:53:42,223 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:53:45,481 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:53:50,482 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:53:55,483 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:53:57,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:53:57,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:53:57,223 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:53:57,479 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:54:01,480 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:54:06,481 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:54:11,481 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:54:12,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:54:12,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:54:12,223 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:54:17,465 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:54:19,559 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 01:54:19,562 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 01:54:19,563 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 01:54:19,564 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 01:54:19,815 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 01:54:21,816 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 01:54:22,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:54:27,222 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:54:27,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:54:27,223 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:54:27,396 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:54:28,396 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:54:33,397 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:54:38,398 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:54:42,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:54:42,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:54:42,223 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:54:43,428 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:54:48,428 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:54:53,429 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:54:57,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:54:57,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:54:57,224 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:54:57,542 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:54:58,543 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:55:03,543 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:55:08,544 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:55:12,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:55:12,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:55:12,224 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:55:14,461 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:55:19,462 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:55:24,462 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:55:27,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:55:27,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:55:27,224 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:55:27,414 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:55:30,415 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:55:35,415 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:55:40,416 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:55:42,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:55:42,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:55:42,224 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:55:45,471 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:55:50,471 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:55:55,472 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:55:57,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:55:57,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:55:57,224 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:55:57,483 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:56:00,484 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:56:05,485 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:56:10,485 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:56:12,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:56:12,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:56:12,224 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:56:15,500 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:56:20,501 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:56:25,501 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:56:27,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:56:27,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:56:27,224 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:56:27,459 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:56:31,459 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:56:36,460 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:56:41,460 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:56:42,223 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:56:42,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:56:42,224 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:56:46,504 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:56:51,505 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:56:56,505 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:56:57,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:56:57,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:56:57,224 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:56:57,497 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:57:02,498 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:57:07,499 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:57:12,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:57:12,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:57:12,224 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:57:13,427 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:57:18,428 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:57:23,428 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:57:27,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:57:27,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:57:27,224 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:57:27,422 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:57:29,423 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:57:34,423 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:57:39,424 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:57:42,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:57:42,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:57:42,225 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:57:44,452 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:57:49,452 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:57:54,453 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:57:57,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:57:57,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:57:57,225 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:57:57,507 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:57:59,508 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:58:04,508 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:58:09,509 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:58:12,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:58:12,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:58:12,225 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:58:15,410 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:58:20,411 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:58:25,412 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:58:27,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:58:27,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:58:27,225 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:58:27,465 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:58:30,466 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:58:35,467 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:58:40,467 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:58:42,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:58:42,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:58:42,225 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:58:45,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:58:50,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:58:55,540 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:58:57,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:58:57,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:58:57,225 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:58:57,440 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:59:01,441 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:59:06,442 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:59:11,442 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:59:12,224 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:59:12,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:59:12,225 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:59:16,504 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:59:21,504 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:59:23,640 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 01:59:23,642 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 01:59:23,643 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 01:59:23,645 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 01:59:23,977 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 01:59:25,978 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 01:59:26,646 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:59:27,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:59:27,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:59:27,225 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:59:27,420 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:59:32,421 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:59:37,422 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:59:42,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:59:42,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:59:42,225 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:59:42,468 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:59:47,468 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:59:52,469 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 01:59:57,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 01:59:57,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 01:59:57,226 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 01:59:57,494 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 01:59:57,494 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:00:02,495 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:00:07,496 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:00:12,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:00:12,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:00:12,226 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:00:13,436 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:00:18,437 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:00:23,437 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:00:27,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:00:27,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:00:27,226 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:00:27,465 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:00:28,465 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:00:33,466 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:00:38,467 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:00:42,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:00:42,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:00:42,226 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:00:43,503 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:00:48,503 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:00:53,504 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:00:57,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:00:57,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:00:57,226 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:00:57,421 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:00:59,422 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:01:04,423 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:01:09,423 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:01:12,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:01:12,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:01:12,226 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:01:14,447 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:01:19,448 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:01:24,449 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:01:27,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:01:27,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:01:27,226 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:01:27,416 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:01:30,417 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:01:35,418 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:01:40,418 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:01:42,225 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:01:42,226 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:01:42,226 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:01:45,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:01:50,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:01:55,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:01:57,289 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:01:57,310 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:01:57,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:01:57,312 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:02:01,512 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:02:06,512 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:02:11,513 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:02:12,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:02:12,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:02:12,311 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:02:16,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:02:21,548 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:02:26,548 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:02:27,292 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:02:27,310 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:02:27,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:02:27,311 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:02:31,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:02:36,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:02:41,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:02:42,310 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:02:42,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:02:42,311 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:02:47,531 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:02:52,532 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:02:57,292 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:02:57,310 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:02:57,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:02:57,311 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:02:57,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:03:02,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:03:07,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:03:12,310 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:03:12,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:03:12,311 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:03:13,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:03:18,553 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:03:23,553 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:03:27,293 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:03:27,310 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:03:27,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:03:27,311 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:03:29,549 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:03:34,549 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:03:39,550 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:03:42,310 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:03:42,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:03:42,311 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:03:44,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:03:49,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:03:54,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:03:57,293 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:03:57,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:03:57,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:03:57,312 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:04:00,528 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:04:05,528 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:04:10,529 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:04:12,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:04:12,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:04:12,312 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:04:15,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:04:20,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:04:25,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:04:27,294 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:04:27,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:04:27,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:04:27,312 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:04:27,512 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 02:04:27,515 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 02:04:27,515 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 02:04:27,516 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 02:04:28,140 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 02:04:30,141 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 02:04:31,517 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:04:36,518 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:04:41,519 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:04:42,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:04:42,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:04:42,312 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:04:46,536 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:04:51,537 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:04:56,537 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:04:57,295 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:04:57,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:04:57,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:04:57,312 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:05:01,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:05:06,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:05:11,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:05:12,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:05:12,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:05:12,313 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:05:17,509 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:05:22,510 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:05:27,296 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:05:27,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:05:27,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:05:27,313 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:05:27,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:05:32,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:05:37,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:05:42,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:05:42,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:05:42,313 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:05:43,544 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:05:48,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:05:53,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:05:57,296 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:05:57,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:05:57,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:05:57,313 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:05:58,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:06:03,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:06:08,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:06:12,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:06:12,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:06:12,313 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:06:14,501 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:06:19,502 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:06:24,503 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:06:27,297 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:06:27,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:06:27,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:06:27,313 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:06:29,531 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:06:34,532 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:06:39,532 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:06:42,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:06:42,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:06:42,313 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:06:45,528 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:06:50,529 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:06:55,529 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:06:57,298 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:06:57,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:06:57,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:06:57,313 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:07:00,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:07:05,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:07:10,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:07:12,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:07:12,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:07:12,313 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:07:15,595 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:07:20,595 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:07:25,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:07:27,299 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:07:27,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:07:27,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:07:27,313 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:07:31,503 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:07:36,504 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:07:41,504 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:07:42,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:07:42,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:07:42,314 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:07:47,497 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:07:52,498 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:07:57,300 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:07:57,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:07:57,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:07:57,314 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:07:57,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:08:02,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:08:07,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:08:12,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:08:12,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:08:12,314 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:08:12,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:08:17,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:08:22,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:08:27,301 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:08:27,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:08:27,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:08:27,314 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:08:28,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:08:33,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:08:38,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:08:42,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:08:42,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:08:42,314 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:08:44,544 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:08:49,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:08:54,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:08:57,302 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:08:57,312 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:08:57,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:08:57,314 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:09:00,538 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:09:05,538 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:09:10,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:09:12,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:09:12,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:09:12,314 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:09:16,528 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:09:21,528 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:09:26,529 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:09:27,304 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:09:27,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:09:27,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:09:27,314 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:09:31,371 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 02:09:31,374 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 02:09:31,374 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 02:09:31,375 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 02:09:32,299 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 02:09:32,300 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 02:09:32,376 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:09:37,376 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:09:42,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:09:42,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:09:42,314 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:09:42,519 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:09:47,520 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:09:52,520 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:09:57,305 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:09:57,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:09:57,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:09:57,315 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:09:57,524 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:10:02,525 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:10:07,525 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:10:12,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:10:12,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:10:12,315 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:10:12,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:10:17,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:10:22,593 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:10:27,307 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:10:27,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:10:27,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:10:27,315 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:10:27,632 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:10:32,633 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:10:37,634 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:10:42,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:10:42,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:10:42,315 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:10:43,527 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:10:48,528 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:10:53,528 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:10:57,307 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:10:57,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:10:57,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:10:57,315 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:10:59,518 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:11:04,519 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:11:09,519 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:11:12,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:11:12,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:11:12,315 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:11:15,505 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:11:20,506 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:11:25,506 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:11:27,308 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:11:27,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:11:27,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:11:27,315 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:11:30,591 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:11:35,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:11:40,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:11:42,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:11:42,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:11:42,315 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:11:46,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:11:51,540 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:11:56,541 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:11:57,309 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:11:57,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:11:57,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:11:57,315 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:12:01,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:12:06,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:12:11,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:12:12,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:12:12,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:12:12,316 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:12:17,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:12:22,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:12:27,310 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:12:27,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:12:27,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:12:27,316 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:12:28,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:12:33,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:12:38,548 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:12:42,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:12:42,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:12:42,316 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:12:44,535 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:12:49,535 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:12:54,536 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:12:57,311 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:12:57,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:12:57,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:12:57,316 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:13:00,504 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:13:05,505 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:13:10,506 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:13:12,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:13:12,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:13:12,316 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:13:15,527 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:13:20,528 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:13:25,528 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:13:27,313 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:13:27,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:13:27,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:13:27,316 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:13:30,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:13:35,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:13:40,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:13:42,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:13:42,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:13:42,316 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:13:46,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:13:51,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:13:56,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:13:57,312 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:13:57,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:13:57,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:13:57,316 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:14:02,502 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:14:07,503 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:14:12,314 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:14:12,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:14:12,316 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:14:12,533 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:14:17,533 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:14:22,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:14:27,314 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:14:27,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:14:27,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:14:27,316 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:14:28,490 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:14:33,490 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:14:35,795 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 02:14:35,798 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 02:14:35,798 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 02:14:35,799 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 02:14:36,463 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 02:14:36,463 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 02:14:38,800 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:14:42,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:14:42,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:14:42,317 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:14:44,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:14:49,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:14:54,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:14:57,314 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:14:57,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:14:57,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:14:57,317 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:15:00,497 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:15:05,497 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:15:10,498 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:15:12,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:15:12,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:15:12,317 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:15:15,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:15:20,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:15:25,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:15:27,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:15:27,315 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:15:27,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:15:27,318 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:15:30,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:15:35,597 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:15:40,598 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:15:42,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:15:42,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:15:42,317 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:15:46,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:15:51,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:15:56,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:15:57,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:15:57,317 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:15:57,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:15:57,319 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:16:02,544 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:16:07,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:16:12,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:16:12,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:16:12,317 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:16:12,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:16:17,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:16:22,590 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:16:27,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:16:27,317 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:16:27,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:16:27,318 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:16:28,509 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:16:33,510 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:16:38,510 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:16:42,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:16:42,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:16:42,317 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:16:44,506 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:16:49,506 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:16:54,507 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:16:57,315 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:16:57,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:16:57,318 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:16:57,320 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:17:00,500 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:17:05,501 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:17:10,502 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:17:12,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:17:12,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:17:12,317 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:17:15,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:17:20,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:17:25,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:17:27,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:17:27,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:17:27,319 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:17:27,319 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:17:31,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:17:36,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:17:41,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:17:42,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:17:42,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:17:42,318 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:17:47,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:17:52,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:17:57,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:17:57,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:17:57,318 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:17:57,587 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:17:57,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:18:02,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:18:07,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:18:12,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:18:12,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:18:12,318 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:18:13,497 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:18:18,497 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:18:23,498 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:18:27,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:18:27,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:18:27,318 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:18:27,508 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:18:28,508 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:18:33,509 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:18:38,510 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:18:42,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:18:42,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:18:42,318 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:18:43,543 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:18:48,543 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:18:53,544 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:18:57,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:18:57,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:18:57,318 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:18:57,578 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:18:58,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:19:03,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:19:08,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:19:12,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:19:12,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:19:12,318 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:19:14,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:19:19,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:19:24,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:19:27,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:19:27,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:19:27,318 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:19:27,514 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:19:30,514 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:19:35,515 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:19:39,671 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 02:19:39,673 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 02:19:39,673 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 02:19:39,674 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 02:19:40,632 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 02:19:40,632 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 02:19:40,713 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:19:42,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:19:42,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:19:42,318 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:19:46,590 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:19:51,591 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:19:56,591 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:19:57,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:19:57,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:19:57,319 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:19:57,585 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:20:02,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:20:07,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:20:12,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:20:12,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:20:12,319 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:20:13,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:20:18,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:20:23,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:20:27,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:20:27,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:20:27,319 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:20:27,535 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:20:29,535 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:20:34,536 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:20:39,537 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:20:42,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:20:42,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:20:42,319 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:20:44,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:20:49,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:20:54,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:20:57,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:20:57,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:20:57,319 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:20:57,562 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:21:00,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:21:05,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:21:10,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:21:12,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:21:12,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:21:12,319 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:21:15,605 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:21:20,605 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:21:25,606 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:21:27,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:21:27,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:21:27,320 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:21:27,918 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:21:30,919 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:21:35,920 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:21:40,920 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:21:42,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:21:42,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:21:42,320 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:21:46,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:21:51,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:21:56,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:21:57,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:21:57,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:21:57,320 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:21:57,553 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:22:02,554 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:22:07,554 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:22:12,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:22:12,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:22:12,320 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:22:12,749 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:22:17,750 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:22:22,750 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:22:27,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:22:27,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:22:27,320 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:22:27,543 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:22:28,544 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:22:33,544 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:22:38,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:22:42,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:22:42,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:22:42,320 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:22:43,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:22:48,577 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:22:53,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:22:57,318 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:22:57,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:22:57,320 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:22:57,534 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:22:59,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:23:04,535 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:23:09,536 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:23:12,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:23:12,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:23:12,320 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:23:15,515 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:23:20,516 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:23:25,517 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:23:27,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:23:27,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:23:27,320 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:23:27,582 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:23:30,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:23:35,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:23:40,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:23:42,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:23:42,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:23:42,320 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:23:45,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:23:50,590 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:23:55,591 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:23:57,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:23:57,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:23:57,321 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:23:58,238 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:24:01,239 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:24:06,240 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:24:11,240 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:24:12,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:24:12,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:24:12,321 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:24:16,590 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:24:21,591 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:24:26,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:24:27,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:24:27,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:24:27,321 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:24:27,867 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:24:31,868 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:24:36,868 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:24:41,869 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:24:42,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:24:42,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:24:42,321 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:24:43,640 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 02:24:43,643 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 02:24:43,643 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 02:24:43,645 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 02:24:43,800 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 02:24:45,801 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 02:24:47,646 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:24:52,646 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:24:57,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:24:57,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:24:57,321 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:24:57,565 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:24:58,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:25:03,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:25:08,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:25:12,319 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:25:12,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:25:12,321 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:25:14,509 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:25:19,510 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:25:24,510 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:25:27,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:25:27,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:25:27,321 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:25:27,555 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:25:29,556 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:25:34,557 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:25:39,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:25:42,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:25:42,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:25:42,321 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:25:44,599 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:25:49,600 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:25:54,600 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:25:57,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:25:57,322 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:25:57,322 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:25:57,528 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:26:00,529 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:26:05,530 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:26:10,530 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:26:12,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:26:12,322 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:26:12,322 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:26:15,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:26:20,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:26:25,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:26:27,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:26:27,322 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:26:27,322 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:26:27,596 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:26:30,597 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:26:35,598 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:26:40,598 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:26:42,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:26:42,322 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:26:42,322 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:26:46,499 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:26:51,499 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:26:56,500 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:26:57,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:26:57,322 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:26:57,322 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:26:57,614 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:27:01,615 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:27:06,615 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:27:11,616 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:27:12,320 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:27:12,322 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:27:12,322 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:27:17,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:27:22,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:27:27,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:27:27,322 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:27:27,322 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:27:27,611 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:27:27,611 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:27:32,612 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:27:37,613 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:27:42,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:27:42,322 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:27:42,323 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:27:43,530 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:27:48,531 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:27:53,531 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:27:57,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:27:57,322 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:27:57,323 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:27:57,506 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:27:59,507 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:28:04,507 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:28:09,508 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:28:12,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:28:12,322 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:28:12,323 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:28:14,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:28:19,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:28:24,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:28:27,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:28:27,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:28:27,323 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:28:27,522 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:28:30,523 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:28:35,523 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:28:40,524 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:28:42,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:28:42,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:28:42,323 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:28:45,594 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:28:50,595 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:28:55,595 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:28:57,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:28:57,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:28:57,323 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:28:57,530 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:29:01,531 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:29:06,532 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:29:11,532 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:29:12,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:29:12,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:29:12,323 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:29:16,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:29:21,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:29:26,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:29:27,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:29:27,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:29:27,323 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:29:27,628 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:29:31,629 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:29:36,630 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:29:41,630 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:29:42,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:29:42,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:29:42,323 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:29:47,171 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 02:29:47,174 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 02:29:47,174 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 02:29:47,175 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:29:47,176 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 02:29:47,962 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 02:29:49,963 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 02:29:52,177 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:29:57,177 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:29:57,321 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:29:57,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:29:57,323 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:29:57,590 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:30:02,591 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:30:07,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:30:12,322 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:30:12,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:30:12,323 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:30:13,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:30:18,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:30:23,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:30:27,322 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:30:27,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:30:27,324 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:30:27,569 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:30:29,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:30:34,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:30:39,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:30:42,322 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:30:42,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:30:42,324 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:30:44,597 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:30:49,598 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:30:54,599 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:30:57,322 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:30:57,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:30:57,324 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:30:57,577 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:31:00,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:31:05,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:31:10,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:31:12,322 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:31:12,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:31:12,324 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:31:15,601 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:31:20,602 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:31:25,603 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:31:27,322 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:31:27,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:31:27,324 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:31:27,578 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:31:31,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:31:36,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:31:41,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:31:42,322 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:31:42,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:31:42,324 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:31:47,499 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:31:52,500 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:31:57,322 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:31:57,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:31:57,324 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:31:57,548 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:31:57,548 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:32:02,549 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:32:07,550 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:32:12,322 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:32:12,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:32:12,324 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:32:12,686 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:32:17,687 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:32:22,687 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:32:27,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:32:27,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:32:27,324 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:32:27,512 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:32:28,512 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:32:33,513 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:32:38,513 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:32:42,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:32:42,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:32:42,324 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:32:43,549 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:32:48,549 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:32:53,550 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:32:57,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:32:57,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:32:57,324 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:32:57,527 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:32:59,528 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:33:04,528 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:33:09,529 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:33:12,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:33:12,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:33:12,325 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:33:14,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:33:19,597 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:33:24,597 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:33:27,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:33:27,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:33:27,325 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:33:27,543 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:33:30,544 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:33:35,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:33:40,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:33:42,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:33:42,325 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:33:42,325 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:33:45,591 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:33:50,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:33:55,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:33:57,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:33:57,325 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:33:57,325 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:33:57,567 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:34:01,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:34:06,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:34:11,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:34:12,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:34:12,325 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:34:12,325 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:34:17,553 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:34:22,554 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:34:27,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:34:27,325 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:34:27,325 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:34:27,581 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:34:27,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:34:32,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:34:37,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:34:42,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:34:42,325 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:34:42,325 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:34:42,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:34:47,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:34:50,831 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 02:34:50,834 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 02:34:50,834 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 02:34:50,835 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 02:34:51,143 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 02:34:52,143 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 02:34:52,836 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:34:57,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:34:57,325 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:34:57,325 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:34:57,528 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:34:58,529 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:35:03,529 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:35:08,530 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:35:12,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:35:12,325 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:35:12,325 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:35:14,505 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:35:19,506 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:35:24,506 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:35:27,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:35:27,325 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:35:27,325 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:35:27,527 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:35:29,527 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:35:34,528 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:35:39,529 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:35:42,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:35:42,325 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:35:42,326 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:35:44,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:35:49,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:35:54,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:35:57,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:35:57,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:35:57,326 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:35:57,509 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:36:00,510 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:36:05,510 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:36:10,511 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:36:12,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:36:12,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:36:12,326 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:36:15,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:36:20,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:36:25,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:36:27,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:36:27,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:36:27,326 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:36:27,546 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:36:31,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:36:36,548 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:36:41,548 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:36:42,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:36:42,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:36:42,326 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:36:46,549 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:36:51,549 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:36:56,550 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:36:57,325 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:36:57,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:36:57,326 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:36:57,559 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:37:01,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:37:06,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:37:11,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:37:12,325 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:37:12,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:37:12,327 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:37:17,525 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:37:22,526 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:37:27,325 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:37:27,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:37:27,326 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:37:27,533 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:37:27,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:37:32,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:37:37,535 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:37:42,325 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:37:42,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:37:42,326 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:37:42,604 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:37:47,605 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:37:52,605 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:37:57,325 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:37:57,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:37:57,327 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:37:57,523 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:37:58,524 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:38:03,524 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:38:08,525 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:38:12,325 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:38:12,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:38:12,327 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:38:13,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:38:18,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:38:23,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:38:27,325 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:38:27,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:38:27,327 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:38:27,559 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:38:29,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:38:34,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:38:39,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:38:42,325 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:38:42,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:38:42,327 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:38:45,519 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:38:50,520 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:38:55,520 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:38:57,325 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:38:57,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:38:57,327 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:38:59,384 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:39:01,385 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:39:06,386 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:39:11,387 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:39:12,325 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:39:12,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:39:12,327 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:39:16,600 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:39:21,601 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:39:26,601 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:39:27,325 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:39:27,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:39:27,327 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:39:27,567 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:39:32,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:39:37,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:39:42,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:39:42,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:39:42,328 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:39:42,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:39:47,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:39:52,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:39:55,072 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 02:39:55,075 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 02:39:55,075 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 02:39:55,077 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 02:39:55,301 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 02:39:56,301 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 02:39:57,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:39:57,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:39:57,328 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:39:57,534 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:39:58,535 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:40:03,536 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:40:08,536 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:40:12,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:40:12,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:40:12,328 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:40:14,518 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:40:19,519 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:40:24,519 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:40:27,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:40:27,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:40:27,328 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:40:27,562 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:40:29,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:40:34,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:40:39,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:40:42,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:40:42,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:40:42,328 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:40:45,513 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:40:50,513 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:40:55,514 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:40:57,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:40:57,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:40:57,328 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:40:57,573 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:41:00,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:41:05,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:41:10,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:41:12,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:41:12,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:41:12,328 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:41:15,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:41:20,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:41:25,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:41:27,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:41:27,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:41:27,328 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:41:27,612 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:41:30,613 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:41:35,613 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:41:40,614 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:41:42,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:41:42,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:41:42,328 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:41:46,515 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:41:51,516 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:41:56,516 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:41:57,326 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:41:57,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:41:57,328 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:41:57,581 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:42:01,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:42:06,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:42:11,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:42:12,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:42:12,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:42:12,329 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:42:17,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:42:22,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:42:27,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:42:27,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:42:27,329 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:42:27,549 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:42:28,550 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:42:33,550 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:42:38,551 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:42:42,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:42:42,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:42:42,329 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:42:44,519 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:42:49,520 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:42:54,520 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:42:57,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:42:57,329 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:42:57,329 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:42:57,503 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:43:00,504 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:43:05,504 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:43:10,505 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:43:12,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:43:12,329 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:43:12,329 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:43:15,606 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:43:20,607 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:43:25,607 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:43:27,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:43:27,329 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:43:27,329 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:43:27,498 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:43:31,499 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:43:36,499 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:43:41,500 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:43:42,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:43:42,329 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:43:42,329 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:43:46,599 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:43:51,599 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:43:56,600 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:43:57,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:43:57,329 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:43:57,329 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:43:57,675 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:44:01,676 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:44:06,677 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:44:11,678 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:44:12,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:44:12,329 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:44:12,329 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:44:17,594 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:44:22,595 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:44:27,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:44:27,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:44:27,330 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:44:27,533 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:44:28,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:44:33,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:44:38,535 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:44:42,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:44:42,329 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:44:42,330 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:44:44,516 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:44:49,517 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:44:54,518 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:44:57,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:44:57,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:44:57,330 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:44:57,552 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:44:59,060 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 02:44:59,062 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 02:44:59,063 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 02:44:59,064 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 02:44:59,465 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 02:45:00,064 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:45:00,465 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 02:45:05,065 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:45:10,065 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:45:12,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:45:12,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:45:12,330 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:45:15,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:45:20,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:45:25,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:45:27,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:45:27,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:45:27,330 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:45:27,573 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:45:30,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:45:35,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:45:40,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:45:42,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:45:42,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:45:42,330 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:45:45,602 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:45:50,602 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:45:55,603 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:45:57,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:45:57,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:45:57,330 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:45:57,599 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:46:01,600 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:46:06,601 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:46:11,601 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:46:12,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:46:12,331 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:46:12,331 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:46:17,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:46:22,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:46:27,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:46:27,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:46:27,330 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:46:27,715 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:46:27,715 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:46:32,716 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:46:37,717 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:46:42,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:46:42,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:46:42,331 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:46:43,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:46:48,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:46:53,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:46:57,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:46:57,331 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:46:57,331 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:46:57,596 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:46:58,597 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:47:03,597 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:47:08,598 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:47:12,328 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:47:12,331 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:47:12,331 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:47:14,590 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:47:19,591 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:47:24,591 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:47:27,329 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:47:27,331 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:47:27,331 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:47:27,594 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:47:29,595 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:47:34,595 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:47:39,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:47:42,329 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:47:42,331 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:47:42,331 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:47:44,600 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:47:49,600 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:47:54,601 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:47:57,329 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:47:57,331 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:47:57,331 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:47:57,594 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:48:00,595 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:48:05,595 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:48:10,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:48:12,329 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:48:12,331 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:48:12,331 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:48:16,522 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:48:21,523 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:48:26,524 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:48:27,329 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:48:27,331 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:48:27,331 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:48:27,553 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:48:31,555 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:48:36,555 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:48:41,556 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:48:42,329 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:48:42,331 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:48:42,331 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:48:47,551 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:48:52,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:48:57,329 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:48:57,331 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:48:57,331 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:48:57,534 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:48:58,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:49:03,535 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:49:08,536 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:49:12,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:49:12,331 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:49:12,332 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:49:13,598 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:49:18,599 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:49:23,600 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:49:27,329 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:49:27,332 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:49:27,332 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:49:27,602 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:49:28,603 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:49:33,603 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:49:38,604 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:49:42,329 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:49:42,332 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:49:42,332 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:49:44,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:49:49,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:49:54,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:49:57,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:49:57,332 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:49:57,332 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:49:57,562 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:50:00,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:50:02,719 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 02:50:02,722 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 02:50:02,722 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 02:50:02,724 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 02:50:03,629 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 02:50:04,630 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 02:50:05,725 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:50:10,725 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:50:12,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:50:12,332 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:50:12,332 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:50:16,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:50:21,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:50:26,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:50:27,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:50:27,332 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:50:27,332 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:50:27,621 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:50:31,622 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:50:36,623 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:50:41,623 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:50:42,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:50:42,332 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:50:42,332 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:50:47,601 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:50:52,601 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:50:57,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:50:57,332 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:50:57,332 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:50:57,537 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:50:58,538 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:51:03,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:51:08,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:51:12,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:51:12,332 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:51:12,332 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:51:13,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:51:18,548 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:51:23,549 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:51:27,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:51:27,332 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:51:27,333 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:51:27,565 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:51:28,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:51:33,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:51:38,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:51:42,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:51:42,333 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:51:42,333 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:51:43,601 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:51:48,602 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:51:53,603 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:51:57,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:51:57,333 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:51:57,333 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:51:57,599 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:51:59,600 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:52:04,601 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:52:09,601 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:52:12,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:52:12,333 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:52:12,333 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:52:15,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:52:20,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:52:25,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:52:27,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:52:27,333 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:52:27,333 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:52:27,618 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:52:30,618 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:52:35,619 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:52:40,619 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:52:42,331 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:52:42,333 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:52:42,333 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:52:46,519 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:52:51,520 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:52:56,520 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:52:57,331 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:52:57,333 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:52:57,333 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:52:57,511 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:53:02,512 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:53:07,513 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:53:12,331 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:53:12,333 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:53:12,333 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:53:12,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:53:17,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:53:22,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:53:27,331 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:53:27,333 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:53:27,333 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:53:27,608 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:53:27,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:53:32,609 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:53:37,609 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:53:42,331 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:53:42,333 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:53:42,333 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:53:43,532 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:53:48,533 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:53:53,533 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:53:57,331 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:53:57,333 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:53:57,333 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:53:57,514 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:53:59,515 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:54:04,515 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:54:09,516 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:54:12,331 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:54:12,333 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:54:12,334 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:54:14,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:54:19,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:54:24,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:54:27,331 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:54:27,334 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:54:27,334 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:54:27,596 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:54:29,597 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:54:34,598 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:54:39,598 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:54:42,331 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:54:42,334 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:54:42,334 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:54:45,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:54:50,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:54:55,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:54:57,331 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:54:57,334 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:54:57,334 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:54:57,560 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:55:01,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:55:06,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:55:07,127 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 02:55:07,129 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 02:55:07,130 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 02:55:07,132 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 02:55:07,812 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 02:55:08,813 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 02:55:12,133 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:55:12,331 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:55:12,334 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:55:12,334 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:55:17,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:55:22,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:55:27,332 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:55:27,334 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:55:27,334 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:55:27,575 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:55:28,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:55:33,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:55:38,577 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:55:42,332 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:55:42,334 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:55:42,334 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:55:43,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:55:48,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:55:53,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:55:57,332 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:55:57,334 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:55:57,334 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:55:57,574 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:55:59,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:56:04,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:56:09,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:56:12,332 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:56:12,335 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:56:12,335 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:56:15,525 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:56:20,526 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:56:25,526 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:56:27,332 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:56:27,335 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:56:27,335 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:56:27,575 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:56:30,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:56:35,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:56:40,577 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:56:42,332 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:56:42,335 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:56:42,335 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:56:45,699 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:56:50,700 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:56:55,700 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:56:57,332 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:56:57,335 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:56:57,335 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:56:57,568 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:57:01,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:57:06,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:57:11,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:57:12,332 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:57:12,335 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:57:12,335 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:57:16,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:57:21,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:57:26,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:57:27,332 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:57:27,335 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:57:27,335 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:57:27,604 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:57:31,605 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:57:36,606 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:57:41,606 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:57:42,332 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:57:42,335 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:57:42,335 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:57:47,519 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:57:52,520 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:57:57,332 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:57:57,335 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:57:57,335 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:57:57,568 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:57:57,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:58:02,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:58:07,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:58:12,333 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:58:12,335 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:58:12,335 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:58:12,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:58:17,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:58:22,593 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:58:27,333 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:58:27,335 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:58:27,336 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:58:27,567 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:58:28,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:58:33,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:58:38,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:58:42,333 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:58:42,335 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:58:42,336 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:58:43,606 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:58:48,607 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:58:53,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:58:57,333 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:58:57,335 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:58:57,336 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:58:57,523 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:58:59,524 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:59:04,525 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:59:09,525 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:59:12,333 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:59:12,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:59:12,336 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:59:15,512 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:59:20,513 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:59:25,513 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:59:27,334 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:59:27,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:59:27,336 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:59:27,609 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 02:59:30,610 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:59:35,611 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:59:40,611 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:59:42,334 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:59:42,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:59:42,336 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:59:45,614 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:59:50,615 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:59:55,615 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 02:59:57,334 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 02:59:57,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 02:59:57,336 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 02:59:57,620 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:00:00,621 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:00:05,622 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:00:10,623 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:00:11,003 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 03:00:11,005 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 03:00:11,006 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 03:00:11,007 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 03:00:11,996 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 03:00:12,334 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:00:12,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:00:12,336 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:00:13,997 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 03:00:16,604 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:00:21,605 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:00:26,605 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:00:27,334 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:00:27,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:00:27,336 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:00:28,624 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:00:31,625 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:00:36,626 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:00:41,627 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:00:42,334 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:00:42,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:00:42,336 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:00:47,529 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:00:52,530 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:00:57,334 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:00:57,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:00:57,336 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:00:57,538 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:00:57,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:01:02,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:01:07,540 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:01:12,334 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:01:12,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:01:12,337 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:01:12,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:01:17,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:01:22,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:01:27,334 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:01:27,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:01:27,337 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:01:27,582 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:01:27,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:01:32,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:01:37,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:01:42,334 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:01:42,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:01:42,337 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:01:42,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:01:47,621 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:01:52,621 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:01:57,334 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:01:57,337 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:01:57,337 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:01:57,586 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:01:58,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:02:03,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:02:08,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:02:12,334 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:02:12,337 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:02:12,337 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:02:14,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:02:19,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:02:24,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:02:27,335 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:02:27,337 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:02:27,337 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:02:27,562 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:02:30,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:02:35,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:02:40,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:02:42,335 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:02:42,337 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:02:42,337 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:02:45,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:02:50,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:02:55,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:02:57,335 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:02:57,337 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:02:57,337 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:02:57,532 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:03:01,533 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:03:06,533 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:03:11,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:03:12,335 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:03:12,337 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:03:12,337 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:03:16,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:03:21,548 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:03:26,549 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:03:27,335 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:03:27,337 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:03:27,337 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:03:27,564 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:03:31,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:03:36,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:03:41,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:03:42,335 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:03:42,337 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:03:42,337 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:03:46,595 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:03:51,595 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:03:56,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:03:57,335 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:03:57,339 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:03:57,339 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:03:57,544 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:04:02,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:04:07,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:04:12,335 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:04:12,339 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:04:12,339 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:04:12,603 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:04:17,604 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:04:22,604 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:04:27,335 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:04:27,339 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:04:27,339 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:04:27,610 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:04:27,611 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:04:32,611 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:04:37,612 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:04:42,335 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:04:42,339 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:04:42,339 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:04:43,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:04:48,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:04:53,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:04:57,335 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:04:57,339 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:04:57,339 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:04:57,518 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:04:59,519 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:05:04,519 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:05:09,520 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:05:12,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:05:12,339 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:05:12,339 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:05:14,550 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:05:15,235 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 03:05:15,237 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 03:05:15,238 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 03:05:15,240 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 03:05:16,163 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 03:05:18,165 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 03:05:20,241 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:05:25,242 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:05:27,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:05:27,339 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:05:27,339 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:05:27,554 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:05:30,555 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:05:35,556 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:05:40,556 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:05:42,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:05:42,339 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:05:42,340 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:05:46,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:05:51,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:05:56,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:05:57,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:05:57,340 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:05:57,340 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:05:57,522 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:06:02,524 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:06:07,524 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:06:12,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:06:12,340 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:06:12,340 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:06:12,615 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:06:17,615 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:06:22,616 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:06:27,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:06:27,340 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:06:27,340 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:06:27,550 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:06:28,551 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:06:33,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:06:38,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:06:42,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:06:42,340 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:06:42,340 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:06:44,517 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:06:49,518 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:06:54,518 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:06:57,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:06:57,340 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:06:57,340 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:06:57,579 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:06:59,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:07:04,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:07:09,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:07:12,336 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:07:12,340 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:07:12,340 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:07:14,614 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:07:19,614 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:07:24,615 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:07:27,337 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:07:27,340 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:07:27,340 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:07:27,534 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:07:30,535 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:07:35,536 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:07:40,537 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:07:42,337 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:07:42,340 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:07:42,341 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:07:45,540 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:07:50,541 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:07:55,541 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:07:57,337 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:07:57,340 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:07:57,340 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:07:57,512 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:08:01,513 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:08:06,513 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:08:11,514 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:08:12,337 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:08:12,340 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:08:12,341 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:08:16,523 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:08:21,524 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:08:26,524 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:08:27,337 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:08:27,340 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:08:27,341 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:08:27,520 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:08:32,521 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:08:37,522 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:08:42,337 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:08:42,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:08:42,341 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:08:42,538 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:08:47,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:08:52,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:08:57,337 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:08:57,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:08:57,341 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:08:57,610 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:08:57,611 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:09:02,611 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:09:07,612 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:09:12,337 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:09:12,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:09:12,341 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:09:13,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:09:18,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:09:23,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:09:27,337 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:09:27,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:09:27,341 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:09:27,610 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:09:28,611 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:09:33,611 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:09:38,612 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:09:42,337 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:09:42,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:09:42,341 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:09:43,851 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:09:48,851 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:09:53,852 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:09:57,338 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:09:57,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:09:57,341 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:09:57,591 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:09:59,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:10:04,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:10:09,593 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:10:12,338 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:10:12,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:10:12,341 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:10:15,542 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:10:19,248 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 03:10:19,250 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 03:10:19,251 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 03:10:19,252 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 03:10:19,331 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 03:10:20,331 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 03:10:21,253 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:10:26,253 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:10:27,338 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:10:27,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:10:27,341 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:10:27,531 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:10:31,532 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:10:36,533 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:10:41,533 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:10:42,338 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:10:42,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:10:42,341 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:10:46,577 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:10:51,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:10:56,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:10:57,338 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:10:57,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:10:57,342 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:10:57,547 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:11:02,548 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:11:07,548 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:11:12,338 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:11:12,342 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:11:12,342 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:11:12,550 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:11:17,551 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:11:22,551 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:11:27,338 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:11:27,342 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:11:27,342 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:11:27,550 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:11:28,551 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:11:33,551 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:11:38,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:11:42,338 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:11:42,342 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:11:42,342 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:11:44,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:11:49,535 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:11:54,536 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:11:57,338 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:11:57,342 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:11:57,342 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:11:57,522 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:12:00,522 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:12:05,523 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:12:10,524 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:12:12,338 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:12:12,342 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:12:12,342 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:12:15,603 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:12:20,604 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:12:25,604 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:12:27,339 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:12:27,342 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:12:27,342 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:12:27,586 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:12:31,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:12:36,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:12:41,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:12:42,339 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:12:42,342 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:12:42,342 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:12:46,617 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:12:51,617 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:12:56,618 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:12:57,339 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:12:57,342 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:12:57,342 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:12:57,613 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:13:02,614 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:13:07,614 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:13:12,339 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:13:12,342 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:13:12,342 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:13:13,609 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:13:18,610 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:13:23,610 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:13:27,339 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:13:27,342 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:13:27,343 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:13:27,624 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:13:28,625 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:13:33,625 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:13:38,626 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:13:42,339 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:13:42,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:13:42,343 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:13:44,618 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:13:49,619 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:13:54,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:13:57,339 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:13:57,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:13:57,343 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:13:57,627 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:13:59,628 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:14:04,629 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:14:09,629 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:14:12,339 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:14:12,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:14:12,343 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:14:15,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:14:20,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:14:25,540 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:14:27,339 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:14:27,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:14:27,343 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:14:27,595 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:14:30,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:14:35,597 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:14:40,597 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:14:42,340 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:14:42,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:14:42,343 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:14:46,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:14:51,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:14:56,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:14:57,340 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:14:57,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:14:57,343 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:14:57,506 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:15:02,507 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:15:07,507 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:15:12,340 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:15:12,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:15:12,343 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:15:12,595 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:15:17,595 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:15:22,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:15:23,052 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 03:15:23,054 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 03:15:23,054 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 03:15:23,056 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 03:15:23,495 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 03:15:24,496 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 03:15:27,340 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:15:27,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:15:27,343 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:15:27,596 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:15:27,597 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:15:32,597 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:15:37,597 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:15:42,340 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:15:42,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:15:42,343 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:15:42,619 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:15:47,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:15:52,621 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:15:57,340 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:15:57,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:15:57,343 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:15:57,532 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:15:58,533 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:16:03,533 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:16:08,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:16:12,340 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:16:12,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:16:12,344 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:16:13,542 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:16:18,542 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:16:23,543 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:16:27,340 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:16:27,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:16:27,344 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:16:27,538 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:16:29,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:16:34,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:16:39,540 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:16:42,340 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:16:42,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:16:42,344 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:16:44,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:16:49,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:16:54,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:16:57,340 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:16:57,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:16:57,344 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:16:57,603 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:16:59,604 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:17:04,604 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:17:09,605 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:17:12,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:17:12,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:17:12,344 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:17:14,605 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:17:19,606 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:17:24,606 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:17:27,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:17:27,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:17:27,344 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:17:27,584 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:17:30,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:17:35,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:17:40,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:17:42,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:17:42,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:17:42,344 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:17:46,540 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:17:51,541 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:17:56,541 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:17:57,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:17:57,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:17:57,345 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:17:57,534 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:18:02,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:18:07,535 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:18:12,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:18:12,345 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:18:12,345 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:18:12,610 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:18:17,610 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:18:22,611 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:18:27,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:18:27,345 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:18:27,345 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:18:27,555 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:18:28,556 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:18:33,557 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:18:38,557 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:18:42,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:18:42,345 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:18:42,345 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:18:43,609 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:18:48,609 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:18:53,610 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:18:57,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:18:57,345 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:18:57,345 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:18:57,596 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:18:59,597 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:19:04,598 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:19:09,598 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:19:12,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:19:12,345 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:19:12,345 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:19:14,606 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:19:19,606 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:19:24,607 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:19:27,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:19:27,345 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:19:27,346 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:19:27,620 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:19:29,621 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:19:34,621 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:19:39,622 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:19:42,341 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:19:42,345 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:19:42,345 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:19:45,540 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:19:50,540 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:19:55,540 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:19:57,342 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:19:57,345 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:19:57,345 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:19:57,608 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:20:00,609 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:20:05,609 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:20:10,610 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:20:12,342 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:20:12,345 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:20:12,345 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:20:16,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:20:21,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:20:26,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:20:26,920 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 03:20:26,921 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 03:20:26,922 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 03:20:26,924 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 03:20:27,342 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:20:27,345 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:20:27,346 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:20:27,563 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:20:27,662 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 03:20:28,662 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 03:20:32,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:20:37,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:20:42,342 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:20:42,345 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:20:42,346 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:20:43,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:20:48,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:20:53,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:20:57,342 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:20:57,345 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:20:57,346 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:20:57,524 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:20:59,525 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:21:04,525 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:21:09,526 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:21:12,342 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:21:12,346 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:21:12,346 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:21:14,551 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:21:19,551 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:21:24,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:21:27,342 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:21:27,346 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:21:27,347 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:21:27,562 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:21:29,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:21:34,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:21:39,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:21:42,342 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:21:42,346 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:21:42,346 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:21:44,613 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:21:49,614 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:21:54,614 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:21:57,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:21:57,346 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:21:57,346 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:21:57,611 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:22:00,612 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:22:05,612 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:22:10,613 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:22:12,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:22:12,346 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:22:12,347 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:22:16,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:22:21,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:22:26,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:22:27,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:22:27,346 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:22:27,347 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:22:27,511 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:22:32,512 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:22:37,513 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:22:42,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:22:42,347 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:22:42,347 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:22:42,593 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:22:47,594 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:22:52,594 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:22:57,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:22:57,347 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:22:57,347 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:22:57,556 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:22:58,556 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:23:03,557 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:23:08,557 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:23:12,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:23:12,347 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:23:12,347 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:23:13,602 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:23:18,603 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:23:23,603 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:23:27,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:23:27,347 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:23:27,347 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:23:27,615 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:23:28,615 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:23:33,616 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:23:38,616 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:23:42,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:23:42,347 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:23:42,347 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:23:43,646 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:23:48,647 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:23:53,647 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:23:57,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:23:57,347 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:23:57,347 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:23:57,753 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:23:58,753 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:24:03,754 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:24:08,755 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:24:12,343 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:24:12,347 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:24:12,347 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:24:14,533 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:24:19,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:24:24,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:24:27,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:24:27,348 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:24:27,348 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:24:27,579 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:24:29,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:24:34,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:24:39,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:24:42,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:24:42,348 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:24:42,348 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:24:44,604 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:24:49,605 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:24:54,606 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:24:57,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:24:57,348 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:24:57,348 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:24:57,540 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:25:00,541 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:25:05,541 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:25:10,542 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:25:12,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:25:12,348 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:25:12,348 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:25:15,622 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:25:20,623 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:25:25,624 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:25:27,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:25:27,348 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:25:27,348 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:25:27,553 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:25:30,771 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 03:25:30,774 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 03:25:30,775 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 03:25:30,775 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:25:30,776 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 03:25:30,825 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 03:25:32,826 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 03:25:35,777 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:25:40,778 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:25:42,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:25:42,348 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:25:42,348 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:25:46,614 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:25:51,614 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:25:56,615 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:25:57,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:25:57,348 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:25:57,348 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:25:57,611 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:26:02,612 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:26:07,613 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:26:12,344 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:26:12,348 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:26:12,348 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:26:13,598 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:26:18,599 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:26:23,600 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:26:27,345 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:26:27,348 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:26:27,349 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:26:27,593 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:26:29,594 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:26:34,594 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:26:39,595 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:26:42,345 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:26:42,348 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:26:42,349 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:26:44,624 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:26:49,625 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:26:54,626 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:26:57,345 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:26:57,349 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:26:57,349 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:26:57,624 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:27:00,625 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:27:05,626 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:27:10,626 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:27:12,345 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:27:12,349 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:27:12,349 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:27:16,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:27:21,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:27:26,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:27:27,345 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:27:27,349 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:27:27,349 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:27:27,568 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:27:32,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:27:37,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:27:42,346 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:27:42,349 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:27:42,349 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:27:42,605 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:27:47,606 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:27:52,606 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:27:57,346 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:27:57,349 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:27:57,349 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:27:57,562 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:27:58,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:28:03,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:28:08,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:28:12,346 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:28:12,349 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:28:12,349 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:28:14,521 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:28:19,521 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:28:24,522 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:28:27,346 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:28:27,349 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:28:27,349 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:28:27,554 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:28:29,555 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:28:34,556 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:28:39,556 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:28:42,346 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:28:42,349 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:28:42,350 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:28:44,599 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:28:49,600 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:28:54,600 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:28:57,346 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:28:57,349 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:28:57,350 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:28:57,601 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:28:59,602 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:29:04,602 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:29:09,603 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:29:12,346 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:29:12,349 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:29:12,350 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:29:15,548 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:29:20,548 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:29:25,549 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:29:27,346 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:29:27,350 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:29:27,350 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:29:27,538 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:29:31,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:29:36,540 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:29:41,540 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:29:42,346 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:29:42,350 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:29:42,350 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:29:46,628 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:29:51,629 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:29:56,629 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:29:57,346 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:29:57,350 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:29:57,350 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:29:57,612 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:30:02,613 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:30:07,614 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:30:12,347 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:30:12,350 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:30:12,350 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:30:13,542 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:30:18,543 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:30:23,543 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:30:27,347 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:30:27,350 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:30:27,350 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:30:27,536 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:30:29,536 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:30:34,311 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 03:30:34,314 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 03:30:34,314 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 03:30:34,316 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 03:30:34,990 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 03:30:34,991 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 03:30:35,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:30:40,317 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:30:42,347 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:30:42,350 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:30:42,350 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:30:45,557 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:30:50,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:30:55,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:30:57,347 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:30:57,350 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:30:57,350 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:30:57,740 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:31:00,741 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:31:05,741 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:31:10,742 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:31:12,347 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:31:12,350 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:31:12,351 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:31:16,599 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:31:21,599 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:31:26,599 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:31:27,347 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:31:27,351 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:31:27,351 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:31:27,625 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:31:31,626 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:31:36,627 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:31:41,627 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:31:42,347 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:31:42,351 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:31:42,351 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:31:47,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:31:52,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:31:57,347 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:31:57,351 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:31:57,351 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:31:57,622 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:31:57,623 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:32:02,623 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:32:07,624 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:32:12,347 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:32:12,351 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:32:12,351 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:32:13,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:32:18,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:32:23,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:32:27,347 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:32:27,351 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:32:27,351 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:32:27,546 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:32:29,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:32:34,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:32:39,548 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:32:42,347 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:32:42,351 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:32:42,351 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:32:44,621 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:32:49,622 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:32:54,622 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:32:57,347 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:32:57,351 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:32:57,351 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:32:57,535 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:33:00,536 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:33:05,537 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:33:10,537 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:33:12,348 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:33:12,351 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:33:12,352 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:33:15,637 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:33:20,638 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:33:25,639 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:33:27,348 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:33:27,351 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:33:27,352 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:33:27,531 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:33:31,532 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:33:36,532 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:33:41,533 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:33:42,348 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:33:42,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:33:42,352 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:33:46,555 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:33:51,555 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:33:56,556 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:33:57,348 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:33:57,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:33:57,352 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:33:57,614 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:34:01,616 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:34:06,617 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:34:11,617 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:34:12,348 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:34:12,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:34:12,352 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:34:17,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:34:22,548 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:34:27,351 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:34:27,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:34:27,352 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:34:27,636 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:34:27,636 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:34:32,637 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:34:37,638 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:34:42,351 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:34:42,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:34:42,352 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:34:43,631 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:34:48,632 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:34:53,632 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:34:57,351 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:34:57,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:34:57,352 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:35:03,353 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 03:35:03,701 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:35:03,701 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:35:08,702 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:35:12,351 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:35:12,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:35:12,352 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:35:14,535 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:35:19,535 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:35:24,536 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:35:27,351 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:35:27,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:35:27,352 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:35:27,590 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:35:29,590 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:35:34,591 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:35:38,356 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 03:35:38,358 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 03:35:38,358 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 03:35:38,360 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 03:35:39,156 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 03:35:40,157 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 03:35:40,361 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:35:42,351 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:35:42,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:35:42,352 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:35:45,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:35:50,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:35:55,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:35:57,351 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:35:57,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:35:57,353 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:35:57,552 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:36:01,554 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:36:06,554 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:36:11,555 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:36:12,351 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:36:12,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:36:12,353 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:36:16,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:36:21,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:36:26,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:36:27,351 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:36:27,353 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:36:27,353 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:36:27,564 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:36:31,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:36:36,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:36:41,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:36:42,351 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:36:42,353 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:36:42,353 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:36:46,629 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:36:51,630 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:36:56,631 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:36:57,351 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:36:57,353 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:36:57,353 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:36:57,607 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:37:02,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:37:07,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:37:12,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:37:12,353 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:37:12,353 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:37:13,551 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:37:18,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:37:23,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:37:27,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:37:27,353 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:37:27,353 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:37:27,586 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:37:28,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:37:33,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:37:38,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:37:42,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:37:42,353 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:37:42,353 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:37:44,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:37:49,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:37:54,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:37:57,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:37:57,353 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:37:57,353 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:37:57,566 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:38:00,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:38:05,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:38:10,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:38:12,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:38:12,353 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:38:12,353 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:38:16,554 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:38:21,554 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:38:26,555 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:38:27,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:38:27,353 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:38:27,354 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:38:27,643 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:38:31,644 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:38:36,644 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:38:41,645 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:38:42,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:38:42,354 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:38:42,354 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:38:47,629 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:38:52,630 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:38:57,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:38:57,354 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:38:57,354 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:38:57,529 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:38:58,529 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:39:03,530 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:39:08,530 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:39:12,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:39:12,354 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:39:12,354 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:39:13,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:39:18,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:39:23,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:39:27,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:39:27,354 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:39:27,354 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:39:27,549 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:39:29,550 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:39:34,551 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:39:39,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:39:42,352 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:39:42,354 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:39:42,354 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:39:45,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:39:50,548 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:39:55,548 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:39:57,353 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:39:57,354 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:39:57,354 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:39:57,578 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:40:00,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:40:05,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:40:10,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:40:12,353 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:40:12,354 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:40:12,354 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:40:16,532 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:40:21,533 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:40:26,533 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:40:27,353 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:40:27,354 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:40:27,355 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:40:27,563 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:40:31,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:40:36,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:40:41,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:40:42,299 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 03:40:42,302 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 03:40:42,302 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 03:40:42,304 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 03:40:42,325 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 03:40:42,353 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:40:42,354 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:40:42,355 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:40:44,326 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 03:40:46,623 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:40:51,624 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:40:56,625 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:40:57,353 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:40:57,355 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:40:57,355 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:40:57,536 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:41:02,537 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:41:07,538 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:41:12,353 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:41:12,355 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:41:12,355 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:41:12,637 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:41:17,638 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:41:22,639 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:41:27,353 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:41:27,355 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:41:27,355 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:41:27,583 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:41:28,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:41:33,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:41:38,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:41:42,353 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:41:42,355 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:41:42,355 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:41:43,633 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:41:48,634 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:41:53,634 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:41:57,353 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:41:57,355 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:41:57,355 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:41:57,537 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:41:59,543 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:42:04,543 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:42:09,544 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:42:12,353 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:42:12,355 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:42:12,355 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:42:15,536 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:42:20,537 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:42:25,537 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:42:27,354 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:42:27,355 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:42:27,356 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:42:27,544 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:42:30,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:42:35,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:42:40,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:42:42,354 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:42:42,355 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:42:42,356 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:42:45,626 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:42:50,627 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:42:55,628 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:42:57,354 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:42:57,355 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:42:57,356 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:42:57,638 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:43:00,639 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:43:05,640 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:43:10,640 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:43:12,354 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:43:12,355 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:43:12,356 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:43:15,836 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:43:20,837 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:43:25,837 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:43:27,354 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:43:27,355 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:43:27,356 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:43:27,551 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:43:31,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:43:36,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:43:41,553 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:43:42,354 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:43:42,356 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:43:42,356 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:43:46,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:43:51,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:43:56,577 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:43:57,354 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:43:57,356 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:43:57,356 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:43:57,606 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:44:01,606 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:44:06,607 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:44:11,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:44:12,354 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:44:12,356 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:44:12,356 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:44:16,635 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:44:21,636 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:44:26,637 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:44:27,354 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:44:27,356 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:44:27,356 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:44:27,632 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:44:32,633 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:44:37,634 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:44:42,354 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:44:42,356 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:44:42,356 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:44:43,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:44:48,609 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:44:53,609 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:44:57,354 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:44:57,356 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:44:57,356 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:44:57,542 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:44:59,542 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:45:04,543 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:45:09,544 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:45:12,355 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:45:12,356 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:45:12,356 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:45:14,600 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:45:19,601 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:45:24,601 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:45:27,355 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:45:27,356 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:45:27,357 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:45:27,595 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:45:30,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:45:35,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:45:40,597 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:45:42,355 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:45:42,356 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:45:42,357 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:45:46,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:45:46,671 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 03:45:46,673 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 03:45:46,674 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 03:45:46,675 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 03:45:47,492 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 03:45:48,493 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 03:45:51,676 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:45:56,677 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:45:57,355 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:45:57,356 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:45:57,357 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:45:57,639 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:46:02,640 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:46:07,640 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:46:12,355 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:46:12,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:46:12,357 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:46:13,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:46:18,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:46:23,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:46:27,355 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:46:27,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:46:27,357 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:46:27,569 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:46:28,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:46:33,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:46:38,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:46:42,355 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:46:42,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:46:42,357 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:46:43,618 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:46:48,619 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:46:53,619 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:46:57,355 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:46:57,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:46:57,357 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:46:57,787 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:46:58,788 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:47:03,788 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:47:08,789 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:47:12,356 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:47:12,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:47:12,357 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:47:14,619 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:47:19,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:47:24,621 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:47:27,356 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:47:27,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:47:27,358 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:47:27,538 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:47:30,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:47:35,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:47:40,540 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:47:42,356 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:47:42,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:47:42,358 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:47:45,638 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:47:50,639 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:47:55,640 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:47:57,356 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:47:57,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:47:57,358 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:47:57,564 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:48:01,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:48:06,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:48:11,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:48:12,356 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:48:12,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:48:12,358 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:48:16,595 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:48:21,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:48:26,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:48:27,356 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:48:27,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:48:27,358 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:48:27,597 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:48:31,598 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:48:36,599 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:48:41,600 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:48:42,356 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:48:42,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:48:42,359 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:48:47,533 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:48:52,533 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:48:57,356 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:48:57,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:48:57,358 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:48:57,604 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:48:57,604 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:49:02,605 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:49:07,606 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:49:12,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:49:12,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:49:12,358 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:49:13,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:49:18,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:49:23,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:49:27,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:49:27,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:49:27,358 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:49:27,564 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:49:29,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:49:34,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:49:39,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:49:42,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:49:42,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:49:42,358 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:49:44,637 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:49:49,637 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:49:54,638 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:49:57,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:49:57,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:49:57,358 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:49:57,603 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:50:00,604 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:50:05,605 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:50:10,605 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:50:12,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:50:12,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:50:12,358 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:50:16,548 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:50:21,549 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:50:26,550 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:50:27,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:50:27,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:50:27,358 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:50:27,592 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:50:31,593 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:50:36,593 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:50:41,594 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:50:42,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:50:42,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:50:42,359 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:50:46,624 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:50:51,192 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 03:50:51,194 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 03:50:51,195 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 03:50:51,196 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 03:50:51,655 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 03:50:52,196 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:50:52,656 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 03:50:57,197 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:50:57,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:50:57,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:50:57,359 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:50:57,627 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:51:02,628 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:51:07,629 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:51:12,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:51:12,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:51:12,359 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:51:13,591 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:51:18,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:51:23,593 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:51:27,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:51:27,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:51:27,359 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:51:27,554 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:51:29,555 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:51:34,556 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:51:39,556 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:51:42,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:51:42,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:51:42,359 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:51:45,548 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:51:50,549 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:51:55,549 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:51:57,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:51:57,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:51:57,359 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:51:57,580 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:52:00,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:52:05,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:52:10,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:52:12,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:52:12,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:52:12,359 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:52:16,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:52:21,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:52:26,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:52:27,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:52:27,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:52:27,359 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:52:27,556 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:52:32,557 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:52:37,557 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:52:42,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:52:42,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:52:42,359 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:52:42,624 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:52:47,625 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:52:52,625 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:52:57,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:52:57,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:52:57,359 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:52:57,584 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:52:58,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:53:03,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:53:08,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:53:12,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:53:12,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:53:12,360 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:53:13,622 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:53:18,623 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:53:23,623 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:53:27,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:53:27,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:53:27,360 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:53:27,618 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:53:29,619 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:53:34,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:53:39,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:53:42,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:53:42,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:53:42,360 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:53:44,646 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:53:49,647 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:53:54,648 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:53:57,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:53:57,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:53:57,360 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:53:57,640 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:54:00,641 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:54:05,641 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:54:10,642 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:54:12,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:54:12,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:54:12,360 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:54:16,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:54:21,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:54:26,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:54:27,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:54:27,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:54:27,360 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:54:27,629 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:54:31,630 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:54:36,631 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:54:41,631 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:54:42,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:54:42,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:54:42,360 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:54:47,556 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:54:52,557 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:54:57,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:54:57,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:54:57,360 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:54:57,539 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:54:58,540 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:55:03,540 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:55:08,541 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:55:12,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:55:12,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:55:12,360 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:55:13,626 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:55:18,627 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:55:23,628 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:55:27,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:55:27,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:55:27,360 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:55:27,571 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:55:29,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:55:34,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:55:39,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:55:42,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:55:42,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:55:42,360 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:55:44,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:55:49,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:55:54,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:55:54,979 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 03:55:54,981 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 03:55:54,982 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 03:55:54,983 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 03:55:55,821 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 03:55:56,821 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 03:55:57,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:55:57,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:55:57,360 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:55:57,614 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:55:59,615 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:56:04,616 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:56:09,616 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:56:12,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:56:12,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:56:12,360 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:56:15,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:56:20,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:56:25,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:56:27,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:56:27,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:56:27,361 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:56:27,588 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:56:30,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:56:35,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:56:40,590 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:56:42,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:56:42,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:56:42,361 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:56:45,709 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:56:50,710 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:56:55,710 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:56:57,359 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:56:57,361 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:56:57,361 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:56:57,571 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:57:01,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:57:06,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:57:11,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:57:12,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:57:12,361 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:57:12,361 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:57:16,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:57:21,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:57:26,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:57:27,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:57:27,361 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:57:27,361 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:57:27,534 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:57:32,535 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:57:37,536 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:57:42,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:57:42,361 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:57:42,361 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:57:42,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:57:47,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:57:52,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:57:57,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:57:57,361 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:57:57,361 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:57:57,629 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:57:57,630 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:58:02,630 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:58:07,631 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:58:12,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:58:12,361 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:58:12,362 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:58:13,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:58:18,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:58:23,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:58:27,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:58:27,361 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:58:27,362 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:58:27,540 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:58:29,541 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:58:34,541 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:58:39,542 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:58:42,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:58:42,361 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:58:42,362 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:58:44,619 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:58:49,619 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:58:54,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:58:57,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:58:57,361 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:58:57,362 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:58:57,608 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:59:00,610 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:59:05,610 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:59:10,612 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:59:12,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:59:12,361 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:59:12,362 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:59:15,647 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:59:20,647 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:59:25,648 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:59:27,361 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:59:27,362 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:59:27,362 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:59:27,637 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:59:31,638 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:59:36,639 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:59:41,639 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:59:42,360 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:59:42,362 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:59:42,362 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:59:47,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:59:52,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 03:59:57,361 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 03:59:57,362 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 03:59:57,362 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 03:59:57,566 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 03:59:58,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:00:03,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:00:08,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:00:12,361 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:00:12,362 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:00:12,362 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:00:14,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:00:19,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:00:24,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:00:27,361 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:00:27,362 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:00:27,362 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:00:27,619 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:00:29,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:00:34,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:00:39,621 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:00:42,361 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:00:42,362 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:00:42,362 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:00:45,613 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:00:50,614 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:00:55,614 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:00:57,361 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:00:57,362 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:00:57,363 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:00:57,651 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:00:59,040 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 04:00:59,041 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 04:00:59,043 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 04:00:59,044 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 04:00:59,989 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 04:01:00,990 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 04:01:01,045 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:01:06,045 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:01:11,046 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:01:12,361 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:01:12,362 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:01:12,363 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:01:16,621 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:01:21,622 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:01:26,622 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:01:27,361 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:01:27,362 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:01:27,363 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:01:27,596 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:01:32,597 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:01:37,597 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:01:42,361 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:01:42,363 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:01:42,363 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:01:43,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:01:48,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:01:53,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:01:57,361 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:01:57,363 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:01:57,363 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:01:57,634 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:01:58,635 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:02:03,636 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:02:08,636 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:02:12,362 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:02:12,363 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:02:12,363 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:02:14,616 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:02:19,616 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:02:24,617 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:02:27,362 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:02:27,363 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:02:27,363 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:02:27,565 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:02:30,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:02:35,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:02:40,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:02:42,362 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:02:42,363 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:02:42,363 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:02:45,729 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:02:50,730 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:02:55,730 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:02:57,362 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:02:57,363 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:02:57,363 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:02:57,634 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:03:01,635 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:03:06,635 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:03:11,636 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:03:12,362 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:03:12,363 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:03:12,363 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:03:17,544 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:03:22,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:03:27,362 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:03:27,363 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:03:27,363 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:03:27,535 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:03:28,536 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:03:33,537 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:03:38,537 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:03:42,362 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:03:42,363 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:03:42,364 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:03:43,635 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:03:48,635 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:03:53,636 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:03:57,362 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:03:57,363 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:03:57,364 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:03:57,552 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:03:59,553 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:04:04,554 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:04:09,554 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:04:12,363 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:04:12,364 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:04:12,364 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:04:15,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:04:20,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:04:25,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:04:27,363 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:04:27,364 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:04:27,364 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:04:27,614 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:04:30,615 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:04:35,615 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:04:40,616 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:04:42,363 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:04:42,364 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:04:42,364 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:04:46,556 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:04:51,556 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:04:56,557 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:04:57,363 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:04:57,364 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:04:57,364 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:04:57,564 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:05:01,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:05:06,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:05:11,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:05:12,363 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:05:12,364 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:05:12,364 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:05:16,641 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:05:21,642 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:05:26,642 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:05:27,363 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:05:27,364 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:05:27,364 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:05:27,593 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:05:32,594 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:05:37,594 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:05:42,363 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:05:42,364 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:05:42,365 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:05:42,632 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:05:47,632 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:05:52,633 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:05:57,363 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:05:57,364 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:05:57,365 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:05:57,565 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:05:58,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:06:03,408 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 04:06:03,411 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 04:06:03,411 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 04:06:03,413 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 04:06:04,156 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 04:06:04,413 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:06:05,157 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 04:06:09,414 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:06:12,363 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:06:12,364 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:06:12,365 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:06:14,625 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:06:19,625 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:06:24,626 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:06:27,363 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:06:27,364 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:06:27,365 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:06:27,567 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:06:30,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:06:35,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:06:40,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:06:42,363 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:06:42,365 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:06:42,365 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:06:45,599 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:06:50,600 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:06:55,600 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:06:57,364 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:06:57,365 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:06:57,365 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:06:57,626 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:07:00,627 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:07:05,628 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:07:10,628 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:07:12,364 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:07:12,365 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:07:12,365 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:07:16,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:07:21,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:07:26,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:07:27,364 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:07:27,365 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:07:27,365 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:07:27,583 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:07:31,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:07:36,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:07:41,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:07:42,364 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:07:42,365 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:07:42,365 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:07:47,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:07:52,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:07:57,364 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:07:57,365 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:07:57,365 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:07:57,575 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:07:58,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:08:03,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:08:08,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:08:12,364 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:08:12,365 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:08:12,365 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:08:13,632 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:08:18,632 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:08:23,633 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:08:27,364 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:08:27,365 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:08:27,366 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:08:27,616 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:08:29,617 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:08:34,618 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:08:39,618 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:08:42,364 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:08:42,365 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:08:42,366 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:08:45,599 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:08:50,599 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:08:55,600 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:08:57,364 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:08:57,365 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:08:57,366 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:08:57,561 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:09:01,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:09:06,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:09:11,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:09:12,365 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:09:12,365 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:09:12,366 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:09:16,637 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:09:21,638 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:09:26,638 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:09:27,365 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:09:27,366 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:09:27,366 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:09:27,612 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:09:32,613 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:09:37,614 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:09:42,365 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:09:42,366 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:09:42,366 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:09:43,604 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:09:48,605 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:09:53,605 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:09:57,365 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:09:57,366 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:09:57,366 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:09:57,552 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:09:59,553 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:10:04,553 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:10:09,554 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:10:12,365 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:10:12,366 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:10:12,366 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:10:14,668 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:10:19,669 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:10:24,669 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:10:27,365 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:10:27,366 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:10:27,366 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:10:27,629 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:10:30,630 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:10:35,630 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:10:40,631 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:10:42,365 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:10:42,366 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:10:42,367 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:10:46,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:10:51,559 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:10:56,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:10:57,365 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:10:57,366 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:10:57,367 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:10:57,647 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:11:01,648 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:11:06,649 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:11:07,747 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 04:11:07,749 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 04:11:07,749 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 04:11:07,751 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 04:11:08,336 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 04:11:10,337 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 04:11:11,752 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:11:12,365 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:11:12,366 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:11:12,367 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:11:17,634 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:11:22,635 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:11:27,366 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:11:27,367 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:11:27,367 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:11:27,633 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:11:28,634 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:11:33,634 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:11:38,635 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:11:42,366 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:11:42,367 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:11:42,367 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:11:44,617 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:11:49,618 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:11:54,618 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:11:57,366 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:11:57,367 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:11:57,367 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:11:57,601 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:12:00,601 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:12:05,602 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:12:10,603 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:12:12,366 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:12:12,367 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:12:12,367 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:12:16,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:12:21,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:12:26,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:12:27,366 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:12:27,367 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:12:27,367 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:12:27,578 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:12:31,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:12:36,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:12:41,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:12:42,366 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:12:42,367 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:12:42,367 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:12:46,634 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:12:51,634 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:12:56,635 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:12:57,366 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:12:57,367 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:12:57,367 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:12:57,579 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:13:02,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:13:07,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:13:12,366 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:13:12,367 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:13:12,368 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:13:12,637 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:13:17,638 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:13:22,639 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:13:27,366 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:13:27,368 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:13:27,368 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:13:27,546 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:13:28,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:13:33,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:13:38,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:13:42,366 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:13:42,368 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:13:42,368 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:13:43,559 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:13:48,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:13:53,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:13:57,367 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:13:57,368 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:13:57,368 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:13:57,633 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:13:58,634 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:14:03,635 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:14:08,635 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:14:12,367 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:14:12,368 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:14:12,368 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:14:14,559 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:14:19,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:14:24,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:14:27,367 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:14:27,368 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:14:27,368 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:14:27,609 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:14:29,610 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:14:34,611 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:14:39,611 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:14:42,367 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:14:42,368 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:14:42,368 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:14:44,626 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:14:49,627 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:14:54,627 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:14:57,367 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:14:57,368 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:14:57,368 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:14:57,617 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:15:00,618 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:15:05,618 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:15:10,619 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:15:12,367 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:15:12,368 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:15:12,368 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:15:16,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:15:21,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:15:26,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:15:27,367 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:15:27,368 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:15:27,368 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:15:27,587 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:15:31,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:15:36,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:15:41,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:15:42,367 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:15:42,368 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:15:42,369 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:15:47,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:15:52,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:15:57,367 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:15:57,368 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:15:57,369 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:15:57,566 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:15:58,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:16:03,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:16:08,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:16:11,628 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 04:16:11,630 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 04:16:11,631 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 04:16:11,632 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 04:16:12,367 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:16:12,369 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:16:12,369 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:16:12,501 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 04:16:13,607 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:16:14,502 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 04:16:18,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:16:23,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:16:27,367 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:16:27,369 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:16:27,369 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:16:27,616 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:16:28,617 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:16:33,618 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:16:38,618 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:16:42,368 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:16:42,369 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:16:42,369 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:16:43,635 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:16:48,635 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:16:53,636 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:16:57,368 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:16:57,369 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:16:57,369 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:16:57,641 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:16:58,642 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:17:03,643 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:17:08,643 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:17:12,368 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:17:12,369 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:17:12,369 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:17:14,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:17:19,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:17:24,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:17:27,368 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:17:27,369 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:17:27,369 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:17:27,647 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:17:29,648 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:17:34,649 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:17:39,649 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:17:42,368 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:17:42,369 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:17:42,369 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:17:45,634 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:17:50,634 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:17:55,635 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:17:57,368 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:17:57,369 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:17:57,369 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:17:57,613 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:18:01,613 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:18:06,614 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:18:11,615 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:18:12,368 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:18:12,369 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:18:12,369 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:18:17,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:18:22,590 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:18:27,368 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:18:27,370 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:18:27,370 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:18:27,568 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:18:28,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:18:33,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:18:38,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:18:42,368 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:18:42,369 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:18:42,370 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:18:43,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:18:48,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:18:53,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:18:57,368 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:18:57,370 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:18:57,370 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:18:57,647 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:18:58,647 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:19:03,648 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:19:08,648 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:19:12,369 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:19:12,370 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:19:12,370 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:19:14,615 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:19:19,615 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:19:24,616 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:19:27,369 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:19:27,370 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:19:27,370 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:19:27,581 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:19:30,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:19:35,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:19:40,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:19:42,369 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:19:42,370 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:19:42,370 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:19:46,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:19:51,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:19:56,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:19:57,369 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:19:57,370 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:19:57,370 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:19:57,603 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:20:01,604 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:20:06,604 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:20:11,605 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:20:12,369 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:20:12,370 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:20:12,370 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:20:17,595 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:20:22,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:20:27,369 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:20:27,370 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:20:27,370 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:20:27,626 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:20:27,627 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:20:32,627 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:20:37,628 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:20:42,369 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:20:42,370 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:20:42,371 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:20:43,617 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:20:48,618 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:20:53,619 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:20:57,369 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:20:57,371 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:20:57,371 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:20:57,623 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:20:58,624 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:21:03,624 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:21:08,625 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:21:12,369 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:21:12,370 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:21:12,371 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:21:13,654 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:21:16,151 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 04:21:16,153 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 04:21:16,154 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 04:21:16,156 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 04:21:16,669 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 04:21:18,670 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 04:21:19,156 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:21:24,157 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:21:27,369 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:21:27,370 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:21:27,371 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:21:27,553 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:21:29,554 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:21:34,554 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:21:39,555 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:21:42,369 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:21:42,371 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:21:42,371 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:21:44,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:21:49,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:21:54,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:21:57,370 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:21:57,371 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:21:57,371 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:21:57,614 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:21:59,614 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:22:04,615 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:22:09,615 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:22:12,370 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:22:12,371 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:22:12,371 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:22:14,653 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:22:19,654 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:22:24,655 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:22:27,370 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:22:27,371 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:22:27,371 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:22:27,646 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:22:30,647 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:22:35,648 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:22:40,648 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:22:42,370 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:22:42,371 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:22:42,371 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:22:46,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:22:51,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:22:56,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:22:57,370 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:22:57,371 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:22:57,371 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:22:57,637 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:23:01,638 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:23:06,639 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:23:11,639 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:23:12,370 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:23:12,371 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:23:12,371 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:23:17,637 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:23:22,638 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:23:27,370 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:23:27,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:23:27,372 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:23:27,627 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:23:28,628 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:23:33,629 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:23:38,629 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:23:42,370 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:23:42,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:23:42,372 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:23:44,628 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:23:49,628 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:23:54,629 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:23:57,370 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:23:57,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:23:57,372 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:23:57,727 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:23:59,728 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:24:04,728 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:24:09,729 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:24:12,370 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:24:12,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:24:12,372 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:24:15,647 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:24:20,648 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:24:25,648 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:24:27,371 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:24:27,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:24:27,372 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:24:27,624 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:24:31,626 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:24:36,626 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:24:41,627 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:24:42,371 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:24:42,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:24:42,372 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:24:46,639 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:24:51,640 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:24:56,640 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:24:57,371 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:24:57,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:24:57,372 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:24:57,553 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:25:02,554 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:25:07,555 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:25:12,371 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:25:12,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:25:12,372 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:25:12,642 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:25:17,643 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:25:22,643 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:25:27,371 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:25:27,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:25:27,373 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:25:27,672 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:25:27,672 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:25:32,673 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:25:37,673 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:25:42,371 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:25:42,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:25:42,373 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:25:43,621 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:25:48,621 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:25:53,622 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:25:57,371 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:25:57,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:25:57,373 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:25:57,591 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:25:59,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:26:04,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:26:09,593 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:26:12,371 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:26:12,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:26:12,373 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:26:14,628 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:26:19,628 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:26:19,651 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 04:26:19,653 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 04:26:19,654 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 04:26:19,655 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 04:26:19,829 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 04:26:20,829 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 04:26:24,657 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:26:27,371 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:26:27,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:26:27,373 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:26:27,564 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:26:30,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:26:35,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:26:40,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:26:42,371 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:26:42,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:26:42,373 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:26:45,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:26:50,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:26:55,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:26:57,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:26:57,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:26:57,373 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:26:57,598 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:27:00,598 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:27:05,599 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:27:10,599 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:27:12,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:27:12,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:27:12,373 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:27:16,591 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:27:21,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:27:26,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:27:27,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:27:27,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:27:27,373 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:27:27,638 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:27:31,639 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:27:36,639 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:27:41,640 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:27:42,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:27:42,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:27:42,373 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:27:47,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:27:52,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:27:57,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:27:57,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:27:57,373 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:27:57,613 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:27:57,613 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:28:02,614 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:28:07,615 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:28:12,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:28:12,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:28:12,373 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:28:12,631 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:28:17,631 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:28:22,632 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:28:27,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:28:27,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:28:27,373 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:28:27,631 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:28:28,632 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:28:33,632 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:28:38,633 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:28:42,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:28:42,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:28:42,374 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:28:43,650 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:28:48,651 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:28:53,652 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:28:57,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:28:57,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:28:57,374 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:28:57,577 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:28:59,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:29:04,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:29:09,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:29:12,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:29:12,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:29:12,374 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:29:14,651 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:29:19,652 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:29:24,652 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:29:27,372 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:29:27,374 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:29:27,374 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:29:27,685 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:29:29,686 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:29:34,686 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:29:39,687 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:29:42,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:29:42,374 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:29:42,374 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:29:45,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:29:50,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:29:55,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:29:57,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:29:57,374 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:29:57,374 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:29:58,451 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:30:01,452 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:30:06,453 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:30:11,454 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:30:12,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:30:12,374 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:30:12,374 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:30:16,550 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:30:21,550 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:30:26,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:30:27,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:30:27,374 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:30:27,374 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:30:27,605 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:30:31,606 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:30:36,607 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:30:41,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:30:42,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:30:42,374 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:30:42,374 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:30:46,636 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:30:51,637 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:30:56,638 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:30:57,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:30:57,374 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:30:57,374 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:30:57,574 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:31:02,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:31:07,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:31:12,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:31:12,374 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:31:12,374 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:31:13,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:31:18,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:31:23,548 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:31:23,775 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 04:31:23,777 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 04:31:23,777 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 04:31:23,779 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 04:31:23,997 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 04:31:24,998 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 04:31:27,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:31:27,374 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:31:27,374 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:31:27,570 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:31:28,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:31:33,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:31:38,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:31:42,373 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:31:42,374 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:31:42,375 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:31:43,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:31:48,621 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:31:53,621 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:31:57,374 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:31:57,374 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:31:57,375 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:31:57,644 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:31:58,645 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:32:03,645 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:32:08,646 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:32:12,374 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:32:12,375 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:32:12,375 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:32:14,643 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:32:19,643 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:32:24,644 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:32:27,374 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:32:27,375 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:32:27,375 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:32:27,605 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:32:30,606 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:32:35,606 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:32:40,607 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:32:42,374 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:32:42,375 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:32:42,375 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:32:46,027 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:32:51,027 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:32:56,028 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:32:57,374 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:32:57,375 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:32:57,376 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:32:57,621 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:33:01,622 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:33:06,623 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:33:11,623 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:33:12,374 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:33:12,375 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:33:12,376 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:33:17,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:33:22,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:33:27,374 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:33:27,375 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:33:27,376 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:33:27,645 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:33:27,645 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:33:32,646 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:33:37,647 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:33:42,374 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:33:42,375 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:33:42,376 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:33:43,543 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:33:48,543 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:33:53,544 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:33:57,374 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:33:57,376 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:33:57,376 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:33:57,577 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:33:58,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:34:03,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:34:08,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:34:12,374 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:34:12,376 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:34:12,376 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:34:13,675 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:34:18,676 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:34:23,676 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:34:27,375 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:34:27,376 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:34:27,376 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:34:27,593 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:34:29,593 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:34:34,594 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:34:39,594 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:34:42,375 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:34:42,376 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:34:42,376 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:34:44,614 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:34:49,614 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:34:54,615 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:34:57,375 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:34:57,376 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:34:57,376 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:34:57,619 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:34:59,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:35:04,621 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:35:09,622 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:35:12,375 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:35:12,376 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:35:12,376 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:35:15,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:35:20,553 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:35:25,554 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:35:27,375 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:35:27,376 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:35:27,376 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:35:27,572 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:35:30,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:35:35,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:35:40,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:35:42,375 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:35:42,376 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:35:42,376 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:35:45,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:35:50,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:35:55,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:35:57,375 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:35:57,376 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:35:57,376 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:35:57,585 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:36:00,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:36:05,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:36:10,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:36:12,375 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:36:12,376 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:36:12,376 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:36:15,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:36:20,593 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:36:25,593 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:36:27,375 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:36:27,376 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:36:27,377 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:36:27,648 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:36:27,967 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 04:36:27,969 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 04:36:27,970 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 04:36:27,971 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 04:36:28,157 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 04:36:29,158 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 04:36:30,972 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:36:35,972 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:36:40,973 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:36:42,375 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:36:42,376 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:36:42,377 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:36:46,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:36:51,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:36:56,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:36:57,375 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:36:57,376 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:36:57,377 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:36:57,574 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:37:01,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:37:06,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:37:11,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:37:12,376 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:37:12,377 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:37:12,377 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:37:16,640 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:37:21,641 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:37:26,641 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:37:27,376 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:37:27,377 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:37:27,377 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:37:27,589 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:37:32,590 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:37:37,591 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:37:42,376 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:37:42,377 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:37:42,377 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:37:42,623 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:37:47,624 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:37:52,624 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:37:57,376 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:37:57,377 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:37:57,377 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:37:57,649 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:37:57,649 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:38:02,651 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:38:07,651 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:38:12,376 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:38:12,377 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:38:12,377 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:38:13,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:38:18,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:38:23,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:38:27,376 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:38:27,377 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:38:27,377 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:38:27,640 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:38:28,641 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:38:33,642 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:38:38,643 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:38:42,376 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:38:42,377 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:38:42,378 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:38:44,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:38:49,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:38:54,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:38:57,376 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:38:57,377 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:38:57,378 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:38:57,601 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:38:59,602 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:39:04,603 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:39:09,604 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:39:12,376 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:39:12,378 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:39:12,378 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:39:15,557 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:39:20,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:39:25,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:39:27,377 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:39:27,378 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:39:27,378 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:39:27,902 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:39:30,903 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:39:35,903 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:39:40,904 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:39:42,377 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:39:42,378 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:39:42,378 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:39:46,549 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:39:51,549 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:39:56,550 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:39:57,377 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:39:57,378 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:39:57,378 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:39:57,638 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:40:01,640 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:40:06,640 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:40:11,641 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:40:12,378 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:40:12,378 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:40:12,378 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:40:17,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:40:22,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:40:27,378 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:40:27,378 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:40:27,379 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:40:27,607 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:40:27,607 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:40:32,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:40:37,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:40:42,378 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:40:42,378 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:40:42,379 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:40:43,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:40:48,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:40:53,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:40:57,378 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:40:57,378 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:40:57,379 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:40:57,581 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:40:59,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:41:04,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:41:09,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:41:12,378 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:41:12,379 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:41:12,379 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:41:14,594 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:41:19,595 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:41:24,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:41:27,378 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:41:27,379 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:41:27,379 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:41:27,622 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:41:29,623 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:41:32,072 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 04:41:32,075 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 04:41:32,075 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 04:41:32,076 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 04:41:32,316 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 04:41:34,318 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 04:41:35,077 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:41:40,078 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:41:42,378 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:41:42,379 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:41:42,379 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:41:45,619 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:41:50,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:41:55,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:41:57,378 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:41:57,379 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:41:57,379 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:41:57,583 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:42:01,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:42:06,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:42:11,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:42:12,378 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:42:12,379 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:42:12,379 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:42:16,628 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:42:21,629 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:42:26,630 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:42:27,378 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:42:27,379 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:42:27,379 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:42:27,584 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:42:32,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:42:37,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:42:42,379 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:42:42,379 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:42:42,379 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:42:42,647 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:42:47,648 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:42:52,648 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:42:57,379 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:42:57,379 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:42:57,379 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:42:57,610 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:42:58,611 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:43:03,611 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:43:08,612 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:43:12,379 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:43:12,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:43:12,380 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:43:14,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:43:19,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:43:24,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:43:27,379 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:43:27,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:43:27,380 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:43:27,585 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:43:30,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:43:35,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:43:40,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:43:42,379 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:43:42,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:43:42,380 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:43:46,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:43:51,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:43:56,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:43:57,379 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:43:57,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:43:57,380 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:43:57,587 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:44:01,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:44:06,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:44:11,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:44:12,379 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:44:12,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:44:12,380 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:44:17,559 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:44:22,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:44:27,379 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:44:27,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:44:27,381 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:44:27,624 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:44:27,624 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:44:32,625 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:44:37,626 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:44:42,379 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:44:42,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:44:42,381 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:44:43,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:44:48,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:44:53,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:44:57,379 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:44:57,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:44:57,380 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:44:57,623 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:44:58,624 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:45:03,624 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:45:08,625 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:45:12,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:45:12,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:45:12,380 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:45:14,557 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:45:19,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:45:24,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:45:27,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:45:27,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:45:27,381 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:45:27,639 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:45:29,640 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:45:34,640 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:45:39,641 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:45:42,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:45:42,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:45:42,381 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:45:45,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:45:50,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:45:55,590 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:45:57,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:45:57,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:45:57,381 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:45:57,619 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:46:00,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:46:05,621 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:46:10,621 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:46:12,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:46:12,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:46:12,381 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:46:16,615 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:46:21,616 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:46:26,616 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:46:27,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:46:27,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:46:27,381 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:46:27,652 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:46:31,653 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:46:36,087 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 04:46:36,090 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 04:46:36,090 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 04:46:36,092 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 04:46:36,481 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 04:46:37,092 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:46:38,482 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 04:46:42,093 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:46:42,380 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:46:42,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:46:42,381 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:46:47,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:46:52,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:46:57,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:46:57,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:46:57,381 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:46:57,621 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:46:57,621 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:47:02,622 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:47:07,622 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:47:12,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:47:12,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:47:12,381 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:47:12,638 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:47:17,638 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:47:22,639 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:47:27,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:47:27,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:47:27,382 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:47:27,594 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:47:28,594 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:47:33,595 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:47:38,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:47:42,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:47:42,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:47:42,382 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:47:44,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:47:49,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:47:54,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:47:57,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:47:57,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:47:57,382 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:47:57,595 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:47:59,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:48:04,597 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:48:09,597 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:48:12,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:48:12,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:48:12,382 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:48:14,644 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:48:19,644 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:48:24,645 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:48:27,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:48:27,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:48:27,382 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:48:27,646 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:48:29,647 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:48:34,647 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:48:39,648 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:48:42,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:48:42,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:48:42,382 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:48:44,648 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:48:49,649 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:48:54,649 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:48:57,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:48:57,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:48:57,382 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:48:57,645 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:49:00,646 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:49:05,646 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:49:10,646 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:49:12,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:49:12,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:49:12,382 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:49:16,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:49:21,621 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:49:26,621 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:49:27,381 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:49:27,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:49:27,382 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:49:27,597 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:49:32,598 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:49:37,599 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:49:42,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:49:42,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:49:42,382 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:49:42,634 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:49:47,635 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:49:52,636 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:49:57,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:49:57,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:49:57,382 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:49:57,598 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:49:58,599 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:50:03,599 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:50:08,600 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:50:12,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:50:12,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:50:12,383 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:50:13,629 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:50:18,629 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:50:23,630 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:50:27,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:50:27,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:50:27,383 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:50:27,652 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:50:28,653 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:50:33,653 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:50:38,654 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:50:42,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:50:42,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:50:42,383 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:50:44,625 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:50:49,625 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:50:54,626 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:50:57,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:50:57,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:50:57,383 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:50:57,600 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:51:00,601 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:51:05,601 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:51:10,602 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:51:12,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:51:12,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:51:12,383 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:51:16,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:51:21,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:51:26,593 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:51:27,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:51:27,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:51:27,383 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:51:28,197 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:51:32,198 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:51:37,198 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:51:40,176 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 04:51:40,178 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 04:51:40,178 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 04:51:40,180 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 04:51:40,648 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 04:51:42,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:51:42,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:51:42,383 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:51:42,632 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:51:42,649 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 04:51:47,633 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:51:52,633 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:51:57,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:51:57,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:51:57,383 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:51:57,602 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:51:58,602 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:52:03,603 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:52:08,604 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:52:12,382 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:52:12,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:52:12,383 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:52:13,653 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:52:18,654 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:52:23,655 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:52:27,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:52:27,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:52:27,383 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:52:27,616 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:52:29,616 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:52:34,617 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:52:39,617 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:52:42,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:52:42,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:52:42,383 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:52:45,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:52:50,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:52:55,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:52:57,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:52:57,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:52:57,384 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:52:57,633 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:53:00,634 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:53:05,634 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:53:10,635 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:53:12,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:53:12,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:53:12,384 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:53:16,624 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:53:21,624 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:53:26,625 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:53:27,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:53:27,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:53:27,384 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:53:27,745 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:53:31,746 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:53:36,747 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:53:41,747 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:53:42,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:53:42,384 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:53:42,384 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:53:47,650 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:53:52,650 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:53:57,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:53:57,384 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:53:57,384 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:53:57,605 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:53:58,606 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:54:03,607 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:54:08,607 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:54:12,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:54:12,384 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:54:12,384 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:54:13,681 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:54:18,681 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:54:23,682 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:54:27,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:54:27,384 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:54:27,384 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:54:27,609 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:54:29,610 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:54:34,610 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:54:39,611 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:54:42,383 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:54:42,384 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:54:42,384 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:54:45,591 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:54:50,591 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:54:55,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:54:57,384 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:54:57,384 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:54:57,385 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:54:57,607 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:55:00,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:55:05,609 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:55:10,609 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:55:12,384 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:55:12,384 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:55:12,385 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:55:15,619 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:55:20,619 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:55:25,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:55:27,384 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:55:27,384 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:55:27,385 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:55:27,627 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:55:30,628 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:55:35,628 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:55:40,629 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:55:42,384 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:55:42,384 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:55:42,385 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:55:46,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:55:51,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:55:56,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:55:57,384 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:55:57,384 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:55:57,385 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:55:57,609 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:56:01,609 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:56:06,610 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:56:11,611 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:56:12,384 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:56:12,385 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:56:12,385 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:56:16,643 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:56:21,644 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:56:26,644 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:56:27,384 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:56:27,385 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:56:27,385 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:56:27,626 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:56:32,627 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:56:37,627 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:56:42,384 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:56:42,385 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:56:42,385 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:56:43,622 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:56:44,240 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 04:56:44,243 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 04:56:44,244 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 04:56:44,245 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 04:56:44,811 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 04:56:46,812 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 04:56:49,282 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:56:54,282 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:56:57,384 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:56:57,385 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:56:57,385 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:56:57,663 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:56:59,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:57:04,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:57:09,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:57:12,384 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:57:12,385 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:57:12,385 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:57:15,651 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:57:20,652 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:57:25,652 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:57:27,385 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:57:27,385 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:57:27,385 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:57:27,611 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:57:31,612 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:57:36,613 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:57:41,613 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:57:42,385 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:57:42,385 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:57:42,385 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:57:46,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:57:51,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:57:56,666 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:57:57,385 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:57:57,385 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:57:57,385 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:57:57,613 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:58:02,614 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:58:07,614 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:58:12,385 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:58:12,385 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:58:12,386 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:58:13,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:58:18,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:58:23,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:58:27,385 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:58:27,385 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:58:27,386 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:58:27,671 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:58:28,672 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:58:33,672 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:58:38,673 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:58:42,385 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:58:42,386 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:58:42,386 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:58:44,646 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:58:49,647 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:58:54,647 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:58:57,385 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:58:57,386 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:58:57,386 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:58:57,660 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:58:59,660 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:59:04,661 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:59:09,661 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:59:12,385 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:59:12,386 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:59:12,386 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:59:15,631 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:59:20,632 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:59:25,632 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:59:27,385 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:59:27,386 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:59:27,386 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:59:27,614 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:59:31,615 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:59:36,616 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:59:41,617 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:59:42,385 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:59:42,386 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:59:42,386 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:59:47,559 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:59:52,559 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 04:59:57,386 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 04:59:57,386 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 04:59:57,386 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 04:59:57,643 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 04:59:57,643 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:00:02,644 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:00:07,645 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:00:12,386 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:00:12,386 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:00:12,387 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:00:12,669 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:00:17,670 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:00:22,671 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:00:27,386 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:00:27,386 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:00:27,387 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:00:27,657 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:00:28,657 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:00:33,658 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:00:38,659 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:00:42,386 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:00:42,387 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:00:42,387 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:00:44,629 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:00:49,629 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:00:54,630 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:00:57,386 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:00:57,387 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:00:57,387 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:00:57,621 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:01:00,622 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:01:05,622 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:01:10,623 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:01:12,386 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:01:12,387 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:01:12,387 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:01:16,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:01:21,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:01:26,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:01:27,386 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:01:27,387 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:01:27,387 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:01:27,619 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:01:31,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:01:36,621 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:01:41,621 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:01:42,386 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:01:42,387 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:01:42,387 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:01:46,659 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:01:48,983 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 05:01:48,988 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 05:01:48,988 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 05:01:48,989 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 05:01:49,975 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 05:01:50,975 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 05:01:51,990 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:01:56,991 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:01:57,386 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:01:57,387 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:01:57,387 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:01:57,680 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:02:02,681 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:02:07,681 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:02:12,386 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:02:12,387 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:02:12,388 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:02:13,644 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:02:18,644 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:02:23,645 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:02:27,386 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:02:27,387 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:02:27,388 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:02:29,090 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:02:29,090 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:02:34,091 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:02:39,091 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:02:42,387 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:02:42,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:02:42,388 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:02:44,651 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:02:49,652 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:02:54,653 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:02:57,387 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:02:57,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:02:57,388 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:02:57,641 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:03:00,642 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:03:05,642 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:03:10,643 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:03:12,387 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:03:12,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:03:12,388 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:03:16,612 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:03:21,612 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:03:26,613 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:03:27,387 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:03:27,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:03:27,388 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:03:27,646 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:03:31,647 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:03:36,647 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:03:41,648 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:03:42,387 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:03:42,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:03:42,388 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:03:46,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:03:51,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:03:56,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:03:57,387 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:03:57,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:03:57,388 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:03:57,622 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:04:02,623 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:04:07,624 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:04:12,387 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:04:12,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:04:12,388 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:04:13,622 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:04:18,622 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:04:23,623 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:04:27,387 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:04:27,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:04:27,388 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:04:27,647 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:04:28,647 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:04:33,648 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:04:38,648 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:04:42,387 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:04:42,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:04:42,388 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:04:44,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:04:49,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:04:54,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:04:57,387 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:04:57,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:04:57,388 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:04:57,647 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:04:59,648 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:05:04,648 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:05:09,649 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:05:12,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:05:12,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:05:12,388 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:05:15,649 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:05:20,650 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:05:25,650 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:05:27,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:05:27,389 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:05:27,389 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:05:27,648 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:05:31,649 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:05:36,650 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:05:41,650 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:05:42,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:05:42,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:05:42,389 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:05:47,612 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:05:52,613 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:05:57,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:05:57,389 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:05:57,389 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:05:57,659 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:05:57,659 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:06:02,660 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:06:07,661 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:06:12,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:06:12,389 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:06:12,389 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:06:13,661 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:06:18,661 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:06:23,662 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:06:27,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:06:27,389 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:06:27,389 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:06:27,650 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:06:29,651 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:06:34,651 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:06:39,652 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:06:42,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:06:42,389 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:06:42,389 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:06:45,624 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:06:50,625 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:06:53,307 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 05:06:53,309 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 05:06:53,310 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 05:06:53,312 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 05:06:54,133 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 05:06:55,133 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 05:06:56,313 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:06:57,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:06:57,389 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:06:57,389 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:06:57,651 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:07:01,651 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:07:06,652 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:07:11,653 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:07:12,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:07:12,389 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:07:12,390 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:07:17,647 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:07:22,648 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:07:27,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:07:27,389 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:07:27,389 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:07:27,664 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:07:27,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:07:32,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:07:37,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:07:42,389 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:07:42,389 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:07:42,390 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:07:42,803 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:07:47,803 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:07:52,804 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:07:57,388 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:07:57,389 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:07:57,389 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:07:57,655 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:07:58,656 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:08:03,656 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:08:08,657 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:08:12,389 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:08:12,389 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:08:12,390 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:08:13,666 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:08:18,667 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:08:23,667 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:08:27,389 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:08:27,389 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:08:27,390 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:08:27,653 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:08:29,654 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:08:34,655 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:08:39,655 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:08:42,389 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:08:42,389 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:08:42,390 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:08:45,637 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:08:50,638 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:08:55,638 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:08:57,389 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:08:57,390 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:08:57,390 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:08:57,655 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:09:00,656 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:09:05,656 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:09:10,657 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:09:12,389 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:09:12,390 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:09:12,390 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:09:15,667 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:09:20,667 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:09:25,670 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:09:27,389 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:09:27,390 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:09:27,390 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:09:27,655 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:09:31,656 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:09:36,656 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:09:41,657 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:09:42,389 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:09:42,390 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:09:42,390 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:09:47,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:09:52,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:09:57,389 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:09:57,390 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:09:57,390 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:09:57,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:09:57,656 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:10:02,657 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:10:07,657 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:10:12,389 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:10:12,390 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:10:12,390 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:10:13,647 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:10:18,648 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:10:23,648 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:10:27,389 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:10:27,390 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:10:27,390 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:10:27,678 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:10:28,679 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:10:33,680 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:10:38,681 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:10:42,390 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:10:42,390 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:10:42,390 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:10:44,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:10:49,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:10:54,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:10:57,390 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:10:57,390 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:10:57,391 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:10:57,670 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:10:59,671 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:11:04,672 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:11:09,672 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:11:12,390 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:11:12,390 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:11:12,391 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:11:15,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:11:20,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:11:25,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:11:27,390 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:11:27,390 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:11:27,391 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:11:27,659 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:11:30,659 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:11:35,660 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:11:40,661 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:11:42,390 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:11:42,391 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:11:42,391 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:11:46,632 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:11:51,633 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:11:56,634 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:11:57,371 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 05:11:57,374 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 05:11:57,374 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 05:11:57,375 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 05:11:57,390 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:11:57,391 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:11:57,391 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:11:57,661 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:11:58,293 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 05:12:00,294 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 05:12:01,662 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:12:06,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:12:11,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:12:12,390 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:12:12,391 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:12:12,391 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:12:17,659 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:12:22,660 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:12:27,390 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:12:27,391 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:12:27,391 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:12:27,660 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:12:27,660 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:12:32,661 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:12:37,662 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:12:42,390 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:12:42,391 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:12:42,391 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:12:43,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:12:48,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:12:53,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:12:57,390 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:12:57,404 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:12:57,408 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:12:57,661 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:12:58,662 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:13:03,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:13:08,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:13:12,391 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:13:12,391 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:13:12,391 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:13:14,599 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:13:19,600 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:13:24,600 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:13:27,391 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:13:27,391 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:13:27,392 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:13:28,592 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:13:30,593 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:13:35,594 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:13:40,594 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:13:42,391 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:13:42,391 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:13:42,392 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:13:46,590 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:13:51,591 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:13:56,591 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:13:57,391 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:13:57,391 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:13:57,392 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:13:57,663 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:14:01,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:14:06,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:14:11,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:14:12,391 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:14:12,392 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:14:12,392 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:14:17,627 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:14:22,627 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:14:27,391 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:14:27,392 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:14:27,392 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:14:27,664 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:14:27,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:14:32,666 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:14:37,666 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:14:42,391 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:14:42,392 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:14:42,392 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:14:43,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:14:48,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:14:53,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:14:57,391 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:14:57,392 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:14:57,392 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:14:57,665 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:14:58,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:15:03,666 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:15:08,666 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:15:12,391 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:15:12,392 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:15:12,392 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:15:14,604 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:15:19,604 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:15:24,605 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:15:27,391 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:15:27,393 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:15:27,393 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:15:27,758 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:15:29,758 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:15:34,759 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:15:39,760 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:15:42,392 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:15:42,393 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:15:42,393 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:15:45,605 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:15:50,605 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:15:55,606 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:15:57,392 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:15:57,393 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:15:57,393 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:15:57,666 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:16:00,667 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:16:05,668 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:16:10,668 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:16:12,392 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:16:12,393 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:16:12,393 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:16:15,824 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:16:20,825 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:16:25,825 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:16:27,392 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:16:27,393 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:16:27,393 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:16:27,668 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:16:31,669 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:16:36,669 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:16:41,670 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:16:42,392 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:16:42,393 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:16:42,393 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:16:46,672 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:16:51,672 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:16:56,673 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:16:57,392 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:16:57,393 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:16:57,393 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:16:57,668 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:17:01,659 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 05:17:01,661 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 05:17:01,662 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 05:17:01,663 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 05:17:02,456 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 05:17:02,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:17:04,457 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 05:17:07,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:17:12,392 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:17:12,393 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:17:12,393 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:17:13,641 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:17:18,642 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:17:23,643 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:17:27,392 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:17:27,393 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:17:27,394 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:17:27,669 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:17:28,669 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:17:33,670 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:17:38,671 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:17:42,392 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:17:42,393 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:17:42,394 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:17:44,650 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:17:49,651 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:17:54,651 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:17:57,392 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:17:57,393 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:17:57,394 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:17:57,670 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:17:59,670 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:18:04,671 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:18:09,671 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:18:12,393 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:18:12,394 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:18:12,394 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:18:14,672 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:18:19,672 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:18:24,673 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:18:27,393 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:18:27,394 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:18:27,394 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:18:27,670 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:18:30,671 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:18:35,671 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:18:40,672 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:18:42,393 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:18:42,394 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:18:42,394 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:18:46,605 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:18:51,606 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:18:56,607 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:18:57,393 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:18:57,394 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:18:57,394 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:18:57,671 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:19:01,672 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:19:06,672 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:19:11,673 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:19:12,393 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:19:12,394 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:19:12,394 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:19:17,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:19:22,667 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:19:27,393 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:19:27,394 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:19:27,394 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:19:27,672 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:19:27,673 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:19:32,673 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:19:37,674 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:19:42,393 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:19:42,394 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:19:42,394 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:19:43,673 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:19:48,674 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:19:53,674 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:19:57,393 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:19:57,394 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:19:57,394 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:19:57,673 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:19:59,673 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:20:04,674 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:20:09,675 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:20:12,393 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:20:12,394 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:20:12,395 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:20:15,648 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:20:20,648 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:20:25,649 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:20:27,393 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:20:27,394 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:20:27,395 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:20:27,674 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:20:30,674 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:20:35,675 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:20:40,675 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:20:42,394 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:20:42,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:20:42,395 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:20:46,643 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:20:51,644 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:20:56,644 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:20:57,394 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:20:57,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:20:57,395 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:20:57,674 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:21:01,675 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:21:06,676 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:21:11,677 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:21:12,394 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:21:12,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:21:12,395 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:21:17,634 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:21:22,634 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:21:27,394 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:21:27,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:21:27,395 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:21:27,653 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:21:27,675 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:21:32,676 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:21:37,677 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:21:42,394 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:21:42,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:21:42,395 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:21:43,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:21:48,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:21:53,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:21:57,394 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:21:57,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:21:57,395 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:21:57,676 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:21:58,677 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:22:03,677 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:22:05,511 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 05:22:05,513 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 05:22:05,514 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 05:22:05,515 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 05:22:05,618 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 05:22:06,619 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 05:22:09,516 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:22:12,394 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:22:12,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:22:12,395 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:22:14,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:22:19,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:22:24,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:22:27,394 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:22:27,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:22:27,395 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:22:27,677 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:22:29,678 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:22:34,678 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:22:39,679 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:22:42,394 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:22:42,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:22:42,396 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:22:45,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:22:50,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:22:55,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:22:57,394 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:22:57,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:22:57,396 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:22:57,678 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:23:00,678 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:23:05,679 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:23:10,680 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:23:12,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:23:12,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:23:12,396 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:23:16,649 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:23:21,650 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:23:26,650 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:23:27,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:23:27,396 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:23:27,396 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:23:27,680 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:23:31,681 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:23:36,681 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:23:41,682 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:23:42,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:23:42,396 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:23:42,396 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:23:47,649 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:23:52,650 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:23:57,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:23:57,396 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:23:57,396 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:23:57,679 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:23:57,680 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:24:02,680 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:24:07,681 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:24:12,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:24:12,396 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:24:12,396 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:24:13,648 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:24:18,648 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:24:23,649 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:24:27,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:24:27,396 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:24:27,396 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:24:27,681 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:24:28,682 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:24:33,682 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:24:38,683 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:24:42,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:24:42,396 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:24:42,397 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:24:44,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:24:49,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:24:54,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:24:57,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:24:57,396 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:24:57,397 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:24:57,681 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:24:59,682 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:25:04,682 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:25:09,683 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:25:12,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:25:12,396 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:25:12,397 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:25:15,621 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:25:20,622 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:25:25,622 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:25:27,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:25:27,397 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:25:27,397 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:25:27,682 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:25:30,683 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:25:35,683 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:25:40,684 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:25:42,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:25:42,397 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:25:42,397 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:25:46,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:25:51,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:25:56,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:25:57,396 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:25:57,397 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:25:57,397 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:25:57,683 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:26:01,684 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:26:06,684 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:26:11,685 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:26:12,396 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:26:12,397 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:26:12,397 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:26:17,652 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:26:22,653 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:26:27,396 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:26:27,397 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:26:27,397 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:26:27,684 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:26:27,685 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:26:32,685 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:26:37,685 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:26:42,396 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:26:42,397 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:26:42,397 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:26:43,667 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:26:48,667 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:26:53,668 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:26:57,396 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:26:57,397 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:26:57,397 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:26:57,684 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:26:58,685 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:27:03,686 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:27:08,686 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:27:09,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 05:27:09,582 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 05:27:09,582 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 05:27:09,584 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 05:27:09,777 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 05:27:10,778 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 05:27:12,396 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:27:12,397 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:27:12,397 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:27:14,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:27:19,597 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:27:24,597 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:27:27,396 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:27:27,397 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:27:27,398 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:27:27,685 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:27:29,686 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:27:34,686 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:27:39,687 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:27:42,396 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:27:42,398 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:27:42,398 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:27:45,656 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:27:50,657 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:27:55,658 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:27:57,396 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:27:57,398 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:27:57,398 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:27:57,686 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:28:00,687 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:28:05,687 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:28:10,688 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:28:12,396 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:28:12,398 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:28:12,398 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:28:16,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:28:21,666 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:28:26,667 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:28:27,396 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:28:27,398 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:28:27,398 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:28:27,687 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:28:31,688 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:28:36,688 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:28:41,689 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:28:42,397 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:28:42,398 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:28:42,398 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:28:47,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:28:52,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:28:57,397 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:28:57,398 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:28:57,398 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:28:57,678 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:28:57,688 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:29:02,689 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:29:07,689 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:29:12,397 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:29:12,398 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:29:12,398 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:29:13,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:29:18,609 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:29:23,609 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:29:27,397 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:29:27,398 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:29:27,399 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:29:27,689 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:29:28,689 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:29:33,690 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:29:38,691 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:29:42,397 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:29:42,398 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:29:42,399 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:29:44,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:29:49,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:29:54,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:29:57,444 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:29:57,445 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:29:57,445 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:29:57,446 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:29:57,699 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:29:59,700 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:30:04,701 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:30:09,701 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:30:12,444 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:30:12,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:30:12,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:30:12,489 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:30:15,671 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:30:20,671 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:30:25,672 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:30:27,445 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:30:27,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:30:27,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:30:27,488 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:30:27,692 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:30:30,693 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:30:35,694 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:30:40,694 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:30:42,445 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:30:42,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:30:42,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:30:42,488 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:30:45,747 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:30:50,748 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:30:55,748 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:30:57,445 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:30:57,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:30:57,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:30:57,488 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:30:57,695 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:31:01,696 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:31:06,697 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:31:11,697 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:31:12,445 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:31:12,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:31:12,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:31:12,488 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:31:17,670 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:31:22,670 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:31:27,445 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:31:27,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:31:27,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:31:27,488 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:31:27,676 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:31:27,692 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:31:32,693 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:31:37,694 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:31:42,445 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:31:42,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:31:42,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:31:42,489 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:31:42,752 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:31:47,753 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:31:52,754 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:31:57,445 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:31:57,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:31:57,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:31:57,489 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:31:57,721 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:31:58,722 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:32:03,723 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:32:08,723 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:32:12,445 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:32:12,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:32:12,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:32:12,489 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:32:13,287 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 05:32:13,288 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 05:32:13,289 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 05:32:13,290 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 05:32:13,936 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 05:32:14,329 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:32:14,936 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 05:32:19,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:32:24,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:32:27,445 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:32:27,446 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:32:27,446 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:32:27,447 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:32:27,694 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:32:29,695 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:32:34,695 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:32:39,696 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:32:42,445 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:32:42,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:32:42,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:32:42,489 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:32:44,699 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:32:49,700 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:32:54,700 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:32:57,446 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:32:57,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:32:57,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:32:57,488 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:32:57,695 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:33:00,696 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:33:05,697 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:33:10,698 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:33:12,446 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:33:12,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:33:12,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:33:12,489 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:33:15,748 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:33:20,748 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:33:25,749 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:33:27,446 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:33:27,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:33:27,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:33:27,488 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:33:27,768 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:33:30,769 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:33:35,770 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:33:40,770 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:33:42,446 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:33:42,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:33:42,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:33:42,488 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:33:46,667 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:33:51,668 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:33:56,668 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:33:57,446 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:33:57,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:33:57,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:33:57,488 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:33:57,747 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:34:01,748 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:34:06,749 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:34:11,749 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:34:12,446 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:34:12,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:34:12,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:34:12,488 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:34:17,704 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:34:22,704 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:34:27,446 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:34:27,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:34:27,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:34:27,488 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:34:27,724 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:34:27,725 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:34:32,725 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:34:37,726 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:34:42,446 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:34:42,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:34:42,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:34:42,489 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:34:42,753 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:34:47,753 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:34:52,754 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:34:57,447 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:34:57,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:34:57,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:34:57,488 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:34:57,699 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:34:58,700 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:35:03,701 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:35:08,701 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:35:12,447 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:35:12,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:35:12,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:35:12,488 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:35:13,761 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:35:18,762 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:35:23,762 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:35:27,447 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:35:27,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:35:27,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:35:27,488 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:35:27,699 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:35:29,700 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:35:34,701 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:35:39,701 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:35:42,447 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:35:42,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:35:42,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:35:42,488 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:35:44,747 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:35:49,748 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:35:54,749 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:35:57,447 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:35:57,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:35:57,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:35:57,488 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:35:57,738 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:36:00,739 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:36:05,740 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:36:10,740 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:36:12,447 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:36:12,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:36:12,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:36:12,488 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:36:15,743 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:36:20,744 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:36:25,745 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:36:27,447 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:36:27,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:36:27,488 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:36:27,489 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:36:27,726 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:36:31,726 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:36:36,727 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:36:41,728 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:36:42,526 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:36:42,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:36:42,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:36:42,569 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:36:46,844 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:36:51,845 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:36:56,845 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:36:57,526 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:36:57,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:36:57,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:36:57,569 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:36:57,755 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:37:02,756 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:37:07,757 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:37:12,526 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:37:12,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:37:12,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:37:12,569 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:37:13,757 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:37:17,876 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 05:37:17,877 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 05:37:17,878 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 05:37:17,879 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 05:37:18,104 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 05:37:18,917 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:37:19,105 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 05:37:23,918 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:37:27,527 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:37:27,527 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:37:27,527 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:37:27,527 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:37:27,782 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:37:29,783 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:37:34,784 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:37:39,784 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:37:42,527 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:37:42,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:37:42,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:37:42,569 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:37:44,817 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:37:49,818 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:37:54,819 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:37:57,527 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:37:57,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:37:57,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:37:57,568 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:37:57,808 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:38:00,808 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:38:05,809 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:38:10,809 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:38:12,527 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:38:12,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:38:12,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:38:12,568 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:38:15,889 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:38:20,890 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:38:25,890 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:38:27,527 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:38:27,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:38:27,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:38:27,568 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:38:27,816 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:38:31,817 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:38:36,817 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:38:41,818 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:38:42,527 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:38:42,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:38:42,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:38:42,568 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:38:47,780 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:38:52,781 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:38:57,527 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:38:57,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:38:57,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:38:57,569 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:38:57,828 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:38:57,828 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:39:02,829 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:39:07,830 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:39:12,527 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:39:12,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:39:12,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:39:12,569 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:39:13,816 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:39:18,817 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:39:23,817 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:39:27,527 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:39:27,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:39:27,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:39:27,569 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:39:27,779 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:39:29,780 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:39:34,780 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:39:39,781 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:39:42,527 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:39:42,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:39:42,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:39:42,569 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:39:44,826 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:39:49,827 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:39:54,827 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:39:57,528 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:39:57,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:39:57,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:39:57,569 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:39:57,828 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:39:59,828 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:40:04,829 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:40:09,830 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:40:12,528 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:40:12,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:40:12,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:40:12,568 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:40:15,753 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:40:20,754 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:40:25,754 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:40:27,528 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:40:27,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:40:27,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:40:27,572 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:40:27,888 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:40:30,889 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:40:35,890 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:40:40,890 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:40:42,528 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:40:42,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:40:42,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:40:42,572 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:40:46,822 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:40:51,822 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:40:56,823 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:40:57,528 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:40:57,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:40:57,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:40:57,572 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:40:57,834 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:41:01,835 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:41:06,836 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:41:11,836 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:41:12,528 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:41:12,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:41:12,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:41:12,572 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:41:17,770 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:41:22,771 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:41:27,528 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:41:27,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:41:27,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:41:27,572 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:41:27,821 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:41:27,821 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:41:32,822 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:41:37,823 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:41:42,529 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:41:42,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:41:42,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:41:42,572 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:41:43,815 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:41:48,815 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:41:53,816 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:41:57,529 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:41:57,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:41:57,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:41:57,572 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:41:57,822 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:41:58,822 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:42:03,823 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:42:08,824 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:42:12,529 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:42:12,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:42:12,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:42:12,572 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:42:14,767 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:42:19,768 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:42:21,911 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 05:42:21,913 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 05:42:21,913 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 05:42:21,914 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 05:42:22,269 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 05:42:24,271 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 05:42:24,953 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:42:27,529 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:42:27,529 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:42:27,530 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:42:27,530 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:42:27,724 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:42:30,725 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:42:35,725 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:42:40,726 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:42:42,529 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:42:42,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:42:42,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:42:42,573 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:42:45,790 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:42:50,790 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:42:55,791 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:42:57,529 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:42:57,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:42:57,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:42:57,573 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:42:57,783 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:43:01,784 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:43:06,784 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:43:11,785 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:43:12,529 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:43:12,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:43:12,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:43:12,573 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:43:17,782 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:43:22,783 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:43:27,529 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:43:27,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:43:27,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:43:27,572 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:43:27,753 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:43:28,754 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:43:33,755 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:43:38,755 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:43:42,530 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:43:42,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:43:42,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:43:42,573 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:43:43,840 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:43:48,841 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:43:53,841 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:43:57,530 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:43:57,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:43:57,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:43:57,573 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:43:57,833 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:43:59,833 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:44:04,834 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:44:09,835 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:44:12,530 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:44:12,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:44:12,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:44:12,573 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:44:15,751 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:44:20,751 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:44:25,752 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:44:27,530 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:44:27,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:44:27,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:44:27,573 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:44:27,750 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:44:31,751 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:44:36,751 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:44:41,752 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:44:42,530 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:44:42,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:44:42,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:44:42,573 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:44:46,820 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:44:51,821 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:44:56,821 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:44:57,530 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:44:57,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:44:57,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:44:57,573 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:44:57,794 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:45:02,795 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:45:07,795 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:45:12,530 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:45:12,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:45:12,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:45:12,573 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:45:12,797 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:45:17,797 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:45:22,798 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:45:27,530 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:45:27,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:45:27,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:45:27,572 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:45:27,844 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:45:27,844 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:45:32,845 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:45:37,845 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:45:42,530 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:45:42,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:45:42,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:45:42,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:45:43,839 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:45:48,840 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:45:53,840 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:45:57,530 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:45:57,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:45:57,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:45:57,572 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:45:57,812 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:45:59,813 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:46:04,813 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:46:09,814 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:46:12,531 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:46:12,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:46:12,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:46:12,573 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:46:14,839 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:46:19,839 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:46:24,840 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:46:27,531 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:46:27,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:46:27,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:46:27,572 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:46:27,794 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:46:30,795 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:46:35,796 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:46:40,797 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:46:42,531 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:46:42,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:46:42,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:46:42,573 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:46:45,806 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:46:50,807 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:46:55,807 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:46:57,531 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:46:57,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:46:57,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:46:57,572 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:46:57,846 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:47:00,847 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:47:05,847 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:47:10,848 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:47:12,531 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:47:12,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:47:12,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:47:12,572 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:47:16,756 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:47:21,757 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:47:25,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 05:47:25,581 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 05:47:25,581 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 05:47:25,582 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 05:47:26,450 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 05:47:27,531 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:47:27,531 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:47:27,532 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:47:27,532 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:47:27,716 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:47:27,721 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:47:28,451 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 05:47:32,722 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:47:37,723 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:47:42,531 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:47:42,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:47:42,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:47:42,573 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:47:42,833 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:47:47,833 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:47:52,835 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:47:57,531 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:47:57,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:47:57,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:47:57,573 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:47:57,822 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:47:58,823 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:48:03,823 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:48:08,824 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:48:12,531 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:48:12,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:48:12,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:48:12,573 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:48:13,846 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:48:18,847 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:48:23,847 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:48:27,531 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:48:27,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:48:27,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:48:27,573 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:48:27,841 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:48:29,842 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:48:34,843 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:48:39,843 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:48:42,532 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:48:42,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:48:42,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:48:42,573 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:48:45,810 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:48:50,811 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:48:55,811 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:48:57,532 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:48:57,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:48:57,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:48:57,573 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:48:57,876 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:49:00,877 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:49:05,877 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:49:10,878 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:49:12,532 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:49:12,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:49:12,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:49:12,573 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:49:16,809 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:49:21,809 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:49:26,810 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:49:27,532 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:49:27,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:49:27,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:49:27,572 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:49:27,845 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:49:31,846 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:49:36,847 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:49:41,847 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:49:42,532 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:49:42,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:49:42,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:49:42,572 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:49:47,827 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:49:52,828 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:49:57,532 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:49:57,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:49:57,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:49:57,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:49:57,822 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:49:58,822 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:50:03,823 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:50:08,824 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:50:12,532 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:50:12,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:50:12,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:50:12,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:50:14,761 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:50:19,762 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:50:24,762 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:50:27,532 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:50:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:50:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:50:27,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:50:27,777 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:50:29,778 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:50:34,779 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:50:39,779 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:50:42,532 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:50:42,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:50:42,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:50:42,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:50:44,794 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:50:49,795 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:50:54,796 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:50:57,533 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:50:57,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:50:57,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:50:57,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:50:57,804 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:50:59,804 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:51:04,805 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:51:09,805 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:51:12,533 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:51:12,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:51:12,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:51:12,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:51:14,857 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:51:19,858 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:51:24,858 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:51:27,533 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:51:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:51:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:51:27,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:51:27,854 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:51:30,855 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:51:35,855 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:51:40,856 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:51:42,533 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:51:42,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:51:42,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:51:42,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:51:46,819 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:51:51,819 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:51:56,820 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:51:57,533 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:51:57,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:51:57,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:51:57,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:51:57,863 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:52:01,864 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:52:06,865 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:52:11,866 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:52:12,533 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:52:12,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:52:12,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:52:12,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:52:17,849 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:52:22,850 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:52:27,533 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:52:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:52:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:52:27,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:52:27,800 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:52:28,801 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:52:30,048 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 05:52:30,049 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 05:52:30,050 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 05:52:30,051 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 05:52:30,617 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 05:52:32,618 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 05:52:34,090 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:52:39,090 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:52:42,533 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:52:42,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:52:42,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:52:42,534 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:52:44,711 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:52:49,712 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:52:54,713 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:52:57,533 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:52:57,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:52:57,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:52:57,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:52:57,811 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:52:59,811 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:53:04,812 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:53:09,813 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:53:12,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:53:12,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:53:12,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:53:12,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:53:15,774 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:53:20,774 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:53:25,775 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:53:27,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:53:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:53:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:53:27,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:53:27,833 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:53:30,834 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:53:35,835 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:53:40,835 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:53:42,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:53:42,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:53:42,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:53:42,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:53:45,848 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:53:50,848 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:53:55,849 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:53:57,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:53:57,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:53:57,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:53:57,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:53:57,851 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:54:00,852 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:54:05,852 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:54:10,853 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:54:12,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:54:12,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:54:12,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:54:12,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:54:16,831 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:54:21,831 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:54:26,832 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:54:27,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:54:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:54:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:54:27,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:54:27,867 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:54:31,868 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:54:36,868 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:54:41,869 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:54:42,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:54:42,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:54:42,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:54:42,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:54:47,779 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:54:52,779 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:54:57,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:54:57,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:54:57,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:54:57,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:54:57,839 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:54:57,839 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:55:02,840 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:55:07,840 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:55:12,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:55:12,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:55:12,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:55:12,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:55:13,777 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:55:18,777 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:55:23,778 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:55:27,534 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:55:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:55:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:55:27,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:55:27,833 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:55:28,833 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:55:33,834 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:55:38,834 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:55:42,535 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:55:42,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:55:42,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:55:42,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:55:44,810 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:55:49,810 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:55:54,811 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:55:57,535 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:55:57,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:55:57,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:55:57,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:55:57,816 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:55:59,817 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:56:04,817 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:56:09,818 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:56:12,535 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:56:12,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:56:12,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:56:12,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:56:15,760 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:56:20,761 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:56:25,761 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:56:27,535 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:56:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:56:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:56:27,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:56:28,392 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:56:31,393 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:56:36,393 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:56:41,394 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:56:42,535 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:56:42,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:56:42,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:56:42,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:56:46,811 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:56:51,811 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:56:56,812 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:56:57,535 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:56:57,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:56:57,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:56:57,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:56:57,778 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:57:02,779 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:57:07,780 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:57:12,535 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:57:12,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:57:12,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:57:12,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:57:13,777 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:57:18,778 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:57:23,778 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:57:27,535 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:57:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:57:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:57:27,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:57:27,841 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:57:28,842 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:57:33,842 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:57:33,855 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 05:57:33,857 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 05:57:33,857 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 05:57:33,859 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 05:57:34,781 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 05:57:34,781 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 05:57:38,898 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:57:42,535 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:57:42,536 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:57:42,536 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:57:42,536 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:57:44,722 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:57:49,723 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:57:54,723 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:57:57,535 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:57:57,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:57:57,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:57:57,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:57:57,764 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:57:59,765 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:58:04,766 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:58:09,766 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:58:12,536 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:58:12,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:58:12,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:58:12,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:58:14,795 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:58:19,796 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:58:24,796 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:58:27,536 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:58:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:58:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:58:27,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:58:27,802 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:58:29,803 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:58:34,803 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:58:39,804 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:58:42,536 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:58:42,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:58:42,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:58:42,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:58:45,789 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:58:50,790 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:58:55,790 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:58:57,536 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:58:57,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:58:57,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:58:57,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:58:57,826 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:59:00,827 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:59:05,828 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:59:10,829 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:59:12,536 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:59:12,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:59:12,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:59:12,581 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:59:16,753 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:59:21,754 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:59:26,754 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:59:27,536 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:59:27,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:59:27,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:59:27,580 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:59:27,792 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 05:59:31,793 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:59:36,794 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:59:41,795 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:59:42,536 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:59:42,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:59:42,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:59:42,580 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:59:46,845 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:59:51,846 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:59:56,846 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 05:59:57,536 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 05:59:57,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 05:59:57,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 05:59:57,580 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 05:59:57,858 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:00:01,859 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:00:06,859 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:00:11,860 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:00:12,537 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:00:12,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:00:12,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:00:12,580 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:00:17,795 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:00:22,795 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:00:27,537 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:00:27,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:00:27,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:00:27,581 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:00:27,846 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:00:27,847 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:00:32,847 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:00:37,848 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:00:42,537 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:00:42,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:00:42,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:00:42,581 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:00:43,757 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:00:48,758 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:00:53,758 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:00:57,537 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:00:57,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:00:57,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:00:57,580 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:00:57,839 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:00:58,839 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:01:03,840 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:01:08,840 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:01:12,537 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:01:12,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:01:12,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:01:12,581 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:01:14,815 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:01:19,815 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:01:24,816 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:01:27,537 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:01:27,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:01:27,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:01:27,581 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:01:27,819 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:01:29,820 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:01:34,821 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:01:39,821 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:01:42,537 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:01:42,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:01:42,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:01:42,581 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:01:45,773 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:01:50,773 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:01:55,774 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:01:57,537 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:01:57,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:01:57,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:01:57,581 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:01:57,827 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:02:00,828 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:02:05,829 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:02:10,829 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:02:12,537 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:02:12,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:02:12,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:02:12,581 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:02:16,791 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:02:21,791 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:02:26,792 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:02:27,538 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:02:27,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:02:27,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:02:27,580 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:02:27,810 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:02:31,811 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:02:36,812 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:02:38,367 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 06:02:38,369 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 06:02:38,369 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 06:02:38,371 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 06:02:38,944 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 06:02:40,945 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 06:02:42,409 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:02:42,538 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:02:42,538 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:02:42,538 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:02:42,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:02:47,778 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:02:52,778 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:02:57,538 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:02:57,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:02:57,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:02:57,580 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:02:57,785 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:02:57,786 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:03:02,786 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:03:07,787 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:03:12,538 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:03:12,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:03:12,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:03:12,580 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:03:13,768 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:03:18,769 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:03:23,770 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:03:27,538 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:03:27,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:03:27,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:03:27,580 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:03:27,859 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:03:28,860 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:03:33,860 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:03:38,861 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:03:42,538 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:03:42,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:03:42,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:03:42,580 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:03:44,755 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:03:49,756 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:03:54,756 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:03:57,538 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:03:57,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:03:57,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:03:57,580 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:03:57,847 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:03:59,848 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:04:04,849 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:04:09,849 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:04:12,538 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:04:12,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:04:12,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:04:12,581 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:04:15,753 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:04:20,754 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:04:25,755 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:04:27,538 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:04:27,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:04:27,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:04:27,580 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:04:27,840 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:04:30,841 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:04:35,842 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:04:40,843 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:04:42,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:04:42,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:04:42,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:04:42,580 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:04:46,793 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:04:51,793 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:04:56,794 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:04:57,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:04:57,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:04:57,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:04:57,580 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:04:57,848 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:05:01,849 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:05:06,849 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:05:11,850 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:05:12,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:05:12,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:05:12,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:05:12,581 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:05:17,780 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:05:22,781 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:05:27,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:05:27,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:05:27,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:05:27,581 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:05:27,846 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:05:27,847 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:05:32,847 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:05:37,847 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:05:42,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:05:42,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:05:42,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:05:42,581 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:05:43,807 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:05:48,808 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:05:53,808 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:05:57,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:05:57,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:05:57,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:05:57,581 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:05:57,788 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:05:59,789 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:06:04,790 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:06:09,790 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:06:12,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:06:12,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:06:12,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:06:12,581 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:06:14,801 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:06:19,801 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:06:24,802 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:06:27,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:06:27,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:06:27,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:06:27,581 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:06:27,780 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:06:30,781 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:06:35,782 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:06:40,782 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:06:42,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:06:42,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:06:42,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:06:42,581 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:06:45,811 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:06:50,811 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:06:55,812 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:06:57,539 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:06:57,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:06:57,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:06:57,581 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:06:57,853 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:07:00,854 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:07:05,855 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:07:10,855 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:07:12,540 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:07:12,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:07:12,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:07:12,581 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:07:16,789 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:07:21,790 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:07:26,790 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:07:27,540 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:07:27,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:07:27,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:07:27,581 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:07:27,836 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:07:31,836 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:07:36,837 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:07:41,838 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:07:42,327 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 06:07:42,329 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 06:07:42,329 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 06:07:42,331 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 06:07:42,540 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:07:42,542 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:07:42,542 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:07:42,542 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:07:43,103 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 06:07:43,104 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 06:07:47,809 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:07:52,810 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:07:57,540 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:07:57,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:07:57,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:07:57,580 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:07:57,789 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:07:58,790 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:08:03,791 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:08:08,791 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:08:12,540 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:08:12,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:08:12,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:08:12,581 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:08:13,843 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:08:18,843 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:08:23,844 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:08:27,540 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:08:27,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:08:27,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:08:27,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:08:27,865 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:08:28,866 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:08:33,866 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:08:38,867 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:08:42,540 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:08:42,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:08:42,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:08:42,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:08:44,852 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:08:49,853 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:08:54,854 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:08:57,540 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:08:57,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:08:57,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:08:57,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:08:57,819 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:09:00,819 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:09:05,820 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:09:10,821 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:09:12,540 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:09:12,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:09:12,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:09:12,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:09:15,841 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:09:20,842 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:09:25,843 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:09:27,541 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:09:27,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:09:27,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:09:27,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:09:27,775 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:09:31,776 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:09:36,776 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:09:41,777 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:09:42,541 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:09:42,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:09:42,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:09:42,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:09:46,778 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:09:51,779 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:09:56,779 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:09:57,541 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:09:57,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:09:57,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:09:57,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:09:57,811 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:10:01,812 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:10:06,812 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:10:11,813 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:10:12,541 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:10:12,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:10:12,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:10:12,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:10:16,852 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:10:21,853 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:10:26,853 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:10:27,541 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:10:27,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:10:27,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:10:27,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:10:27,846 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:10:32,847 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:10:37,848 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:10:42,541 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:10:42,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:10:42,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:10:42,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:10:43,782 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:10:48,783 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:10:53,784 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:10:57,541 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:10:57,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:10:57,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:10:57,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:10:57,827 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:10:58,828 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:11:03,829 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:11:08,829 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:11:12,541 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:11:12,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:11:12,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:11:12,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:11:13,856 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:11:18,857 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:11:23,857 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:11:27,541 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:11:27,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:11:27,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:11:27,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:11:27,826 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:11:29,826 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:11:34,827 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:11:39,827 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:11:42,542 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:11:42,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:11:42,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:11:42,584 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:11:44,842 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:11:49,843 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:11:54,844 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:11:57,542 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:11:57,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:11:57,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:11:57,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:11:57,791 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:12:00,792 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:12:05,793 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:12:10,793 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:12:12,542 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:12:12,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:12:12,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:12:12,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:12:16,760 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:12:21,760 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:12:26,761 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:12:27,542 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:12:27,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:12:27,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:12:27,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:12:27,776 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:12:31,777 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:12:36,777 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:12:41,778 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:12:42,542 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:12:42,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:12:42,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:12:42,584 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:12:46,751 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 06:12:46,753 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 06:12:46,753 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 06:12:46,754 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 06:12:46,793 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:12:47,267 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 06:12:47,267 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 06:12:51,794 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:12:56,794 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:12:57,542 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:12:57,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:12:57,546 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:12:57,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:12:57,801 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:13:01,802 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:13:06,803 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:13:11,803 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:13:12,542 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:13:12,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:13:12,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:13:12,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:13:17,791 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:13:22,792 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:13:27,542 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:13:27,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:13:27,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:13:27,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:13:27,784 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:13:28,785 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:13:33,785 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:13:38,786 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:13:42,542 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:13:42,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:13:42,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:13:42,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:13:43,850 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:13:48,850 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:13:53,851 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:13:57,542 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:13:57,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:13:57,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:13:57,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:13:57,796 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:13:59,796 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:14:04,797 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:14:09,797 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:14:12,543 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:14:12,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:14:12,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:14:12,584 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:14:14,811 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:14:19,812 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:14:24,812 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:14:27,543 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:14:27,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:14:27,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:14:27,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:14:27,807 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:14:30,808 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:14:35,808 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:14:40,809 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:14:42,543 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:14:42,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:14:42,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:14:42,584 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:14:45,820 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:14:50,820 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:14:55,821 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:14:57,543 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:14:57,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:14:57,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:14:57,584 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:14:57,811 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:15:01,812 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:15:06,813 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:15:11,813 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:15:12,543 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:15:12,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:15:12,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:15:12,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:15:17,793 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:15:22,793 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:15:27,543 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:15:27,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:15:27,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:15:27,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:15:27,772 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:15:28,773 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:15:33,773 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:15:38,774 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:15:42,543 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:15:42,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:15:42,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:15:42,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:15:43,776 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:15:48,777 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:15:53,778 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:15:57,543 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:15:57,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:15:57,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:15:57,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:15:57,813 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:15:58,814 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:16:03,815 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:16:08,815 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:16:12,544 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:16:12,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:16:12,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:16:12,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:16:14,780 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:16:19,781 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:16:24,781 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:16:27,544 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:16:27,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:16:27,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:16:27,584 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:16:27,777 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:16:30,778 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:16:35,778 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:16:40,779 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:16:42,544 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:16:42,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:16:42,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:16:42,584 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:16:46,758 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:16:51,758 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:16:56,759 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:16:57,544 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:16:57,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:16:57,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:16:57,584 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:16:57,775 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:17:01,776 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:17:06,776 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:17:11,777 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:17:12,544 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:17:12,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:17:12,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:17:12,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:17:16,786 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:17:21,787 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:17:26,787 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:17:27,544 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:17:27,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:17:27,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:17:27,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:17:27,796 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:17:31,797 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:17:36,797 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:17:41,798 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:17:42,544 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:17:42,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:17:42,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:17:42,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:17:46,823 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:17:50,408 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 06:17:50,409 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 06:17:50,410 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 06:17:50,411 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 06:17:50,428 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 06:17:52,429 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 06:17:52,449 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:17:57,450 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:17:57,544 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:17:57,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:17:57,547 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:17:57,549 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:17:57,776 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:18:02,777 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:18:07,778 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:18:12,544 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:18:12,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:18:12,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:18:12,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:18:12,854 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:18:17,855 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:18:22,855 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:18:27,544 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:18:27,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:18:27,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:18:27,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:18:27,800 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:18:28,800 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:18:33,801 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:18:38,802 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:18:42,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:18:42,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:18:42,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:18:42,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:18:44,782 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:18:49,783 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:18:54,784 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:18:57,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:18:57,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:18:57,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:18:57,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:18:57,810 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:18:59,811 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:19:04,811 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:19:09,812 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:19:12,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:19:12,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:19:12,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:19:12,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:19:14,854 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:19:19,855 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:19:24,855 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:19:27,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:19:27,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:19:27,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:19:27,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:19:27,865 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:19:29,865 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:19:34,866 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:19:39,866 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:19:42,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:19:42,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:19:42,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:19:42,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:19:44,874 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:19:49,875 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:19:54,876 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:19:57,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:19:57,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:19:57,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:19:57,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:19:57,853 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:20:00,854 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:20:05,855 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:20:10,855 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:20:12,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:20:12,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:20:12,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:20:12,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:20:16,843 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:20:21,843 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:20:26,844 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:20:27,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:20:27,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:20:27,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:20:27,589 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:20:27,786 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:20:32,787 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:20:37,787 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:20:42,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:20:42,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:20:42,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:20:42,589 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:20:42,853 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:20:47,854 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:20:52,854 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:20:57,545 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:20:57,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:20:57,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:20:57,589 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:20:57,815 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:20:58,816 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:21:03,817 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:21:08,817 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:21:12,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:21:12,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:21:12,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:21:12,589 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:21:14,806 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:21:19,806 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:21:24,807 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:21:27,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:21:27,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:21:27,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:21:27,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:21:27,816 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:21:29,817 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:21:34,817 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:21:39,818 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:21:42,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:21:42,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:21:42,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:21:42,589 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:21:44,882 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:21:49,882 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:21:54,883 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:21:57,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:21:57,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:21:57,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:21:57,589 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:21:57,793 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:22:00,794 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:22:05,794 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:22:10,795 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:22:12,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:22:12,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:22:12,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:22:12,589 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:22:15,843 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:22:20,843 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:22:25,844 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:22:27,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:22:27,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:22:27,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:22:27,589 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:22:27,785 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:22:31,785 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:22:36,786 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:22:41,787 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:22:42,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:22:42,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:22:42,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:22:42,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:22:47,785 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:22:52,785 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:22:54,103 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 06:22:54,105 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 06:22:54,105 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 06:22:54,106 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 06:22:54,590 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 06:22:56,591 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 06:22:57,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:22:57,550 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:22:57,550 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:22:57,550 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:22:57,802 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:22:57,802 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:23:02,803 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:23:07,803 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:23:12,546 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:23:12,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:23:12,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:23:12,589 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:23:13,765 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:23:18,765 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:23:23,766 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:23:27,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:23:27,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:23:27,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:23:27,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:23:27,798 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:23:28,798 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:23:33,799 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:23:38,800 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:23:42,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:23:42,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:23:42,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:23:42,589 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:23:43,869 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:23:48,870 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:23:53,871 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:23:57,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:23:57,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:23:57,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:23:57,589 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:23:57,878 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:23:58,879 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:24:03,879 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:24:08,880 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:24:12,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:24:12,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:24:12,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:24:12,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:24:14,784 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:24:19,785 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:24:24,785 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:24:27,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:24:27,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:24:27,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:24:27,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:24:27,834 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:24:29,835 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:24:34,835 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:24:39,836 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:24:42,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:24:42,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:24:42,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:24:42,589 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:24:44,881 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:24:49,882 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:24:54,883 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:24:57,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:24:57,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:24:57,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:24:57,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:24:57,818 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:25:00,818 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:25:05,819 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:25:10,820 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:25:12,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:25:12,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:25:12,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:25:12,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:25:15,843 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:25:20,844 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:25:25,845 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:25:27,547 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:25:27,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:25:27,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:25:27,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:25:27,820 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:25:31,820 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:25:36,821 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:25:41,822 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:25:42,548 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:25:42,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:25:42,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:25:42,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:25:46,895 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:25:51,895 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:25:56,896 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:25:57,548 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:25:57,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:25:57,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:25:57,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:25:57,830 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:26:02,831 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:26:07,831 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:26:12,548 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:26:12,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:26:12,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:26:12,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:26:13,784 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:26:18,785 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:26:23,786 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:26:27,548 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:26:27,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:26:27,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:26:27,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:26:27,804 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:26:28,804 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:26:33,805 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:26:38,805 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:26:42,548 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:26:42,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:26:42,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:26:42,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:26:43,815 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:26:48,815 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:26:53,816 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:26:57,548 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:26:57,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:26:57,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:26:57,592 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:26:57,852 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:26:58,852 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:27:03,853 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:27:08,854 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:27:12,548 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:27:12,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:27:12,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:27:12,593 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:27:14,827 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:27:19,828 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:27:24,828 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:27:27,548 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:27:27,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:27:27,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:27:27,593 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:27:27,885 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:27:29,886 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:27:34,886 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:27:39,887 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:27:42,549 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:27:42,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:27:42,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:27:42,592 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:27:45,779 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:27:50,780 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:27:55,780 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:27:57,551 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:27:57,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:27:57,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:27:57,592 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:27:57,863 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:27:58,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 06:27:58,317 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 06:27:58,317 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 06:27:58,319 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 06:27:58,751 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 06:28:00,752 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 06:28:01,357 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:28:06,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:28:11,358 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:28:12,551 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:28:12,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:28:12,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:28:12,552 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:28:16,795 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:28:21,796 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:28:26,796 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:28:27,551 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:28:27,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:28:27,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:28:27,592 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:28:27,795 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:28:32,796 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:28:37,797 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:28:42,551 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:28:42,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:28:42,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:28:42,593 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:28:42,814 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:28:47,815 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:28:52,815 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:28:57,551 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:28:57,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:28:57,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:28:57,593 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:28:57,817 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:28:57,817 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:29:02,818 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:29:07,819 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:29:12,551 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:29:12,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:29:12,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:29:12,593 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:29:13,780 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:29:18,781 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:29:23,781 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:29:27,551 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:29:27,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:29:27,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:29:27,592 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:29:27,802 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:29:28,803 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:29:33,803 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:29:38,804 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:29:42,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:29:42,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:29:42,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:29:42,592 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:29:43,860 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:29:48,861 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:29:53,861 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:29:57,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:29:57,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:29:57,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:29:57,593 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:29:57,843 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:29:59,844 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:30:04,844 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:30:09,845 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:30:12,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:30:12,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:30:12,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:30:12,592 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:30:15,836 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:30:20,836 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:30:25,837 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:30:27,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:30:27,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:30:27,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:30:27,592 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:30:27,834 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:30:31,835 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:30:36,836 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:30:41,836 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:30:42,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:30:42,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:30:42,592 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:30:42,593 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:30:47,819 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:30:52,820 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:30:57,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:30:57,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:30:57,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:30:57,596 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:30:57,801 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:30:58,802 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:31:03,802 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:31:08,803 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:31:12,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:31:12,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:31:12,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:31:12,596 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:31:13,843 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:31:18,844 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:31:23,845 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:31:27,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:31:27,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:31:27,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:31:27,597 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:31:27,801 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:31:29,802 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:31:34,802 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:31:39,803 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:31:42,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:31:42,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:31:42,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:31:42,596 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:31:44,882 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:31:49,882 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:31:54,883 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:31:57,552 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:31:57,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:31:57,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:31:57,596 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:31:57,802 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:32:00,803 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:32:05,804 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:32:10,804 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:32:12,553 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:32:12,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:32:12,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:32:12,597 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:32:15,820 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:32:20,821 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:32:25,821 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:32:27,553 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:32:27,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:32:27,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:32:27,597 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:32:27,823 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:32:30,824 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:32:35,825 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:32:40,825 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:32:42,553 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:32:42,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:32:42,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:32:42,597 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:32:45,840 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:32:50,840 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:32:55,841 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:32:57,553 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:32:57,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:32:57,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:32:57,597 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:32:57,813 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:33:01,814 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:33:02,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 06:33:02,665 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 06:33:02,666 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 06:33:02,668 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 06:33:02,909 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 06:33:04,910 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 06:33:07,705 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:33:12,553 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:33:12,553 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:33:12,554 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:33:12,555 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:33:12,799 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:33:17,800 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:33:22,800 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:33:27,553 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:33:27,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:33:27,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:33:27,597 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:33:27,804 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:33:27,805 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:33:32,806 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:33:37,806 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:33:42,553 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:33:42,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:33:42,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:33:42,597 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:33:42,827 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:33:47,828 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:33:52,828 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:33:57,553 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:33:57,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:33:57,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:33:57,596 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:33:57,826 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:33:58,827 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:34:03,828 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:34:08,828 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:34:12,553 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:34:12,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:34:12,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:34:12,596 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:34:14,826 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:34:19,826 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:34:24,827 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:34:27,553 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:34:27,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:34:27,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:34:27,596 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:34:27,862 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:34:29,863 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:34:34,864 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:34:39,864 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:34:42,554 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:34:42,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:34:42,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:34:42,597 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:34:44,871 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:34:49,871 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:34:54,872 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:34:57,554 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:34:57,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:34:57,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:34:57,596 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:34:57,823 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:35:00,823 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:35:05,824 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:35:10,825 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:35:12,554 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:35:12,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:35:12,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:35:12,596 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:35:16,798 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:35:21,799 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:35:26,799 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:35:27,554 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:35:27,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:35:27,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:35:27,596 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:35:27,850 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:35:31,851 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:35:36,852 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:35:41,853 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:35:42,554 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:35:42,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:35:42,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:35:42,596 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:35:47,813 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:35:52,814 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:35:57,554 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:35:57,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:35:57,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:35:57,596 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:35:57,809 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:35:58,809 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:36:03,810 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:36:08,811 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:36:12,554 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:36:12,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:36:12,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:36:12,597 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:36:13,838 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:36:18,839 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:36:23,839 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:36:27,554 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:36:27,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:36:27,596 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:36:27,596 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:36:27,837 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:36:29,838 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:36:34,839 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:36:39,839 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:36:42,557 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:36:42,600 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:36:42,600 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:36:42,601 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:36:44,866 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:36:49,866 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:36:54,867 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:36:57,557 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:36:57,600 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:36:57,600 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:36:57,601 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:36:57,863 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:37:00,864 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:37:05,865 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:37:10,865 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:37:12,557 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:37:12,557 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:37:12,600 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:37:12,600 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:37:16,844 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:37:21,844 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:37:26,845 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:37:27,557 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:37:27,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:37:27,558 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:37:27,812 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:37:32,812 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:37:37,813 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:37:42,557 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:37:42,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:37:42,558 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:37:43,787 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:37:48,788 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:37:53,788 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:37:57,557 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:37:57,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:37:57,558 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:37:57,813 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:37:58,814 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:38:03,814 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:38:06,884 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 06:38:06,886 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 06:38:06,887 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 06:38:06,888 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 06:38:07,070 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 06:38:08,889 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:38:09,071 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 06:38:12,557 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:38:12,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:38:12,559 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:38:14,776 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:38:19,777 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:38:24,778 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:38:27,557 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:38:27,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:38:27,558 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:38:27,815 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:38:29,816 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:38:34,816 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:38:39,817 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:38:42,557 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:38:42,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:38:42,558 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:38:45,792 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:38:50,792 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:38:55,793 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:38:57,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:38:57,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:38:57,558 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:38:57,814 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:39:00,815 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:39:05,815 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:39:10,816 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:39:12,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:39:12,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:39:12,559 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:39:16,748 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:39:21,748 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:39:26,749 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:39:27,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:39:27,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:39:27,559 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:39:27,815 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:39:31,816 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:39:36,817 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:39:41,818 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:39:42,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:39:42,559 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:39:42,559 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:39:46,830 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:39:51,831 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:39:56,832 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:39:57,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:39:57,559 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:39:57,559 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:39:57,837 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:40:01,838 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:40:06,839 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:40:11,840 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:40:12,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:40:12,559 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:40:12,559 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:40:17,729 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:40:22,730 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:40:27,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:40:27,559 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:40:27,559 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:40:27,746 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:40:27,817 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:40:32,818 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:40:37,818 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:40:42,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:40:42,559 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:40:42,559 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:40:42,824 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:40:47,824 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:40:52,825 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:40:57,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:40:57,559 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:40:57,559 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:40:57,818 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:40:58,819 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:41:03,819 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:41:08,820 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:41:12,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:41:12,559 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:41:12,559 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:41:13,825 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:41:18,826 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:41:23,826 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:41:27,559 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:41:27,559 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:41:27,559 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:41:27,828 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:41:28,828 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:41:33,829 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:41:38,829 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:41:42,559 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:41:42,559 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:41:42,559 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:41:44,753 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:41:49,754 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:41:54,755 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:41:57,559 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:41:57,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:41:57,600 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:41:57,600 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:41:57,846 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:41:59,847 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:42:04,847 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:42:09,848 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:42:12,559 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:42:12,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:42:12,560 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:42:15,772 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:42:20,772 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:42:25,773 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:42:27,559 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:42:27,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:42:27,560 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:42:27,820 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:42:30,821 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:42:35,821 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:42:40,822 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:42:42,559 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:42:42,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:42:42,560 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:42:46,772 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:42:51,773 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:42:56,773 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:42:57,559 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:42:57,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:42:57,560 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:42:57,822 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:43:01,822 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:43:06,823 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:43:10,607 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 06:43:10,609 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 06:43:10,610 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 06:43:10,611 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 06:43:11,245 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 06:43:11,245 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 06:43:12,559 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:43:12,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:43:12,560 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:43:12,833 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:43:17,834 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:43:22,834 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:43:27,559 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:43:27,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:43:27,560 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:43:27,825 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:43:28,825 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:43:33,826 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:43:38,827 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:43:42,559 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:43:42,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:43:42,561 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:43:44,779 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:43:49,780 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:43:54,780 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:43:57,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:43:57,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:43:57,560 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:43:57,823 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:43:59,823 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:44:04,824 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:44:09,825 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:44:12,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:44:12,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:44:12,561 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:44:15,740 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:44:20,741 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:44:25,742 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:44:27,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:44:27,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:44:27,561 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:44:27,824 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:44:30,825 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:44:35,826 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:44:40,827 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:44:42,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:44:42,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:44:42,561 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:44:45,831 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:44:50,832 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:44:55,832 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:44:57,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:44:57,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:44:57,561 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:44:57,824 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:45:01,826 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:45:06,827 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:45:11,827 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:45:12,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:45:12,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:45:12,561 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:45:17,792 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:45:22,792 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:45:27,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:45:27,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:45:27,561 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:45:27,801 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:45:27,825 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:45:32,826 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:45:37,827 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:45:42,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:45:42,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:45:42,561 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:45:43,760 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:45:48,760 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:45:53,761 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:45:57,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:45:57,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:45:57,561 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:45:57,826 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:45:58,827 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:46:03,828 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:46:08,828 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:46:12,560 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:46:12,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:46:12,561 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:46:14,808 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:46:19,808 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:46:24,809 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:46:27,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:46:27,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:46:27,562 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:46:27,863 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:46:29,864 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:46:34,865 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:46:39,865 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:46:42,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:46:42,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:46:42,562 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:46:45,791 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:46:50,792 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:46:55,793 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:46:57,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:46:57,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:46:57,562 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:46:57,828 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:47:00,829 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:47:05,829 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:47:10,830 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:47:12,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:47:12,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:47:12,562 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:47:16,744 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:47:21,744 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:47:26,745 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:47:27,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:47:27,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:47:27,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:47:27,562 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:47:27,829 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:47:31,830 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:47:36,831 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:47:41,831 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:47:42,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:47:42,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:47:42,562 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:47:47,805 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:47:52,806 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:47:57,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:47:57,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:47:57,562 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:47:57,829 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:47:57,830 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:48:02,830 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:48:07,831 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:48:12,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:48:12,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:48:12,562 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:48:13,787 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:48:14,647 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 06:48:14,650 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 06:48:14,650 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 06:48:14,652 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 06:48:15,410 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 06:48:16,411 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 06:48:19,653 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:48:24,653 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:48:27,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:48:27,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:48:27,562 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:48:27,830 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:48:29,831 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:48:34,831 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:48:39,832 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:48:42,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:48:42,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:48:42,562 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:48:45,752 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:48:50,753 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:48:55,753 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:48:57,561 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:48:57,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:48:57,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:48:57,563 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:48:57,831 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:49:00,832 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:49:05,833 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:49:10,833 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:49:12,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:49:12,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:49:12,563 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:49:16,816 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:49:21,817 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:49:26,818 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:49:27,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:49:27,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:49:27,563 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:49:27,832 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:49:31,833 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:49:36,833 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:49:41,834 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:49:42,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:49:42,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:49:42,563 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:49:47,758 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:49:52,759 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:49:57,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:49:57,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:49:57,563 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:49:57,840 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:49:57,840 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:50:02,840 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:50:07,841 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:50:12,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:50:12,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:50:12,563 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:50:13,782 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:50:18,783 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:50:23,784 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:50:27,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:50:27,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:50:27,563 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:50:27,834 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:50:28,834 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:50:33,835 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:50:38,836 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:50:42,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:50:42,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:50:42,563 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:50:44,756 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:50:49,756 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:50:54,757 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:50:57,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:50:57,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:50:57,563 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:50:57,835 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:50:59,836 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:51:04,837 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:51:09,837 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:51:12,562 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:51:12,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:51:12,563 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:51:15,778 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:51:20,779 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:51:25,779 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:51:27,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:51:27,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:51:27,604 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:51:27,604 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:51:27,838 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:51:30,838 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:51:35,839 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:51:40,840 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:51:42,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:51:42,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:51:42,564 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:51:46,797 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:51:51,797 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:51:56,798 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:51:57,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:51:57,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:51:57,604 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:51:57,604 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:51:57,852 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:52:01,853 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:52:06,854 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:52:11,855 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:52:12,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:52:12,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:52:12,564 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:52:17,787 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:52:22,787 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:52:27,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:52:27,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:52:27,564 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:52:27,791 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:52:27,837 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:52:32,838 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:52:37,839 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:52:42,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:52:42,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:52:42,564 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:52:43,752 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:52:48,753 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:52:53,754 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:52:57,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:52:57,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:52:57,564 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:52:57,838 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:52:58,839 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:53:03,840 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:53:08,840 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:53:12,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:53:12,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:53:12,564 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:53:14,794 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:53:18,239 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 06:53:18,242 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 06:53:18,242 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 06:53:18,244 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 06:53:18,580 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 06:53:20,281 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:53:20,581 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 06:53:25,282 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:53:27,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:53:27,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:53:27,564 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:53:27,839 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:53:30,839 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:53:35,840 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:53:40,840 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:53:42,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:53:42,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:53:42,564 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:53:46,834 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:53:51,835 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:53:56,835 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:53:57,563 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:53:57,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:53:57,564 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:53:57,840 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:54:01,840 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:54:06,841 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:54:11,841 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:54:12,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:54:12,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:54:12,564 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:54:17,817 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:54:22,818 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:54:27,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:54:27,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:54:27,565 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:54:27,845 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:54:27,845 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:54:32,846 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:54:37,846 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:54:42,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:54:42,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:54:42,565 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:54:43,745 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:54:48,746 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:54:53,746 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:54:57,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:54:57,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:54:57,565 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:54:57,841 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:54:58,842 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:55:03,842 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:55:08,843 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:55:12,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:55:12,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:55:12,565 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:55:14,834 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:55:19,834 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:55:24,835 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:55:27,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:55:27,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:55:27,565 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:55:27,843 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:55:29,844 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:55:34,844 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:55:39,845 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:55:42,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:55:42,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:55:42,565 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:55:45,844 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:55:50,844 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:55:55,845 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:55:57,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:55:57,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:55:57,565 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:55:57,844 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:56:01,844 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:56:06,845 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:56:11,846 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:56:12,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:56:12,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:56:12,565 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:56:17,794 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:56:22,795 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:56:27,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:56:27,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:56:27,565 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:56:27,844 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:56:27,845 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:56:32,845 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:56:37,846 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:56:42,564 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:56:42,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:56:42,565 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:56:43,843 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:56:48,844 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:56:53,844 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:56:57,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:56:57,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:56:57,565 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:56:57,858 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:56:58,859 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:57:03,859 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:57:08,860 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:57:12,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:57:12,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:57:12,566 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:57:14,784 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:57:19,784 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:57:24,785 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:57:27,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:57:27,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:57:27,567 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:57:27,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:57:27,846 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:57:29,846 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:57:34,847 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:57:39,847 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:57:42,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:57:42,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:57:42,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:57:42,609 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:57:45,804 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:57:50,805 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:57:55,806 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:57:57,565 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:57:57,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:57:57,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:57:57,609 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:57:57,866 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:58:00,867 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:58:05,867 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:58:10,868 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:58:12,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:58:12,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:58:12,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:58:12,609 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:58:16,830 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:58:21,831 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:58:21,964 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 06:58:21,965 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 06:58:21,965 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 06:58:21,967 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 06:58:22,749 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 06:58:24,750 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 06:58:27,005 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:58:27,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:58:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:58:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:58:27,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:58:27,847 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:58:32,849 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:58:37,849 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:58:42,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:58:42,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:58:42,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:58:42,609 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:58:43,833 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:58:48,833 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:58:53,834 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:58:57,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:58:57,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:58:57,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:58:57,609 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:58:57,861 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:58:58,862 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:59:03,862 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:59:08,863 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:59:12,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:59:12,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:59:12,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:59:12,609 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:59:14,796 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:59:19,796 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:59:24,797 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:59:27,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:59:27,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:59:27,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:59:27,609 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:59:27,863 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 06:59:29,863 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:59:34,864 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:59:39,864 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:59:42,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:59:42,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:59:42,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:59:42,609 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:59:45,797 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:59:50,797 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:59:55,798 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 06:59:57,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 06:59:57,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 06:59:57,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 06:59:57,609 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 06:59:57,850 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:00:00,851 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:00:05,852 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:00:10,852 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:00:12,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:00:12,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:00:12,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:00:12,609 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:00:15,858 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:00:20,858 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:00:25,859 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:00:27,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:00:27,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:00:27,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:00:27,609 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:00:27,852 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:00:31,853 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:00:36,853 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:00:41,854 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:00:42,566 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:00:42,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:00:42,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:00:42,609 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:00:46,881 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:00:51,882 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:00:56,883 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:00:57,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:00:57,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:00:57,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:00:57,609 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:00:57,852 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:01:02,853 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:01:07,853 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:01:12,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:01:12,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:01:12,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:01:12,609 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:01:12,871 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:01:17,872 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:01:22,873 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:01:27,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:01:27,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:01:27,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:01:27,609 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:01:27,883 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:01:27,884 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:01:32,885 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:01:37,885 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:01:42,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:01:42,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:01:42,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:01:42,609 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:01:43,874 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:01:48,875 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:01:53,875 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:01:57,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:01:57,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:01:57,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:01:57,609 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:01:57,854 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:01:59,854 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:02:04,855 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:02:09,855 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:02:12,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:02:12,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:02:12,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:02:12,609 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:02:15,809 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:02:20,809 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:02:25,810 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:02:27,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:02:27,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:02:27,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:02:27,609 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:02:27,937 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:02:30,938 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:02:35,939 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:02:40,939 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:02:42,567 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:02:42,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:02:42,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:02:42,609 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:02:46,793 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:02:51,793 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:02:56,794 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:02:57,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:02:57,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:02:57,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:02:57,609 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:02:57,855 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:03:01,856 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:03:06,857 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:03:11,857 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:03:12,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:03:12,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:03:12,608 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:03:12,609 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:03:17,813 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:03:22,813 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:03:26,192 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 07:03:26,194 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 07:03:26,194 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 07:03:26,195 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 07:03:26,911 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 07:03:27,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:03:27,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:03:27,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:03:27,570 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:03:27,856 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:03:27,857 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:03:28,912 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 07:03:32,857 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:03:37,858 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:03:42,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:03:42,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:03:42,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:03:42,570 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:03:43,836 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:03:48,837 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:03:53,837 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:03:57,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:03:57,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:03:57,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:03:57,570 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:03:57,858 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:03:58,859 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:04:03,859 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:04:08,860 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:04:12,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:04:12,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:04:12,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:04:12,570 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:04:14,749 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:04:19,750 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:04:24,750 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:04:27,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:04:27,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:04:27,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:04:27,570 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:04:27,858 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:04:29,858 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:04:34,859 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:04:39,860 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:04:42,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:04:42,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:04:42,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:04:42,570 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:04:45,806 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:04:50,807 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:04:55,807 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:04:57,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:04:57,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:04:57,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:04:57,570 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:04:57,859 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:05:00,860 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:05:05,861 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:05:10,861 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:05:12,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:05:12,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:05:12,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:05:12,570 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:05:16,817 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:05:21,818 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:05:26,819 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:05:27,568 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:05:27,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:05:27,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:05:27,570 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:05:27,862 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:05:31,863 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:05:36,863 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:05:41,864 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:05:42,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:05:42,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:05:42,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:05:42,570 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:05:47,831 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:05:52,832 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:05:57,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:05:57,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:05:57,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:05:57,571 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:05:57,844 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:05:57,860 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:06:02,861 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:06:07,862 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:06:12,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:06:12,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:06:12,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:06:12,570 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:06:13,813 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:06:18,813 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:06:23,814 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:06:27,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:06:27,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:06:27,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:06:27,571 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:06:27,862 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:06:28,863 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:06:33,863 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:06:38,864 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:06:42,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:06:42,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:06:42,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:06:42,571 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:06:44,803 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:06:49,804 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:06:54,805 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:06:57,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:06:57,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:06:57,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:06:57,571 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:06:57,862 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:06:59,863 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:07:04,864 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:07:09,864 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:07:12,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:07:12,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:07:12,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:07:12,571 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:07:15,760 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:07:20,760 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:07:25,761 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:07:27,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:07:27,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:07:27,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:07:27,571 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:07:27,865 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:07:30,865 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:07:35,866 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:07:40,866 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:07:42,569 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:07:42,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:07:42,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:07:42,571 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:07:46,759 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:07:51,759 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:07:56,760 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:07:57,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:07:57,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:07:57,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:07:57,571 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:07:57,865 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:08:01,866 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:08:06,867 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:08:11,867 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:08:12,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:08:12,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:08:12,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:08:12,571 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:08:17,788 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:08:22,788 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:08:27,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:08:27,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:08:27,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:08:27,571 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:08:27,866 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:08:27,867 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:08:30,395 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 07:08:30,397 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 07:08:30,398 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 07:08:30,399 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 07:08:31,074 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 07:08:31,074 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 07:08:33,400 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:08:38,401 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:08:42,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:08:42,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:08:42,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:08:42,571 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:08:43,834 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:08:48,834 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:08:53,835 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:08:57,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:08:57,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:08:57,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:08:57,571 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:08:57,867 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:08:58,867 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:09:03,868 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:09:08,869 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:09:12,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:09:12,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:09:12,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:09:12,572 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:09:14,753 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:09:19,754 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:09:24,754 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:09:27,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:09:27,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:09:27,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:09:27,572 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:09:27,868 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:09:29,869 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:09:34,869 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:09:39,870 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:09:42,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:09:42,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:09:42,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:09:42,572 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:09:45,771 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:09:50,772 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:09:55,772 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:09:57,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:09:57,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:09:57,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:09:57,572 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:09:57,869 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:10:00,869 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:10:05,870 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:10:10,871 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:10:12,570 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:10:12,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:10:12,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:10:12,573 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:10:16,755 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:10:21,756 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:10:26,756 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:10:27,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:10:27,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:10:27,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:10:27,573 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:10:27,870 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:10:31,871 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:10:36,871 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:10:41,872 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:10:42,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:10:42,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:10:42,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:10:42,573 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:10:47,840 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:10:52,841 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:10:57,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:10:57,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:10:57,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:10:57,573 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:10:57,870 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:10:57,871 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:11:02,871 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:11:07,872 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:11:12,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:11:12,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:11:12,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:11:12,573 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:11:13,795 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:11:18,795 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:11:23,796 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:11:27,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:11:27,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:11:27,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:11:27,573 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:11:27,871 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:11:28,872 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:11:33,872 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:11:38,873 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:11:42,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:11:42,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:11:42,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:11:42,573 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:11:44,796 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:11:49,797 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:11:54,798 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:11:57,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:11:57,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:11:57,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:11:57,573 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:11:57,874 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:11:59,874 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:12:04,875 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:12:09,876 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:12:12,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:12:12,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:12:12,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:12:12,574 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:12:15,766 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:12:20,767 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:12:25,767 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:12:27,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:12:27,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:12:27,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:12:27,574 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:12:27,874 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:12:30,875 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:12:35,875 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:12:40,876 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:12:42,571 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:12:42,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:12:42,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:12:42,574 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:12:46,831 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:12:51,832 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:12:56,833 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:12:57,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:12:57,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:12:57,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:12:57,574 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:12:57,875 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:13:01,875 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:13:06,876 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:13:11,877 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:13:12,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:13:12,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:13:12,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:13:12,574 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:13:17,028 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:13:22,029 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:13:27,030 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:13:27,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:13:27,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:13:27,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:13:27,574 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:13:27,875 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:13:32,876 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:13:34,479 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 07:13:34,481 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 07:13:34,481 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 07:13:34,483 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 07:13:35,240 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 07:13:35,240 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 07:13:38,484 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:13:42,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:13:42,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:13:42,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:13:42,574 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:13:43,770 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:13:48,771 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:13:53,772 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:13:57,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:13:57,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:13:57,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:13:57,574 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:13:57,876 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:13:58,877 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:14:03,877 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:14:08,878 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:14:12,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:14:12,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:14:12,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:14:12,574 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:14:14,755 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:14:19,756 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:14:24,756 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:14:27,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:14:27,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:14:27,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:14:27,574 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:14:27,877 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:14:29,878 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:14:34,878 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:14:39,879 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:14:42,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:14:42,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:14:42,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:14:42,574 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:14:45,804 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:14:50,804 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:14:55,805 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:14:57,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:14:57,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:14:57,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:14:57,574 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:14:57,878 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:15:00,879 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:15:05,879 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:15:10,880 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:15:12,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:15:12,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:15:12,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:15:12,575 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:15:16,778 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:15:21,778 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:15:26,779 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:15:27,572 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:15:27,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:15:27,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:15:27,575 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:15:27,880 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:15:31,881 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:15:36,882 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:15:41,882 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:15:42,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:15:42,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:15:42,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:15:42,575 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:15:47,780 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:15:52,781 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:15:57,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:15:57,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:15:57,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:15:57,575 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:15:57,852 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:15:57,880 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:16:02,881 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:16:07,881 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:16:12,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:16:12,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:16:12,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:16:12,575 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:16:13,811 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:16:18,812 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:16:23,812 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:16:27,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:16:27,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:16:27,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:16:27,575 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:16:27,882 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:16:28,883 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:16:33,883 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:16:38,884 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:16:42,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:16:42,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:16:42,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:16:42,575 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:16:44,851 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:16:49,851 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:16:54,852 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:16:57,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:16:57,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:16:57,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:16:57,575 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:16:57,881 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:16:59,882 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:17:04,882 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:17:09,883 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:17:12,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:17:12,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:17:12,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:17:12,575 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:17:15,837 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:17:20,838 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:17:25,839 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:17:27,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:17:27,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:17:27,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:17:27,575 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:17:27,882 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:17:30,883 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:17:35,883 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:17:40,884 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:17:42,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:17:42,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:17:42,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:17:42,575 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:17:46,758 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:17:51,759 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:17:56,760 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:17:57,573 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:17:57,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:17:57,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:17:57,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:17:57,883 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:18:01,883 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:18:06,884 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:18:11,885 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:18:12,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:18:12,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:18:12,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:18:12,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:18:17,769 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:18:22,769 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:18:27,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:18:27,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:18:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:18:27,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:18:27,793 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:18:27,884 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:18:32,885 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:18:37,885 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:18:38,558 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 07:18:38,560 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 07:18:38,561 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 07:18:38,562 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 07:18:39,411 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 07:18:39,411 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 07:18:42,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:18:42,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:18:42,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:18:42,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:18:43,811 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:18:48,811 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:18:53,812 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:18:57,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:18:57,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:18:57,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:18:57,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:18:57,884 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:18:58,885 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:19:03,886 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:19:08,886 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:19:12,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:19:12,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:19:12,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:19:12,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:19:14,839 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:19:19,840 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:19:24,840 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:19:27,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:19:27,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:19:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:19:27,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:19:27,885 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:19:29,886 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:19:34,887 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:19:39,887 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:19:42,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:19:42,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:19:42,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:19:42,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:19:45,851 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:19:50,851 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:19:55,852 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:19:57,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:19:57,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:19:57,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:19:57,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:19:57,887 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:20:00,887 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:20:05,888 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:20:10,889 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:20:12,574 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:20:12,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:20:12,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:20:12,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:20:16,851 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:20:21,851 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:20:26,852 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:20:27,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:20:27,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:20:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:20:27,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:20:27,887 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:20:31,888 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:20:36,888 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:20:41,889 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:20:42,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:20:42,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:20:42,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:20:42,576 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:20:47,760 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:20:52,760 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:20:57,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:20:57,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:20:57,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:20:57,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:20:57,780 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:20:57,888 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:21:02,889 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:21:07,889 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:21:12,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:21:12,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:21:12,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:21:12,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:21:13,848 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:21:18,849 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:21:23,850 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:21:27,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:21:27,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:21:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:21:27,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:21:27,889 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:21:28,889 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:21:33,890 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:21:38,891 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:21:42,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:21:42,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:21:42,577 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:21:42,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:21:44,760 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:21:49,761 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:21:54,762 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:21:57,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:21:57,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:21:57,577 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:21:57,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:21:57,889 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:21:59,890 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:22:04,891 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:22:09,892 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:22:12,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:22:12,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:22:12,577 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:22:12,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:22:15,849 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:22:20,850 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:22:25,850 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:22:27,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:22:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:22:27,577 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:22:27,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:22:27,890 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:22:30,891 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:22:35,892 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:22:40,892 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:22:42,575 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:22:42,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:22:42,577 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:22:42,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:22:46,843 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:22:51,844 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:22:56,845 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:22:57,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:22:57,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:22:57,577 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:22:57,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:22:57,891 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:23:01,892 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:23:06,893 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:23:11,893 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:23:12,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:23:12,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:23:12,577 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:23:12,577 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:23:17,852 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:23:22,852 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:23:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:23:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:23:27,577 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:23:27,578 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:23:27,892 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:23:27,893 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:23:32,893 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:23:37,894 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:23:42,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:23:42,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:23:42,577 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:23:42,578 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:23:42,940 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 07:23:42,942 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 07:23:42,943 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 07:23:42,943 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:23:42,944 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 07:23:43,582 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 07:23:44,583 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 07:23:47,945 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:23:52,946 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:23:57,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:23:57,577 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:23:57,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:23:57,578 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:23:57,893 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:23:58,893 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:24:03,894 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:24:08,895 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:24:12,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:24:12,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:24:12,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:24:12,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:24:14,845 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:24:19,845 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:24:24,846 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:24:27,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:24:27,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:24:27,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:24:27,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:24:27,894 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:24:29,894 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:24:34,895 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:24:39,896 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:24:42,576 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:24:42,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:24:42,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:24:42,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:24:44,904 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:24:49,904 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:24:54,905 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:24:57,577 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:24:57,624 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:24:57,624 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:24:57,624 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:24:57,930 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:24:59,931 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:25:04,932 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:25:09,932 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:25:12,577 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:25:12,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:25:12,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:25:12,620 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:25:15,830 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:25:20,830 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:25:25,831 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:25:27,577 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:25:27,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:25:27,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:25:27,620 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:25:27,895 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:25:30,896 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:25:35,897 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:25:40,897 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:25:42,577 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:25:42,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:25:42,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:25:42,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:25:46,822 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:25:51,823 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:25:56,823 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:25:57,577 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:25:57,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:25:57,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:25:57,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:25:57,897 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:26:01,898 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:26:06,898 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:26:11,899 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:26:12,577 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:26:12,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:26:12,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:26:12,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:26:17,806 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:26:22,806 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:26:27,577 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:26:27,624 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:26:27,624 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:26:27,625 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:26:27,869 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:26:27,897 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:26:32,898 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:26:37,899 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:26:42,577 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:26:42,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:26:42,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:26:42,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:26:43,876 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:26:48,877 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:26:53,878 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:26:57,577 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:26:57,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:26:57,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:26:57,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:26:57,899 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:26:58,900 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:27:03,900 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:27:08,901 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:27:12,577 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:27:12,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:27:12,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:27:12,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:27:14,817 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:27:19,818 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:27:24,818 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:27:27,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:27:27,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:27:27,621 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:27:27,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:27:27,899 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:27:29,899 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:27:34,900 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:27:39,900 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:27:42,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:27:42,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:27:42,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:27:42,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:27:45,833 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:27:50,833 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:27:55,833 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:27:57,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:27:57,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:27:57,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:27:57,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:27:57,900 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:28:00,901 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:28:05,902 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:28:10,902 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:28:12,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:28:12,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:28:12,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:28:12,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:28:16,866 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:28:21,866 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:28:26,867 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:28:27,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:28:27,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:28:27,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:28:27,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:28:27,901 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:28:31,901 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:28:36,902 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:28:41,902 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:28:42,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:28:42,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:28:42,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:28:42,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:28:46,888 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 07:28:46,889 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 07:28:46,889 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 07:28:46,891 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 07:28:46,929 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:28:47,750 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 07:28:48,751 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 07:28:51,930 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:28:56,930 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:28:57,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:28:57,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:28:57,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:28:57,580 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:28:57,901 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:29:02,902 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:29:07,903 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:29:12,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:29:12,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:29:12,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:29:12,580 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:29:13,850 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:29:18,851 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:29:23,851 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:29:27,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:29:27,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:29:27,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:29:27,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:29:27,902 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:29:28,903 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:29:33,903 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:29:38,904 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:29:42,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:29:42,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:29:42,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:29:42,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:29:44,896 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:29:49,897 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:29:54,897 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:29:57,578 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:29:57,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:29:57,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:29:57,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:29:57,903 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:29:59,904 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:30:04,904 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:30:09,905 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:30:12,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:30:12,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:30:12,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:30:12,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:30:15,825 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:30:20,826 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:30:25,827 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:30:27,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:30:27,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:30:27,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:30:27,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:30:27,904 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:30:30,905 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:30:35,905 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:30:40,906 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:30:42,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:30:42,621 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:30:42,621 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:30:42,622 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:30:46,830 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:30:51,830 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:30:56,831 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:30:57,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:30:57,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:30:57,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:30:57,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:30:57,905 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:31:01,906 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:31:06,906 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:31:11,907 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:31:12,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:31:12,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:31:12,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:31:12,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:31:17,791 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:31:22,792 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:31:27,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:31:27,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:31:27,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:31:27,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:31:27,864 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:31:27,905 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:31:32,906 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:31:37,907 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:31:42,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:31:42,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:31:42,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:31:42,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:31:43,887 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:31:48,888 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:31:53,888 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:31:57,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:31:57,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:31:57,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:31:57,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:31:57,906 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:31:58,907 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:32:03,908 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:32:08,908 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:32:12,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:32:12,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:32:12,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:32:12,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:32:14,853 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:32:19,854 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:32:24,855 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:32:27,579 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:32:27,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:32:27,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:32:27,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:32:27,907 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:32:29,908 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:32:34,908 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:32:39,909 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:32:42,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:32:42,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:32:42,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:32:42,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:32:45,890 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:32:50,890 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:32:55,891 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:32:57,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:32:57,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:32:57,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:32:57,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:32:57,908 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:33:00,909 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:33:05,909 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:33:10,910 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:33:12,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:33:12,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:33:12,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:33:12,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:33:16,847 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:33:21,847 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:33:26,848 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:33:27,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:33:27,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:33:27,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:33:27,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:33:27,909 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:33:31,910 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:33:36,910 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:33:41,911 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:33:42,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:33:42,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:33:42,620 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:33:42,621 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:33:47,882 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:33:51,400 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 07:33:51,402 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 07:33:51,402 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 07:33:51,403 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 07:33:51,919 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 07:33:52,920 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 07:33:53,441 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:33:57,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:33:57,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:33:57,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:33:57,582 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:33:57,910 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:33:58,910 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:34:03,911 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:34:08,911 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:34:12,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:34:12,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:34:12,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:34:12,582 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:34:14,750 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:34:19,751 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:34:24,751 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:34:27,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:34:27,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:34:27,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:34:27,582 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:34:27,910 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:34:29,911 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:34:34,912 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:34:39,912 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:34:42,580 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:34:42,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:34:42,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:34:42,582 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:34:45,831 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:34:50,831 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:34:55,832 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:34:57,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:34:57,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:34:57,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:34:57,582 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:34:57,911 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:35:00,912 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:35:05,913 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:35:10,913 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:35:12,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:35:12,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:35:12,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:35:12,582 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:35:16,838 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:35:21,839 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:35:26,839 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:35:27,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:35:27,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:35:27,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:35:27,582 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:35:27,912 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:35:31,913 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:35:36,914 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:35:41,915 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:35:42,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:35:42,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:35:42,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:35:42,582 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:35:47,823 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:35:52,824 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:35:57,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:35:57,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:35:57,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:35:57,583 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:35:57,841 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:35:57,913 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:36:02,914 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:36:07,914 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:36:12,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:36:12,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:36:12,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:36:12,583 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:36:13,831 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:36:18,832 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:36:23,833 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:36:27,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:36:27,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:36:27,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:36:27,583 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:36:27,914 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:36:28,914 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:36:33,915 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:36:38,916 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:36:42,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:36:42,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:36:42,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:36:42,583 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:36:44,859 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:36:49,860 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:36:54,861 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:36:57,581 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:36:57,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:36:57,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:36:57,583 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:36:57,914 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:36:59,915 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:37:04,915 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:37:09,916 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:37:12,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:37:12,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:37:12,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:37:12,583 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:37:15,831 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:37:20,831 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:37:25,832 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:37:27,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:37:27,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:37:27,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:37:27,583 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:37:27,915 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:37:30,916 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:37:35,917 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:37:40,917 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:37:42,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:37:42,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:37:42,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:37:42,583 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:37:46,848 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:37:51,849 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:37:56,849 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:37:57,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:37:57,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:37:57,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:37:57,583 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:37:57,917 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:38:01,918 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:38:06,919 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:38:11,919 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:38:12,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:38:12,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:38:12,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:38:12,584 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:38:17,840 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:38:22,841 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:38:27,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:38:27,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:38:27,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:38:27,584 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:38:27,917 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:38:27,917 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:38:32,918 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:38:37,918 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:38:42,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:38:42,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:38:42,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:38:42,584 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:38:43,036 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:38:48,036 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:38:53,037 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:38:55,259 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 07:38:55,261 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 07:38:55,262 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 07:38:55,264 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 07:38:56,082 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 07:38:57,083 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 07:38:57,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:38:57,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:38:57,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:38:57,584 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:38:57,918 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:38:58,918 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:39:03,919 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:39:08,919 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:39:12,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:39:12,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:39:12,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:39:12,584 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:39:14,867 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:39:19,868 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:39:24,868 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:39:27,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:39:27,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:39:27,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:39:27,584 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:39:27,918 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:39:29,920 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:39:34,920 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:39:39,921 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:39:42,582 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:39:42,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:39:42,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:39:42,584 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:39:45,858 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:39:50,858 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:39:55,859 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:39:57,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:39:57,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:39:57,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:39:57,584 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:39:57,919 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:40:00,920 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:40:05,920 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:40:10,921 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:40:12,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:40:12,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:40:12,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:40:12,584 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:40:16,768 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:40:21,768 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:40:26,769 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:40:27,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:40:27,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:40:27,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:40:27,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:40:27,920 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:40:31,921 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:40:36,922 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:40:41,922 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:40:42,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:40:42,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:40:42,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:40:42,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:40:47,832 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:40:52,833 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:40:57,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:40:57,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:40:57,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:40:57,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:40:57,836 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:40:57,921 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:41:02,922 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:41:07,922 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:41:12,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:41:12,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:41:12,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:41:12,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:41:13,810 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:41:18,811 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:41:23,811 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:41:27,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:41:27,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:41:27,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:41:27,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:41:27,922 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:41:28,922 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:41:33,923 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:41:38,924 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:41:42,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:41:42,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:41:42,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:41:42,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:41:44,812 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:41:49,813 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:41:54,814 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:41:57,583 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:41:57,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:41:57,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:41:57,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:41:57,923 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:41:59,923 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:42:04,924 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:42:09,924 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:42:12,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:42:12,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:42:12,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:42:12,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:42:15,840 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:42:20,840 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:42:25,841 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:42:27,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:42:27,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:42:27,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:42:27,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:42:27,923 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:42:30,924 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:42:35,925 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:42:40,925 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:42:42,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:42:42,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:42:42,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:42:42,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:42:46,774 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:42:51,775 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:42:56,776 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:42:57,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:42:57,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:42:57,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:42:57,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:42:57,924 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:43:01,925 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:43:06,926 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:43:11,926 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:43:12,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:43:12,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:43:12,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:43:12,585 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:43:17,770 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:43:22,771 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:43:27,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:43:27,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:43:27,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:43:27,586 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:43:27,860 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:43:27,925 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:43:32,926 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:43:37,926 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:43:42,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:43:42,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:43:42,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:43:42,586 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:43:43,825 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:43:48,826 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:43:53,826 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:43:57,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:43:57,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:43:57,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:43:57,586 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:43:57,926 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:43:58,926 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:43:59,190 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 07:43:59,192 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 07:43:59,193 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 07:43:59,194 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 07:43:59,243 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 07:44:01,245 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 07:44:04,195 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:44:09,196 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:44:12,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:44:12,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:44:12,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:44:12,586 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:44:14,323 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:44:19,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:44:24,324 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:44:27,584 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:44:27,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:44:27,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:44:27,586 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:44:28,146 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:44:30,146 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:44:35,147 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:44:40,147 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:44:42,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:44:42,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:44:42,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:44:42,586 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:44:45,776 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:44:50,777 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:44:55,778 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:44:57,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:44:57,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:44:57,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:44:57,586 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:44:57,928 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:45:00,928 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:45:05,929 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:45:10,929 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:45:12,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:45:12,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:45:12,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:45:12,586 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:45:16,839 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:45:21,840 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:45:26,841 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:45:27,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:45:27,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:45:27,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:45:27,586 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:45:27,929 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:45:31,930 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:45:36,931 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:45:41,931 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:45:42,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:45:42,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:45:42,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:45:42,587 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:45:47,780 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:45:52,780 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:45:57,585 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:45:57,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:45:57,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:45:57,587 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:45:57,929 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:45:57,930 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:46:02,930 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:46:07,931 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:46:12,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:46:12,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:46:12,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:46:12,587 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:46:13,831 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:46:18,831 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:46:23,832 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:46:27,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:46:27,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:46:27,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:46:27,587 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:46:27,930 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:46:28,931 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:46:33,931 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:46:38,932 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:46:42,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:46:42,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:46:42,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:46:42,587 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:46:44,833 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:46:49,833 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:46:54,834 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:46:57,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:46:57,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:46:57,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:46:57,587 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:46:57,931 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:46:59,932 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:47:04,932 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:47:09,933 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:47:12,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:47:12,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:47:12,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:47:12,587 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:47:15,858 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:47:20,859 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:47:25,859 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:47:27,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:47:27,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:47:27,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:47:27,587 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:47:27,932 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:47:30,932 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:47:35,933 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:47:40,934 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:47:42,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:47:42,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:47:42,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:47:42,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:47:46,828 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:47:51,829 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:47:56,829 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:47:57,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:47:57,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:47:57,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:47:57,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:47:57,933 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:48:01,933 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:48:06,934 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:48:11,934 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:48:12,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:48:12,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:48:12,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:48:12,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:48:17,809 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:48:22,809 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:48:27,586 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:48:27,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:48:27,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:48:27,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:48:27,821 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:48:27,934 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:48:32,934 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:48:37,935 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:48:42,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:48:42,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:48:42,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:48:43,806 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:48:48,806 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:48:53,807 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:48:57,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:48:57,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:48:57,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:48:57,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:48:57,934 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:48:58,935 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:49:03,092 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 07:49:03,095 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 07:49:03,096 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 07:49:03,097 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 07:49:03,407 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 07:49:04,098 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:49:05,408 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 07:49:09,098 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:49:12,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:49:12,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:49:12,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:49:14,781 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:49:19,782 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:49:24,783 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:49:27,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:49:27,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:49:27,588 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:49:27,935 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:49:29,936 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:49:34,936 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:49:39,937 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:49:42,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:49:42,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:49:42,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:49:42,589 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:49:45,848 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:49:50,849 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:49:55,850 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:49:57,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:49:57,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:49:57,589 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:49:57,936 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:50:00,937 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:50:05,937 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:50:10,938 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:50:12,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:50:12,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:50:12,589 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:50:16,853 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:50:21,854 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:50:26,854 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:50:27,587 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:50:27,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:50:27,589 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:50:27,937 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:50:31,938 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:50:36,938 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:50:41,939 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:50:42,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:50:42,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:50:42,589 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:50:47,867 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:50:52,867 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:50:57,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:50:57,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:50:57,589 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:50:57,938 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:50:57,939 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:51:02,939 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:51:07,939 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:51:12,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:51:12,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:51:12,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:51:12,589 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:51:13,846 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:51:18,847 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:51:23,848 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:51:27,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:51:27,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:51:27,589 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:51:27,938 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:51:28,939 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:51:33,940 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:51:38,940 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:51:42,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:51:42,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:51:42,589 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:51:44,867 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:51:49,867 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:51:54,868 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:51:57,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:51:57,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:51:57,589 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:51:57,939 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:51:59,940 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:52:04,941 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:52:09,941 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:52:12,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:52:12,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:52:12,590 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:52:15,779 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:52:20,780 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:52:25,781 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:52:27,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:52:27,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:52:27,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:52:27,590 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:52:27,940 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:52:30,941 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:52:35,941 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:52:40,942 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:52:42,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:52:42,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:52:42,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:52:42,590 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:52:46,800 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:52:51,801 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:52:56,801 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:52:57,588 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:52:57,590 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:52:57,590 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:52:57,941 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:53:01,942 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:53:06,942 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:53:11,943 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:53:12,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:53:12,590 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:53:12,590 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:53:17,854 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:53:22,854 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:53:27,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:53:27,590 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:53:27,590 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:53:27,942 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:53:27,943 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:53:32,943 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:53:37,944 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:53:42,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:53:42,590 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:53:42,590 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:53:43,859 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:53:48,860 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:53:53,860 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:53:57,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:53:57,590 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:53:57,590 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:53:57,943 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:53:58,943 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:54:03,944 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:54:07,191 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 07:54:07,194 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 07:54:07,194 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 07:54:07,195 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 07:54:07,579 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 07:54:07,579 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 07:54:09,196 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:54:12,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:54:12,590 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:54:12,590 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:54:12,590 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:54:14,841 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:54:19,841 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:54:24,842 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:54:27,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:54:27,590 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:54:27,590 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:54:27,943 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:54:29,944 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:54:34,945 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:54:39,945 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:54:42,589 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:54:42,590 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:54:42,591 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:54:42,591 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:54:45,799 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:54:50,800 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:54:55,800 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:54:57,590 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:54:57,590 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:54:57,591 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:54:57,591 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:54:57,944 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:55:00,945 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:55:05,946 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:55:10,947 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:55:12,590 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:55:12,591 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:55:12,591 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:55:16,809 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:55:21,810 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:55:26,811 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:55:27,590 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:55:27,591 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:55:27,591 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:55:27,945 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:55:31,946 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:55:36,947 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:55:41,947 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:55:42,590 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:55:42,591 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:55:42,591 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:55:47,781 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:55:52,782 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:55:57,590 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:55:57,590 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:55:57,591 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:55:57,591 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:55:57,856 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:55:57,946 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:56:02,947 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:56:07,948 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:56:12,641 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:56:12,641 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:56:12,642 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:56:12,642 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:56:13,910 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:56:18,911 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:56:23,911 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:56:27,641 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:56:27,641 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:56:27,641 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:56:27,642 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:56:27,947 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:56:28,947 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:56:33,948 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:56:38,948 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:56:42,641 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:56:42,641 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:56:42,642 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:56:42,642 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:56:44,919 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:56:49,919 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:56:54,920 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:56:57,641 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:56:57,642 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:56:57,642 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:56:57,642 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:56:57,947 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:56:59,948 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:57:04,949 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:57:09,949 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:57:12,660 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:57:12,661 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:57:12,661 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:57:15,831 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:57:20,832 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:57:25,832 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:57:27,660 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:57:27,661 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:57:27,661 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:57:27,948 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:57:30,949 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:57:35,950 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:57:40,950 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:57:42,660 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:57:42,660 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:57:42,661 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:57:46,926 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:57:51,926 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:57:56,927 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:57:57,660 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:57:57,661 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:57:57,661 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:57:57,949 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:58:01,950 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:58:06,951 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:58:11,952 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:58:12,660 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:58:12,661 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:58:12,661 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:58:17,873 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:58:22,874 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:58:27,660 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:58:27,661 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:58:27,661 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:58:27,951 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:58:27,953 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:58:32,953 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:58:37,953 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:58:42,660 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:58:42,661 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:58:42,661 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:58:43,924 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:58:48,925 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:58:53,926 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:58:57,661 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:58:57,661 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:58:57,661 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:58:57,952 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:58:58,953 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:59:03,953 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:59:08,954 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:59:11,761 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 07:59:11,764 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 07:59:11,764 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 07:59:11,765 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 07:59:12,661 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:59:12,661 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:59:12,661 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:59:12,741 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 07:59:14,742 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 07:59:14,841 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:59:19,842 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:59:24,842 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:59:27,661 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:59:27,661 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:59:27,662 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:59:27,953 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 07:59:29,954 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:59:34,954 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:59:39,955 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:59:42,661 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:59:42,661 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:59:42,662 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:59:45,852 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:59:50,852 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:59:55,853 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 07:59:57,662 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 07:59:57,704 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 07:59:57,704 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 07:59:57,704 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 07:59:57,955 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 08:00:00,955 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:00:05,956 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:00:10,957 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:00:12,661 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:00:12,662 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:00:12,662 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:00:16,854 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:00:21,855 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:00:26,855 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:00:27,662 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:00:27,662 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:00:27,662 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:00:27,955 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 08:00:31,955 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:00:36,956 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:00:41,956 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:00:42,662 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:00:42,662 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:00:42,662 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:00:47,931 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:00:52,932 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:00:57,662 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:00:57,662 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:00:57,662 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:00:57,955 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 08:00:57,956 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:01:02,957 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:01:07,958 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:01:12,662 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:01:12,662 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:01:12,663 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:01:13,850 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:01:18,850 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:01:23,851 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:01:27,662 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 08:01:27,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:01:27,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:01:27,663 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:01:27,956 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 08:01:28,957 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:01:33,958 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:01:38,958 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:01:42,662 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:01:42,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 08:01:42,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:01:42,663 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:01:44,833 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:01:49,834 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:01:54,834 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:01:57,662 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:01:57,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: keepalive +2024-08-05 08:01:57,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:01:57,663 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:01:57,957 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 08:01:59,958 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:02:04,959 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:02:09,959 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:02:12,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:02:12,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:02:12,663 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:02:15,829 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:02:20,829 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:02:25,830 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:02:27,662 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:02:27,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:02:27,663 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:02:27,958 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 08:02:30,959 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:02:35,959 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:02:40,960 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:02:42,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:02:42,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:02:42,663 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:02:46,920 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:02:51,920 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:02:56,921 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:02:57,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:02:57,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:02:57,663 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:02:57,959 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 08:03:01,960 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:03:06,960 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:03:11,961 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:03:12,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:03:12,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:03:12,663 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:03:17,875 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:03:22,876 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:03:27,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:03:27,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:03:27,664 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:03:27,960 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 08:03:27,960 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:03:32,961 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:03:37,961 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:03:42,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:03:42,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:03:42,664 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:03:43,927 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:03:48,927 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:03:53,928 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:03:57,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:03:57,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:03:57,664 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:03:57,960 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 08:03:58,961 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:04:03,962 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:04:08,962 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:04:12,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:04:12,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:04:12,664 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:04:14,843 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:04:16,287 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 08:04:16,289 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 08:04:16,289 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 08:04:16,290 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 08:04:16,899 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 08:04:18,900 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 08:04:20,329 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:04:25,330 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:04:27,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:04:27,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:04:27,664 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:04:27,961 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 08:04:30,962 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:04:35,963 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:04:40,963 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:04:42,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:04:42,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:04:42,664 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:04:46,848 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:04:51,849 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:04:56,849 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:04:57,663 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:04:57,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:04:57,664 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:04:57,962 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 08:05:01,963 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:05:06,964 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:05:11,964 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:05:12,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:05:12,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:05:12,664 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:05:17,836 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:05:22,837 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:05:27,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:05:27,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:05:27,665 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:05:27,912 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:05:27,963 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 08:05:32,964 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:05:37,965 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:05:42,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:05:42,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:05:42,665 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:05:43,858 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:05:48,858 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:05:53,859 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:05:57,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:05:57,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:05:57,665 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:05:57,964 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 08:05:58,964 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:06:03,965 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:06:08,966 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:06:12,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:06:12,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:06:12,665 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:06:14,883 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:06:19,883 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:06:24,884 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:06:27,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:06:27,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:06:27,665 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:06:27,965 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 08:06:29,965 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:06:34,966 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:06:39,967 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:06:42,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:06:42,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:06:42,665 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:06:45,861 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:06:50,862 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:06:55,862 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:06:57,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:06:57,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:06:57,665 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:06:57,966 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 08:07:00,966 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:07:05,967 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:07:10,968 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:07:12,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:07:12,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:07:12,665 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:07:16,931 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:07:21,932 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:07:26,932 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:07:27,664 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:07:27,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:07:27,665 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:07:27,966 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 08:07:31,967 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:07:36,968 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:07:41,969 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:07:42,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:07:42,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:07:42,665 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:07:47,840 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:07:52,841 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:07:57,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:07:57,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:07:57,666 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:07:57,912 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:07:57,967 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 08:08:02,968 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:08:07,969 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:08:12,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:08:12,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:08:12,666 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:08:13,915 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:08:18,916 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:08:23,917 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:08:27,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:08:27,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:08:27,666 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:08:27,968 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 08:08:28,969 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:08:33,969 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:08:38,970 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:08:42,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:08:42,666 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:08:42,666 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:08:44,851 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:08:49,852 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:08:54,852 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:08:57,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:08:57,666 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:08:57,666 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:08:57,969 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 08:08:59,970 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:09:04,970 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:09:09,971 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:09:12,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:09:12,666 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:09:12,666 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:09:15,931 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:09:20,636 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: partial_history +2024-08-05 08:09:20,639 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 08:09:20,639 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 08:09:20,640 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 08:09:21,062 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 08:09:21,641 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:09:23,063 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 08:09:26,642 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:09:27,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:09:27,666 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:09:27,666 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:09:27,970 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 08:09:31,971 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:09:36,971 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:09:41,972 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:09:42,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:09:42,666 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:09:42,666 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:09:47,943 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:09:52,943 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:09:57,665 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:09:57,666 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:09:57,666 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:09:57,971 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 08:09:57,972 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:10:02,972 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:10:07,972 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:10:12,666 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:10:12,666 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:10:12,666 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:10:13,879 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:10:18,879 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:10:23,880 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:10:27,666 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:10:27,666 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:10:27,667 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:10:27,971 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 08:10:28,972 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:10:33,973 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:10:38,973 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:10:42,666 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:10:42,666 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:10:42,667 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:10:44,864 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:10:49,864 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:10:54,865 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:10:57,666 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:10:57,666 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:10:57,667 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:10:57,972 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 08:10:59,973 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:11:04,974 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:11:09,974 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:11:12,666 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:11:12,667 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:11:12,667 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:11:15,936 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:11:20,937 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:11:25,937 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:11:27,666 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:11:27,667 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:11:27,667 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:11:27,974 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 08:11:30,975 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:11:35,976 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:11:40,976 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:11:42,666 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:11:42,667 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:11:42,667 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:11:46,931 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:11:51,932 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:11:56,932 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:11:57,666 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:11:57,667 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: stop_status +2024-08-05 08:11:57,667 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: stop_status +2024-08-05 08:11:57,900 DEBUG SenderThread:12967 [sender.py:send():382] send: exit +2024-08-05 08:11:57,900 INFO SenderThread:12967 [sender.py:send_exit():589] handling exit code: 255 +2024-08-05 08:11:57,900 INFO SenderThread:12967 [sender.py:send_exit():591] handling runtime: 35371 +2024-08-05 08:11:57,901 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 08:11:57,901 INFO SenderThread:12967 [sender.py:send_exit():597] send defer +2024-08-05 08:11:57,902 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: defer +2024-08-05 08:11:57,902 INFO HandlerThread:12967 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-05 08:11:57,902 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: defer +2024-08-05 08:11:57,902 INFO SenderThread:12967 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-05 08:11:57,902 INFO SenderThread:12967 [sender.py:transition_state():617] send defer: 1 +2024-08-05 08:11:57,902 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: defer +2024-08-05 08:11:57,902 INFO HandlerThread:12967 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-05 08:11:57,902 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: defer +2024-08-05 08:11:57,902 INFO SenderThread:12967 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-05 08:11:57,902 INFO SenderThread:12967 [sender.py:transition_state():617] send defer: 2 +2024-08-05 08:11:57,902 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: defer +2024-08-05 08:11:57,902 INFO HandlerThread:12967 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-05 08:11:57,902 INFO HandlerThread:12967 [system_monitor.py:finish():203] Stopping system monitor +2024-08-05 08:11:57,903 DEBUG SystemMonitor:12967 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-05 08:11:57,903 DEBUG SystemMonitor:12967 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-05 08:11:57,903 INFO HandlerThread:12967 [interfaces.py:finish():202] Joined cpu monitor +2024-08-05 08:11:57,904 INFO HandlerThread:12967 [interfaces.py:finish():202] Joined disk monitor +2024-08-05 08:11:57,936 INFO HandlerThread:12967 [interfaces.py:finish():202] Joined gpu monitor +2024-08-05 08:11:57,936 INFO HandlerThread:12967 [interfaces.py:finish():202] Joined memory monitor +2024-08-05 08:11:57,937 INFO HandlerThread:12967 [interfaces.py:finish():202] Joined network monitor +2024-08-05 08:11:57,937 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: defer +2024-08-05 08:11:57,937 INFO SenderThread:12967 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-05 08:11:57,937 INFO SenderThread:12967 [sender.py:transition_state():617] send defer: 3 +2024-08-05 08:11:57,937 DEBUG SenderThread:12967 [sender.py:send():382] send: stats +2024-08-05 08:11:57,937 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: defer +2024-08-05 08:11:57,938 INFO HandlerThread:12967 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-05 08:11:57,939 DEBUG SenderThread:12967 [sender.py:send():382] send: history +2024-08-05 08:11:57,939 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: summary_record +2024-08-05 08:11:57,940 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 08:11:57,941 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: defer +2024-08-05 08:11:57,941 INFO SenderThread:12967 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-05 08:11:57,941 INFO SenderThread:12967 [sender.py:transition_state():617] send defer: 4 +2024-08-05 08:11:57,941 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: defer +2024-08-05 08:11:57,941 INFO HandlerThread:12967 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-05 08:11:57,941 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: defer +2024-08-05 08:11:57,941 INFO SenderThread:12967 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-05 08:11:57,941 INFO SenderThread:12967 [sender.py:transition_state():617] send defer: 5 +2024-08-05 08:11:57,941 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: defer +2024-08-05 08:11:57,941 INFO HandlerThread:12967 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-05 08:11:57,942 DEBUG SenderThread:12967 [sender.py:send():382] send: summary +2024-08-05 08:11:57,943 INFO SenderThread:12967 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-05 08:11:57,943 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: defer +2024-08-05 08:11:57,943 INFO SenderThread:12967 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-05 08:11:57,943 INFO SenderThread:12967 [sender.py:transition_state():617] send defer: 6 +2024-08-05 08:11:57,943 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: defer +2024-08-05 08:11:57,943 INFO HandlerThread:12967 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-05 08:11:57,944 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: defer +2024-08-05 08:11:57,944 INFO SenderThread:12967 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-05 08:11:57,944 INFO SenderThread:12967 [sender.py:transition_state():617] send defer: 7 +2024-08-05 08:11:57,944 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:11:57,944 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: defer +2024-08-05 08:11:57,944 INFO HandlerThread:12967 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-05 08:11:57,944 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: defer +2024-08-05 08:11:57,944 INFO SenderThread:12967 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-05 08:11:58,145 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 08:11:58,900 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-05 08:11:59,146 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 08:12:00,416 INFO SenderThread:12967 [sender.py:transition_state():617] send defer: 8 +2024-08-05 08:12:00,416 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: poll_exit +2024-08-05 08:12:00,416 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: defer +2024-08-05 08:12:00,417 INFO HandlerThread:12967 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-05 08:12:00,417 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: defer +2024-08-05 08:12:00,417 INFO SenderThread:12967 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-05 08:12:00,417 INFO SenderThread:12967 [job_builder.py:build():296] Attempting to build job artifact +2024-08-05 08:12:00,435 INFO SenderThread:12967 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-05 08:12:00,487 INFO SenderThread:12967 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-05 08:12:00,511 INFO SenderThread:12967 [sender.py:transition_state():617] send defer: 9 +2024-08-05 08:12:00,511 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: defer +2024-08-05 08:12:00,511 DEBUG SenderThread:12967 [sender.py:send():382] send: artifact +2024-08-05 08:12:00,512 INFO HandlerThread:12967 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-05 08:12:00,900 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-05 08:12:01,147 INFO Thread-12 :12967 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 08:12:01,810 INFO wandb-upload_0:12967 [upload_job.py:push():86] Skipped uploading /singularity_home/.local/share/wandb/artifacts/staging/tmp80fw5a_h +2024-08-05 08:12:02,201 INFO wandb-upload_1:12967 [upload_job.py:push():89] Uploaded file /singularity_home/.local/share/wandb/artifacts/staging/tmp7w7kheqa +2024-08-05 08:12:03,480 INFO SenderThread:12967 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTEwMTE3MzM3Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjM2MjY3MjMzNA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTA5ODUzNDkwNw==', 'versionIndex': 4}}} +2024-08-05 08:12:03,480 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: defer +2024-08-05 08:12:03,480 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: status_report +2024-08-05 08:12:03,481 INFO SenderThread:12967 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-05 08:12:03,481 INFO SenderThread:12967 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-05 08:12:04,147 INFO SenderThread:12967 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240804_222226-kh5katc1/files +2024-08-05 08:12:04,148 INFO SenderThread:12967 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_222226-kh5katc1/files/requirements.txt requirements.txt +2024-08-05 08:12:04,148 INFO SenderThread:12967 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_222226-kh5katc1/files/config.yaml config.yaml +2024-08-05 08:12:04,149 INFO SenderThread:12967 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-metadata.json wandb-metadata.json +2024-08-05 08:12:04,150 INFO SenderThread:12967 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json wandb-summary.json +2024-08-05 08:12:04,151 INFO SenderThread:12967 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_222226-kh5katc1/files/output.log output.log +2024-08-05 08:12:04,152 INFO SenderThread:12967 [sender.py:transition_state():617] send defer: 10 +2024-08-05 08:12:04,152 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: poll_exit +2024-08-05 08:12:04,153 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: defer +2024-08-05 08:12:04,153 INFO HandlerThread:12967 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-05 08:12:04,154 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: defer +2024-08-05 08:12:04,154 INFO SenderThread:12967 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-05 08:12:04,154 INFO SenderThread:12967 [file_pusher.py:finish():172] shutting down file pusher +2024-08-05 08:12:04,606 INFO wandb-upload_0:12967 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_222226-kh5katc1/files/requirements.txt +2024-08-05 08:12:04,765 INFO wandb-upload_2:12967 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_222226-kh5katc1/files/wandb-summary.json +2024-08-05 08:12:04,843 INFO wandb-upload_1:12967 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_222226-kh5katc1/files/config.yaml +2024-08-05 08:12:04,897 INFO wandb-upload_3:12967 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_222226-kh5katc1/files/output.log +2024-08-05 08:12:04,901 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-05 08:12:04,902 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: poll_exit +2024-08-05 08:12:05,098 INFO Thread-11 (_thread_body):12967 [sender.py:transition_state():617] send defer: 11 +2024-08-05 08:12:05,098 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: defer +2024-08-05 08:12:05,098 INFO HandlerThread:12967 [handler.py:handle_request_defer():172] handle defer: 11 +2024-08-05 08:12:05,098 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: defer +2024-08-05 08:12:05,098 INFO SenderThread:12967 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-08-05 08:12:05,098 INFO SenderThread:12967 [file_pusher.py:join():178] waiting for file pusher +2024-08-05 08:12:05,098 INFO SenderThread:12967 [sender.py:transition_state():617] send defer: 12 +2024-08-05 08:12:05,099 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: defer +2024-08-05 08:12:05,099 INFO HandlerThread:12967 [handler.py:handle_request_defer():172] handle defer: 12 +2024-08-05 08:12:05,099 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: defer +2024-08-05 08:12:05,099 INFO SenderThread:12967 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-08-05 08:12:05,099 INFO SenderThread:12967 [file_stream.py:finish():595] file stream finish called +2024-08-05 08:12:05,306 INFO SenderThread:12967 [file_stream.py:finish():599] file stream finish is done +2024-08-05 08:12:05,306 INFO SenderThread:12967 [sender.py:transition_state():617] send defer: 13 +2024-08-05 08:12:05,306 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: defer +2024-08-05 08:12:05,307 INFO HandlerThread:12967 [handler.py:handle_request_defer():172] handle defer: 13 +2024-08-05 08:12:05,307 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: defer +2024-08-05 08:12:05,307 INFO SenderThread:12967 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-08-05 08:12:05,307 INFO SenderThread:12967 [sender.py:transition_state():617] send defer: 14 +2024-08-05 08:12:05,307 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: defer +2024-08-05 08:12:05,307 DEBUG SenderThread:12967 [sender.py:send():382] send: final +2024-08-05 08:12:05,307 INFO HandlerThread:12967 [handler.py:handle_request_defer():172] handle defer: 14 +2024-08-05 08:12:05,307 DEBUG SenderThread:12967 [sender.py:send():382] send: footer +2024-08-05 08:12:05,307 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: defer +2024-08-05 08:12:05,308 INFO SenderThread:12967 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-08-05 08:12:05,308 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-05 08:12:05,308 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-05 08:12:05,308 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: poll_exit +2024-08-05 08:12:05,308 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: server_info +2024-08-05 08:12:05,309 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: poll_exit +2024-08-05 08:12:05,309 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: get_summary +2024-08-05 08:12:05,309 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: server_info +2024-08-05 08:12:05,310 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: sampled_history +2024-08-05 08:12:05,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-05 08:12:05,316 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: job_info +2024-08-05 08:12:05,478 DEBUG SenderThread:12967 [sender.py:send_request():409] send_request: job_info +2024-08-05 08:12:05,478 INFO MainThread:12967 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-08-05 08:12:05,480 INFO MainThread:12967 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-08-05 08:12:05,480 INFO MainThread:12967 [wandb_run.py:_footer_sync_info():3825] logging synced files +2024-08-05 08:12:05,481 DEBUG HandlerThread:12967 [handler.py:handle_request():146] handle_request: shutdown +2024-08-05 08:12:05,481 INFO HandlerThread:12967 [handler.py:finish():869] shutting down handler +2024-08-05 08:12:06,317 INFO WriterThread:12967 [datastore.py:close():296] close: /project/wandb/run-20240804_222226-kh5katc1/run-kh5katc1.wandb +2024-08-05 08:12:06,478 INFO SenderThread:12967 [sender.py:finish():1572] shutting down sender +2024-08-05 08:12:06,479 INFO SenderThread:12967 [file_pusher.py:finish():172] shutting down file pusher +2024-08-05 08:12:06,479 INFO SenderThread:12967 [file_pusher.py:join():178] waiting for file pusher diff --git a/wandb/run-20240804_222226-kh5katc1/logs/debug.log b/wandb/run-20240804_222226-kh5katc1/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ab173388f61b0dd96eddd858944d7a0fa632d114 --- /dev/null +++ b/wandb/run-20240804_222226-kh5katc1/logs/debug.log @@ -0,0 +1,30 @@ +2024-08-04 22:22:26,260 INFO MainThread:12896 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-04 22:22:26,261 INFO MainThread:12896 [wandb_setup.py:_flush():76] Configure stats pid to 12896 +2024-08-04 22:22:26,261 INFO MainThread:12896 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-04 22:22:26,261 INFO MainThread:12896 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-04 22:22:26,261 INFO MainThread:12896 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train sample'} +2024-08-04 22:22:26,261 INFO MainThread:12896 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-04 22:22:26,261 INFO MainThread:12896 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-04 22:22:26,261 INFO MainThread:12896 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240804_222226-kh5katc1/logs/debug.log +2024-08-04 22:22:26,261 INFO MainThread:12896 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240804_222226-kh5katc1/logs/debug-internal.log +2024-08-04 22:22:26,261 INFO MainThread:12896 [wandb_init.py:init():566] calling init triggers +2024-08-04 22:22:26,261 INFO MainThread:12896 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document'], 'valid_data_path': ['235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document'], 'test_data_path': ['235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 4096, 'num_workers': 2, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/google/gemma-2-2b', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'yans-sample-gemma-2-2b_train_2024-08-04-22:22:15', 'wandb_project': 'llm_tutorial', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/yans-sample-gemma-2-2b', 'save': '/work/llm_recipes/models/yans-sample-gemma-2-2b', 'base_model': '/share/pretrained_lm/google/gemma-2-2b', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 200, 'save_interval': 200, 'eval_iters': 10, 'optimizer': 'anyprecision', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 20000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 20000, 'train_samples': None, 'global_batch_size': 320, 'micro_batch_size': 1, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 4096, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/yans-sample-gemma-2-2b', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 256000, 'gradient_accumulation_steps': 320} +2024-08-04 22:22:26,261 INFO MainThread:12896 [wandb_init.py:init():616] starting backend +2024-08-04 22:22:26,261 INFO MainThread:12896 [wandb_init.py:init():620] setting up manager +2024-08-04 22:22:26,266 INFO MainThread:12896 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-04 22:22:26,266 INFO MainThread:12896 [wandb_init.py:init():628] backend started and connected +2024-08-04 22:22:26,271 INFO MainThread:12896 [wandb_init.py:init():720] updated telemetry +2024-08-04 22:22:26,282 INFO MainThread:12896 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-04 22:22:26,766 INFO MainThread:12896 [wandb_run.py:_on_init():2262] communicating current version +2024-08-04 22:22:26,847 INFO MainThread:12896 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.5 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-04 22:22:26,847 INFO MainThread:12896 [wandb_init.py:init():804] starting run threads in backend +2024-08-04 22:22:26,902 INFO MainThread:12896 [wandb_run.py:_console_start():2241] atexit reg +2024-08-04 22:22:26,902 INFO MainThread:12896 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-04 22:22:26,903 INFO MainThread:12896 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-04 22:22:26,903 INFO MainThread:12896 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-04 22:22:26,903 INFO MainThread:12896 [wandb_init.py:init():847] run started, returning control to user process +2024-08-04 22:22:32,202 INFO MainThread:12896 [wandb_run.py:_config_callback():1343] config_cb None None {'activation_function': 'gelu_pytorch_tanh', 'hidden_size': 2304, 'model_type': 'gemma2', 'max_position_embeddings': 4096, 'num_attention_heads': 8, 'num_hidden_layers': 26, 'model_architecture': 'Gemma2ForCausalLM'} +2024-08-04 22:22:32,203 INFO MainThread:12896 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1} +2024-08-05 08:12:06,481 WARNING MsgRouterThr:12896 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20240812_063447-whqmtxyq/files/config.yaml b/wandb/run-20240812_063447-whqmtxyq/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32ea435346084ac255a325b0de9b54b7506d4495 --- /dev/null +++ b/wandb/run-20240812_063447-whqmtxyq/files/config.yaml @@ -0,0 +1,335 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: FULL_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '235289369' + - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document +valid_data_path: + desc: null + value: + - '235289369' + - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document +test_data_path: + desc: null + value: + - '235289369' + - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 1021 +num_workers: + desc: null + value: 2 +tokenizer_type: + desc: null + value: HFPreTrainedTokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/google/gemma-2-2b +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: yans-sample-gemma-2-2b_train_2024-08-12-06:34:36 +wandb_project: + desc: null + value: llm_tutorial +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/yans-sample-gemma-2-2b +save: + desc: null + value: /work/llm_recipes/models/yans-sample-gemma-2-2b +base_model: + desc: null + value: /share/pretrained_lm/google/gemma-2-2b +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 3 +save_interval: + desc: null + value: 3 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: anyprecision +lr: + desc: null + value: 2.0e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 20000 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 1.0e-06 +train_iters: + desc: null + value: 20000 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 320 +micro_batch_size: + desc: null + value: 1 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 4096 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-06 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/yans-sample-gemma-2-2b +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: false +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 256000 +gradient_accumulation_steps: + desc: null + value: 320 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1723412087.358797 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +model_architecture: + desc: null + value: Gemma2ForCausalLM +activation_function: + desc: null + value: gelu_pytorch_tanh +hidden_size: + desc: null + value: 2304 +model_type: + desc: null + value: gemma2 +max_position_embeddings: + desc: null + value: 1021 +num_attention_heads: + desc: null + value: 8 +num_hidden_layers: + desc: null + value: 26 diff --git a/wandb/run-20240812_063447-whqmtxyq/files/output.log b/wandb/run-20240812_063447-whqmtxyq/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..484b06ab2010e0e18a34c7247914e49f8c33ca06 --- /dev/null +++ b/wandb/run-20240812_063447-whqmtxyq/files/output.log @@ -0,0 +1,144 @@ +Created Hugging Face repository with ID koichi12/yans-sample-gemma-2-2b. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +File not found: /work/llm_recipes/models/yans-sample-gemma-2-2b/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-sample-gemma-2-2b/latest_iteration.txt +File not found: /work/llm_recipes/models/yans-sample-gemma-2-2b/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-sample-gemma-2-2b/latest_iteration.txt +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. + +Loading checkpoint shards: 33%|███▎ | 1/3 [00:01<00:03, 1.92s/it] +File not found: /work/llm_recipes/models/yans-sample-gemma-2-2b/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-sample-gemma-2-2b/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/yans-sample-gemma-2-2b, skipping model loading +--> Model /share/pretrained_lm/google/gemma-2-2b +--> /share/pretrained_lm/google/gemma-2-2b has 2614.341888 Million params +Loading checkpoint shards: 100%|██████████| 3/3 [01:18<00:00, 26.21s/it] +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1. + warnings.warn( +Let split = None +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 6400000 + validation: 21334400 + test: 3200 +> building train, validation, and test datasets for GPT ... +> finished creating GPT datasets ... +File not found: /work/llm_recipes/models/yans-sample-gemma-2-2b/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-sample-gemma-2-2b/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/yans-sample-gemma-2-2b, skipping optimizer loading +File not found: /work/llm_recipes/models/yans-sample-gemma-2-2b/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-sample-gemma-2-2b/latest_iteration.txt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): Gemma2ForCausalLM( + (model): Gemma2Model( + (embed_tokens): Embedding(256000, 2304, padding_idx=0) + (layers): ModuleList( + (0-25): 26 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): Gemma2DecoderLayer( + (self_attn): Gemma2FlashAttention2( + (q_proj): Linear(in_features=2304, out_features=2048, bias=False) + (k_proj): Linear(in_features=2304, out_features=1024, bias=False) + (v_proj): Linear(in_features=2304, out_features=1024, bias=False) + (o_proj): Linear(in_features=2048, out_features=2304, bias=False) + (rotary_emb): Gemma2RotaryEmbedding() + ) + (mlp): Gemma2MLP( + (gate_proj): Linear(in_features=2304, out_features=9216, bias=False) + (up_proj): Linear(in_features=2304, out_features=9216, bias=False) + (down_proj): Linear(in_features=9216, out_features=2304, bias=False) + (act_fn): PytorchGELUTanh() + ) + (input_layernorm): Gemma2RMSNorm() + (post_attention_layernorm): Gemma2RMSNorm() + (pre_feedforward_layernorm): Gemma2RMSNorm() + (post_feedforward_layernorm): Gemma2RMSNorm() + ) + ) + ) + ) + (norm): Gemma2RMSNorm() + ) + (lm_head): Linear(in_features=2304, out_features=256000, bias=False) + ) +) +model config: Gemma2Config { + "_name_or_path": "/share/pretrained_lm/google/gemma-2-2b", + "architectures": [ + "Gemma2ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": 50.0, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "eos_token_id": 1, + "final_logit_softcapping": 30.0, + "head_dim": 256, + "hidden_act": "gelu_pytorch_tanh", + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 2304, + "initializer_range": 0.02, + "intermediate_size": 9216, + "label_smoothing": 0.0, + "max_position_embeddings": 1021, + "model_type": "gemma2", +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +It is strongly recommended to train Gemma2 models with the `eager` attention implementation instead of `flash_attention_2`. Use `eager` with `AutoModelForCausalLM.from_pretrained('', attn_implementation='eager')`. + "num_attention_heads": 8, + "num_hidden_layers": 26, + "num_key_value_heads": 4, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_theta": 10000.0, + "sliding_window": 4096, + "torch_dtype": "float32", + "transformers_version": "4.43.3", + "use_cache": false, + "vocab_size": 256000 +} +------------------------------------------------------------------ +iteration: 1 , TFLOPS: 52.56331460229552, Tokens per sec: 3927.6626762354495, Loss: 16.080825805664062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2 , TFLOPS: 52.356892101499724, Tokens per sec: 3912.238269345489, Loss: 15.729490280151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3 , TFLOPS: 52.39645244456057, Tokens per sec: 3915.194317381553, Loss: 15.54540729522705 +------------------------------------------------------------------ + eval ppl=4948606.5, eval loss=15.414616584777832 +Saving checkpoint to /work/llm_recipes/models/yans-sample-gemma-2-2b/iter_0000003 +Saving model state dict to /work/llm_recipes/models/yans-sample-gemma-2-2b/iter_0000003/model.pt +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +Saved model state dict to /work/llm_recipes/models/yans-sample-gemma-2-2b/iter_0000003/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-sample-gemma-2-2b/iter_0000003/optimizer.pt +[rank0]:[2024-08-12 06:40:35,335] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.008401250000133587, 'preprocessing_with_comm': 0.0009138020004684222, 'state_converting': 5.079375774000255, : 5.090390497000044}) +Saved optimizer state dict to /work/llm_recipes/models/yans-sample-gemma-2-2b/iter_0000003/optimizer.pt +Traceback (most recent call last): + File "/project/examples/finetuning.py", line 13, in + main() + File "/project/src/llama_recipes/finetuning.py", line 281, in main + train( + File "/project/src/llama_recipes/utils/train_utils.py", line 175, in train + save_checkpoint( + File "/project/src/llama_recipes/utils/checkpoint.py", line 168, in save_checkpoint + tokenizer.tokenizer.save_pretrained(tokenizer_path) + File "/project/lib/transformers/src/transformers/tokenization_utils_base.py", line 2622, in save_pretrained + if os.path.isfile(save_directory): + File "/usr/lib/python3.10/genericpath.py", line 30, in isfile + st = os.stat(path) +TypeError: stat: path should be string, bytes, os.PathLike or integer, not NoneType +Saving scheduler state dict to /work/llm_recipes/models/yans-sample-gemma-2-2b/iter_0000003/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-sample-gemma-2-2b/iter_0000003/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-sample-gemma-2-2b/iter_0000003/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-sample-gemma-2-2b/iter_0000003/rng.pt \ No newline at end of file diff --git a/wandb/run-20240812_063447-whqmtxyq/files/requirements.txt b/wandb/run-20240812_063447-whqmtxyq/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7c90fd1c54d0a9881f6b4c6465b2a4fa88c9056c --- /dev/null +++ b/wandb/run-20240812_063447-whqmtxyq/files/requirements.txt @@ -0,0 +1,271 @@ +absl-py==2.1.0 +accelerate==0.33.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bleach==6.1.0 +blis==0.7.11 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +distributed==2023.11.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +einops==0.7.0 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +grpcio==1.60.1 +huggingface-hub==0.24.5 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +jedi==0.19.1 +jinja2==3.1.3 +joblib==1.3.2 +json5==0.9.14 +jsonnet==0.19.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +kiwisolver==1.4.5 +langcodes==3.3.0 +lazy-loader==0.3 +librosa==0.10.1 +llvmlite==0.40.1 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +murmurhash==1.0.10 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +onnx==1.15.0rc2 +opencv==4.7.0 +optree==0.10.0 +packaging==23.2 +pandas==1.5.3 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +peft==0.11.1 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==14.0.1.dev0+gba5374836.d20240125 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pynvml==11.4.1 +pyparsing==3.1.1 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests==2.31.0 +rich==13.7.0 +rmm==23.12.0 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.0 +safetensors==0.4.3 +scikit-learn==1.2.0 +scipy==1.12.0 +send2trash==1.8.2 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +srsly==2.4.8 +stack-data==0.6.3 +sympy==1.12 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm==4.66.1 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.9.0 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +urllib3==1.26.18 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +xdoctest==1.0.2 +xgboost==1.7.6 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 \ No newline at end of file diff --git a/wandb/run-20240812_063447-whqmtxyq/files/wandb-metadata.json b/wandb/run-20240812_063447-whqmtxyq/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..6a88aca5504d32b03282f03384c713fbfed461fa --- /dev/null +++ b/wandb/run-20240812_063447-whqmtxyq/files/wandb-metadata.json @@ -0,0 +1,215 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-11T21:34:47.942238", + "startedAt": "2024-08-11T21:34:47.345817", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "1021", + "--sliding-window-size", + "4096", + "--micro-batch-size", + "1", + "--global-batch-size", + "320", + "--train-iters", + "20000", + "--tokenizer-type", + "HFPreTrainedTokenizer", + "--tokenizer-model", + "/share/pretrained_lm/google/gemma-2-2b", + "--train-data-path", + "235289369", + "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document", + "--valid-data-path", + "235289369", + "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document", + "--test-data-path", + "235289369", + "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document", + "--lr", + "2e-5", + "--min-lr", + "1e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "20000", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "anyprecision", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-6", + "--save-interval", + "3", + "--eval-interval", + "3", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/google/gemma-2-2b", + "--save", + "/work/llm_recipes/models/yans-sample-gemma-2-2b", + "--load", + "/work/llm_recipes/models/yans-sample-gemma-2-2b", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "FULL_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/yans-sample-gemma-2-2b", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "llm_tutorial", + "--wandb-name", + "yans-sample-gemma-2-2b_train_2024-08-12-06:34:36" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "6da01327e78c302bc0cfdb335f3ca297e2a19c8c" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.0429999999997, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.487823486328125 + } +} diff --git a/wandb/run-20240812_063447-whqmtxyq/files/wandb-summary.json b/wandb/run-20240812_063447-whqmtxyq/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..183879e9c03dab604407e7f205ae13c3c8c0075f --- /dev/null +++ b/wandb/run-20240812_063447-whqmtxyq/files/wandb-summary.json @@ -0,0 +1 @@ +{"training/loss": 15.54540729522705, "training/perplexity": 5640071.469138662, "utils/batch_size": 1, "utils/global_batch_size": 320, "utils/seq_len": 1022, "utils/gradient_accumulation_steps": 320, "utils/iteration": 3, "optimizer/lr": 1.114e-06, "optimizer/variance_l2": 0.0003583679885385243, "optimizer/variance_sqrt_l2": 0.3777214531330342, "optimizer/momentum_l2": 0.26258589724268894, "optimizer/weight_l2": 1167.8420269882395, "optimizer/variance_l1": 0.14256858825683594, "optimizer/variance_sqrt_l1": 5085.8125, "optimizer/momentum_l1": 3147.65625, "optimizer/weight_l1": 29773824.0, "optimizer/variance_abs_max": 7.009506225585938e-05, "optimizer/variance_sqrt_abs_max": 0.00836181640625, "optimizer/momentum_abs_max": 0.005950927734375, "optimizer/weight_abs_max": 12.9375, "stats/1_iteration_time": 83.53097534600056, "stats/tokens_per_sec": 3915.194317381553, "stats/tokens_per_sec_per_gpu": 3915.194317381553, "stats/tflops": 52.39645244456057, "_timestamp": 1723412421.3049276, "_runtime": 333.9461305141449, "_step": 3, "_wandb": {"runtime": 356}, "evaluation/val_loss": 15.414616584777832, "evaluation/val_ppl": 4948606.5} \ No newline at end of file diff --git a/wandb/run-20240812_063447-whqmtxyq/logs/debug-internal.log b/wandb/run-20240812_063447-whqmtxyq/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f5bf056ead17854e29d88fcb386bc53e982df63e --- /dev/null +++ b/wandb/run-20240812_063447-whqmtxyq/logs/debug-internal.log @@ -0,0 +1,359 @@ +2024-08-12 06:34:47,360 INFO StreamThr :13101 [internal.py:wandb_internal():86] W&B internal server running at pid: 13101, started at: 2024-08-12 06:34:47.359620 +2024-08-12 06:34:47,362 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status +2024-08-12 06:34:47,363 INFO WriterThread:13101 [datastore.py:open_for_write():87] open: /project/wandb/run-20240812_063447-whqmtxyq/run-whqmtxyq.wandb +2024-08-12 06:34:47,364 DEBUG SenderThread:13101 [sender.py:send():382] send: header +2024-08-12 06:34:47,378 DEBUG SenderThread:13101 [sender.py:send():382] send: run +2024-08-12 06:34:47,829 INFO SenderThread:13101 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240812_063447-whqmtxyq/files +2024-08-12 06:34:47,829 INFO SenderThread:13101 [sender.py:_start_run_threads():1136] run started: whqmtxyq with start time 1723412087.358797 +2024-08-12 06:34:47,835 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: check_version +2024-08-12 06:34:47,835 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: check_version +2024-08-12 06:34:47,922 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: run_start +2024-08-12 06:34:47,929 DEBUG HandlerThread:13101 [system_info.py:__init__():27] System info init +2024-08-12 06:34:47,929 DEBUG HandlerThread:13101 [system_info.py:__init__():42] System info init done +2024-08-12 06:34:47,929 INFO HandlerThread:13101 [system_monitor.py:start():194] Starting system monitor +2024-08-12 06:34:47,929 INFO SystemMonitor:13101 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-12 06:34:47,929 INFO HandlerThread:13101 [system_monitor.py:probe():214] Collecting system info +2024-08-12 06:34:47,930 INFO SystemMonitor:13101 [interfaces.py:start():190] Started cpu monitoring +2024-08-12 06:34:47,930 INFO SystemMonitor:13101 [interfaces.py:start():190] Started disk monitoring +2024-08-12 06:34:47,930 INFO SystemMonitor:13101 [interfaces.py:start():190] Started gpu monitoring +2024-08-12 06:34:47,931 INFO SystemMonitor:13101 [interfaces.py:start():190] Started memory monitoring +2024-08-12 06:34:47,932 INFO SystemMonitor:13101 [interfaces.py:start():190] Started network monitoring +2024-08-12 06:34:47,942 DEBUG HandlerThread:13101 [system_info.py:probe():151] Probing system +2024-08-12 06:34:47,944 DEBUG HandlerThread:13101 [system_info.py:_probe_git():136] Probing git +2024-08-12 06:34:47,957 DEBUG HandlerThread:13101 [system_info.py:_probe_git():144] Probing git done +2024-08-12 06:34:47,957 DEBUG HandlerThread:13101 [system_info.py:probe():199] Probing system done +2024-08-12 06:34:47,957 DEBUG HandlerThread:13101 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-11T21:34:47.942238', 'startedAt': '2024-08-11T21:34:47.345817', 'docker': None, 'cuda': None, 'args': ('--seq-length', '1021', '--sliding-window-size', '4096', '--micro-batch-size', '1', '--global-batch-size', '320', '--train-iters', '20000', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/google/gemma-2-2b', '--train-data-path', '235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document', '--valid-data-path', '235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document', '--test-data-path', '235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '20000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'anyprecision', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '3', '--eval-interval', '3', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/google/gemma-2-2b', '--save', '/work/llm_recipes/models/yans-sample-gemma-2-2b', '--load', '/work/llm_recipes/models/yans-sample-gemma-2-2b', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/yans-sample-gemma-2-2b', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial', '--wandb-name', 'yans-sample-gemma-2-2b_train_2024-08-12-06:34:36'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '6da01327e78c302bc0cfdb335f3ca297e2a19c8c'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.0429999999997, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.487823486328125}} +2024-08-12 06:34:47,957 INFO HandlerThread:13101 [system_monitor.py:probe():224] Finished collecting system info +2024-08-12 06:34:47,957 INFO HandlerThread:13101 [system_monitor.py:probe():227] Publishing system info +2024-08-12 06:34:47,958 INFO HandlerThread:13101 [system_monitor.py:probe():229] Finished publishing system info +2024-08-12 06:34:47,964 DEBUG SenderThread:13101 [sender.py:send():382] send: files +2024-08-12 06:34:47,964 INFO SenderThread:13101 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-12 06:34:47,974 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: python_packages +2024-08-12 06:34:47,974 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: python_packages +2024-08-12 06:34:47,975 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:34:47,975 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:34:47,976 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:34:48,281 DEBUG SenderThread:13101 [sender.py:send():382] send: telemetry +2024-08-12 06:34:48,615 INFO wandb-upload_0:13101 [upload_job.py:push():131] Uploaded file /tmp/tmpxyme_qqmwandb/cck49p4b-wandb-metadata.json +2024-08-12 06:34:48,831 INFO Thread-12 :13101 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_063447-whqmtxyq/files/output.log +2024-08-12 06:34:48,831 INFO Thread-12 :13101 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_063447-whqmtxyq/files/requirements.txt +2024-08-12 06:34:48,832 INFO Thread-12 :13101 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_063447-whqmtxyq/files/wandb-metadata.json +2024-08-12 06:34:50,832 INFO Thread-12 :13101 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063447-whqmtxyq/files/output.log +2024-08-12 06:34:52,543 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:34:52,833 INFO Thread-12 :13101 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063447-whqmtxyq/files/output.log +2024-08-12 06:34:57,543 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:35:02,544 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:35:02,974 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:35:02,974 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:35:02,975 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:35:08,234 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:35:13,235 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:35:17,973 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:35:17,974 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:35:18,016 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:35:18,247 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:35:18,849 INFO Thread-12 :13101 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063447-whqmtxyq/files/config.yaml +2024-08-12 06:35:23,452 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:35:28,453 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:35:32,974 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:35:32,974 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:35:33,016 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:35:34,202 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:35:39,202 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:35:44,203 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:35:47,932 DEBUG SystemMonitor:13101 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-12 06:35:47,934 DEBUG SenderThread:13101 [sender.py:send():382] send: stats +2024-08-12 06:35:47,973 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:35:47,974 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:35:48,016 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:35:49,237 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:35:54,238 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:35:59,239 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:36:02,974 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:36:02,974 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:36:03,016 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:36:05,234 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:36:08,884 INFO Thread-12 :13101 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063447-whqmtxyq/files/output.log +2024-08-12 06:36:09,915 DEBUG SenderThread:13101 [sender.py:send():382] send: config +2024-08-12 06:36:09,915 DEBUG SenderThread:13101 [sender.py:send():382] send: config +2024-08-12 06:36:10,885 INFO Thread-12 :13101 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063447-whqmtxyq/files/output.log +2024-08-12 06:36:11,119 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:36:16,120 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:36:17,935 DEBUG SenderThread:13101 [sender.py:send():382] send: stats +2024-08-12 06:36:17,974 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:36:17,974 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:36:17,975 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:36:21,237 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:36:21,893 INFO Thread-12 :13101 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063447-whqmtxyq/files/config.yaml +2024-08-12 06:36:26,451 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:36:31,452 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:36:32,974 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:36:32,974 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:36:33,016 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:36:37,174 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:36:42,174 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:36:47,175 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:36:47,936 DEBUG SenderThread:13101 [sender.py:send():382] send: stats +2024-08-12 06:36:47,975 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:36:47,975 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:36:48,016 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:36:52,199 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:36:57,199 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:37:02,200 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:37:02,975 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:37:02,975 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:37:03,016 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:37:08,177 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:37:13,178 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:37:17,937 DEBUG SenderThread:13101 [sender.py:send():382] send: stats +2024-08-12 06:37:17,975 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:37:17,975 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:37:18,016 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:37:18,239 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:37:23,240 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:37:28,240 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:37:32,975 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:37:32,975 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:37:33,016 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:37:33,471 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 06:37:33,513 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:37:34,938 INFO Thread-12 :13101 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063447-whqmtxyq/files/output.log +2024-08-12 06:37:38,514 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:37:43,515 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:37:47,938 DEBUG SenderThread:13101 [sender.py:send():382] send: stats +2024-08-12 06:37:47,975 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:37:47,975 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:37:47,977 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:37:49,236 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:37:54,236 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:37:59,237 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:38:02,975 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:38:02,975 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:38:03,016 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:38:05,173 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:38:10,174 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:38:15,175 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:38:17,940 DEBUG SenderThread:13101 [sender.py:send():382] send: stats +2024-08-12 06:38:17,975 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:38:17,975 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:38:18,016 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:38:20,189 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:38:25,189 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:38:30,190 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:38:32,975 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:38:32,976 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:38:33,016 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:38:36,181 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:38:41,181 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:38:46,182 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:38:47,941 DEBUG SenderThread:13101 [sender.py:send():382] send: stats +2024-08-12 06:38:47,975 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:38:47,976 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:38:48,016 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:38:52,158 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:38:57,068 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 06:38:57,070 DEBUG SenderThread:13101 [sender.py:send():382] send: history +2024-08-12 06:38:57,071 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: summary_record +2024-08-12 06:38:57,072 INFO SenderThread:13101 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 06:38:57,991 INFO Thread-12 :13101 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_063447-whqmtxyq/files/wandb-summary.json +2024-08-12 06:38:58,109 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:38:58,991 INFO Thread-12 :13101 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063447-whqmtxyq/files/output.log +2024-08-12 06:39:02,976 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:39:02,977 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:39:02,977 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:39:03,220 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:39:08,220 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:39:13,221 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:39:17,942 DEBUG SenderThread:13101 [sender.py:send():382] send: stats +2024-08-12 06:39:17,976 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:39:17,977 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:39:18,020 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:39:19,166 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:39:24,167 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:39:29,167 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:39:32,976 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:39:32,976 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:39:33,016 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:39:34,262 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:39:39,263 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:39:44,264 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:39:47,943 DEBUG SenderThread:13101 [sender.py:send():382] send: stats +2024-08-12 06:39:47,976 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:39:47,976 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:39:48,016 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:39:50,213 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:39:55,214 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:40:00,215 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:40:02,976 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:40:02,977 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:40:03,016 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:40:05,253 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:40:10,254 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:40:15,254 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:40:17,944 DEBUG SenderThread:13101 [sender.py:send():382] send: stats +2024-08-12 06:40:17,976 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:40:17,977 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:40:18,016 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:40:20,601 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 06:40:20,603 DEBUG SenderThread:13101 [sender.py:send():382] send: history +2024-08-12 06:40:20,604 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: summary_record +2024-08-12 06:40:20,604 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:40:20,605 INFO SenderThread:13101 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 06:40:21,044 INFO Thread-12 :13101 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063447-whqmtxyq/files/output.log +2024-08-12 06:40:21,045 INFO Thread-12 :13101 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063447-whqmtxyq/files/wandb-summary.json +2024-08-12 06:40:21,305 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 06:40:23,046 INFO Thread-12 :13101 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063447-whqmtxyq/files/output.log +2024-08-12 06:40:26,337 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:40:31,051 INFO Thread-12 :13101 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063447-whqmtxyq/files/output.log +2024-08-12 06:40:32,226 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:40:32,977 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:40:32,977 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:40:32,979 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:40:37,055 INFO Thread-12 :13101 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063447-whqmtxyq/files/output.log +2024-08-12 06:40:37,381 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:40:42,382 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:40:44,855 DEBUG SenderThread:13101 [sender.py:send():382] send: exit +2024-08-12 06:40:44,856 INFO SenderThread:13101 [sender.py:send_exit():589] handling exit code: 1 +2024-08-12 06:40:44,856 INFO SenderThread:13101 [sender.py:send_exit():591] handling runtime: 356 +2024-08-12 06:40:44,857 INFO SenderThread:13101 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 06:40:44,857 INFO SenderThread:13101 [sender.py:send_exit():597] send defer +2024-08-12 06:40:44,857 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:40:44,857 INFO HandlerThread:13101 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-12 06:40:44,857 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: defer +2024-08-12 06:40:44,857 INFO SenderThread:13101 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-12 06:40:44,857 INFO SenderThread:13101 [sender.py:transition_state():617] send defer: 1 +2024-08-12 06:40:44,858 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:40:44,858 INFO HandlerThread:13101 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-12 06:40:44,858 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: defer +2024-08-12 06:40:44,858 INFO SenderThread:13101 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-12 06:40:44,858 INFO SenderThread:13101 [sender.py:transition_state():617] send defer: 2 +2024-08-12 06:40:44,858 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:40:44,858 INFO HandlerThread:13101 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-12 06:40:44,858 INFO HandlerThread:13101 [system_monitor.py:finish():203] Stopping system monitor +2024-08-12 06:40:44,858 DEBUG SystemMonitor:13101 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-12 06:40:44,858 INFO HandlerThread:13101 [interfaces.py:finish():202] Joined cpu monitor +2024-08-12 06:40:44,859 DEBUG SystemMonitor:13101 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-12 06:40:44,859 INFO HandlerThread:13101 [interfaces.py:finish():202] Joined disk monitor +2024-08-12 06:40:44,893 INFO HandlerThread:13101 [interfaces.py:finish():202] Joined gpu monitor +2024-08-12 06:40:44,893 INFO HandlerThread:13101 [interfaces.py:finish():202] Joined memory monitor +2024-08-12 06:40:44,893 INFO HandlerThread:13101 [interfaces.py:finish():202] Joined network monitor +2024-08-12 06:40:44,894 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: defer +2024-08-12 06:40:44,894 INFO SenderThread:13101 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-12 06:40:44,894 INFO SenderThread:13101 [sender.py:transition_state():617] send defer: 3 +2024-08-12 06:40:44,894 DEBUG SenderThread:13101 [sender.py:send():382] send: stats +2024-08-12 06:40:44,894 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:40:44,894 INFO HandlerThread:13101 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-12 06:40:44,896 DEBUG SenderThread:13101 [sender.py:send():382] send: history +2024-08-12 06:40:44,896 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: summary_record +2024-08-12 06:40:44,897 INFO SenderThread:13101 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 06:40:44,898 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: defer +2024-08-12 06:40:44,898 INFO SenderThread:13101 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-12 06:40:44,898 INFO SenderThread:13101 [sender.py:transition_state():617] send defer: 4 +2024-08-12 06:40:44,898 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:40:44,898 INFO HandlerThread:13101 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-12 06:40:44,898 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: defer +2024-08-12 06:40:44,898 INFO SenderThread:13101 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-12 06:40:44,898 INFO SenderThread:13101 [sender.py:transition_state():617] send defer: 5 +2024-08-12 06:40:44,898 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:40:44,898 INFO HandlerThread:13101 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-12 06:40:44,899 DEBUG SenderThread:13101 [sender.py:send():382] send: summary +2024-08-12 06:40:44,900 INFO SenderThread:13101 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 06:40:44,900 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: defer +2024-08-12 06:40:44,900 INFO SenderThread:13101 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-12 06:40:44,900 INFO SenderThread:13101 [sender.py:transition_state():617] send defer: 6 +2024-08-12 06:40:44,900 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:40:44,900 INFO HandlerThread:13101 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-12 06:40:44,900 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: defer +2024-08-12 06:40:44,900 INFO SenderThread:13101 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-12 06:40:44,901 INFO SenderThread:13101 [sender.py:transition_state():617] send defer: 7 +2024-08-12 06:40:44,901 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:40:44,901 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:40:44,901 INFO HandlerThread:13101 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-12 06:40:44,901 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: defer +2024-08-12 06:40:44,901 INFO SenderThread:13101 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-12 06:40:45,060 INFO Thread-12 :13101 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063447-whqmtxyq/files/output.log +2024-08-12 06:40:45,061 INFO Thread-12 :13101 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063447-whqmtxyq/files/wandb-summary.json +2024-08-12 06:40:45,855 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-12 06:40:47,007 INFO SenderThread:13101 [sender.py:transition_state():617] send defer: 8 +2024-08-12 06:40:47,007 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: poll_exit +2024-08-12 06:40:47,007 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:40:47,008 INFO HandlerThread:13101 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-12 06:40:47,008 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: defer +2024-08-12 06:40:47,008 INFO SenderThread:13101 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-12 06:40:47,008 INFO SenderThread:13101 [job_builder.py:build():296] Attempting to build job artifact +2024-08-12 06:40:47,009 INFO SenderThread:13101 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-12 06:40:47,023 INFO SenderThread:13101 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-12 06:40:47,031 INFO SenderThread:13101 [sender.py:transition_state():617] send defer: 9 +2024-08-12 06:40:47,032 DEBUG SenderThread:13101 [sender.py:send():382] send: artifact +2024-08-12 06:40:47,032 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:40:47,033 INFO HandlerThread:13101 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-12 06:40:47,062 INFO Thread-12 :13101 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063447-whqmtxyq/files/output.log +2024-08-12 06:40:47,856 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-12 06:40:47,912 INFO SenderThread:13101 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTE0MDA5NDY1MQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjM2MjY3MjMzNA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTE0MDA5NDY1MQ==', 'versionIndex': 9}}} +2024-08-12 06:40:47,912 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: defer +2024-08-12 06:40:47,912 INFO SenderThread:13101 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-12 06:40:47,913 INFO SenderThread:13101 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-12 06:40:48,063 INFO SenderThread:13101 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240812_063447-whqmtxyq/files +2024-08-12 06:40:48,063 INFO SenderThread:13101 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_063447-whqmtxyq/files/requirements.txt requirements.txt +2024-08-12 06:40:48,063 INFO SenderThread:13101 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_063447-whqmtxyq/files/config.yaml config.yaml +2024-08-12 06:40:48,065 INFO SenderThread:13101 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_063447-whqmtxyq/files/wandb-metadata.json wandb-metadata.json +2024-08-12 06:40:48,065 INFO SenderThread:13101 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_063447-whqmtxyq/files/wandb-summary.json wandb-summary.json +2024-08-12 06:40:48,067 INFO SenderThread:13101 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_063447-whqmtxyq/files/output.log output.log +2024-08-12 06:40:48,067 INFO SenderThread:13101 [sender.py:transition_state():617] send defer: 10 +2024-08-12 06:40:48,068 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: poll_exit +2024-08-12 06:40:48,069 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:40:48,069 INFO HandlerThread:13101 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-12 06:40:48,070 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: defer +2024-08-12 06:40:48,071 INFO SenderThread:13101 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-12 06:40:48,071 INFO SenderThread:13101 [file_pusher.py:finish():172] shutting down file pusher +2024-08-12 06:40:48,555 INFO wandb-upload_1:13101 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240812_063447-whqmtxyq/files/config.yaml +2024-08-12 06:40:48,607 INFO wandb-upload_0:13101 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240812_063447-whqmtxyq/files/requirements.txt +2024-08-12 06:40:48,857 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-12 06:40:48,857 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: poll_exit +2024-08-12 06:40:49,047 INFO wandb-upload_2:13101 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240812_063447-whqmtxyq/files/wandb-summary.json +2024-08-12 06:40:49,065 INFO wandb-upload_3:13101 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240812_063447-whqmtxyq/files/output.log +2024-08-12 06:40:49,265 INFO Thread-11 (_thread_body):13101 [sender.py:transition_state():617] send defer: 11 +2024-08-12 06:40:49,266 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:40:49,266 INFO HandlerThread:13101 [handler.py:handle_request_defer():172] handle defer: 11 +2024-08-12 06:40:49,266 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: defer +2024-08-12 06:40:49,266 INFO SenderThread:13101 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-08-12 06:40:49,266 INFO SenderThread:13101 [file_pusher.py:join():178] waiting for file pusher +2024-08-12 06:40:49,266 INFO SenderThread:13101 [sender.py:transition_state():617] send defer: 12 +2024-08-12 06:40:49,267 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:40:49,267 INFO HandlerThread:13101 [handler.py:handle_request_defer():172] handle defer: 12 +2024-08-12 06:40:49,267 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: defer +2024-08-12 06:40:49,267 INFO SenderThread:13101 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-08-12 06:40:49,267 INFO SenderThread:13101 [file_stream.py:finish():595] file stream finish called +2024-08-12 06:40:49,435 INFO SenderThread:13101 [file_stream.py:finish():599] file stream finish is done +2024-08-12 06:40:49,435 INFO SenderThread:13101 [sender.py:transition_state():617] send defer: 13 +2024-08-12 06:40:49,436 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:40:49,436 INFO HandlerThread:13101 [handler.py:handle_request_defer():172] handle defer: 13 +2024-08-12 06:40:49,436 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: defer +2024-08-12 06:40:49,436 INFO SenderThread:13101 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-08-12 06:40:49,436 INFO SenderThread:13101 [sender.py:transition_state():617] send defer: 14 +2024-08-12 06:40:49,436 DEBUG SenderThread:13101 [sender.py:send():382] send: final +2024-08-12 06:40:49,436 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:40:49,436 DEBUG SenderThread:13101 [sender.py:send():382] send: footer +2024-08-12 06:40:49,436 INFO HandlerThread:13101 [handler.py:handle_request_defer():172] handle defer: 14 +2024-08-12 06:40:49,437 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: defer +2024-08-12 06:40:49,437 INFO SenderThread:13101 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-08-12 06:40:49,437 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-12 06:40:49,437 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: poll_exit +2024-08-12 06:40:49,438 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-12 06:40:49,438 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: poll_exit +2024-08-12 06:40:49,438 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: server_info +2024-08-12 06:40:49,438 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: server_info +2024-08-12 06:40:49,439 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: get_summary +2024-08-12 06:40:49,440 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: sampled_history +2024-08-12 06:40:49,442 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:40:49,442 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: job_info +2024-08-12 06:40:49,609 DEBUG SenderThread:13101 [sender.py:send_request():409] send_request: job_info +2024-08-12 06:40:49,610 INFO MainThread:13101 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-08-12 06:40:49,610 INFO MainThread:13101 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-08-12 06:40:49,611 INFO MainThread:13101 [wandb_run.py:_footer_sync_info():3825] logging synced files +2024-08-12 06:40:49,611 DEBUG HandlerThread:13101 [handler.py:handle_request():146] handle_request: shutdown +2024-08-12 06:40:49,611 INFO HandlerThread:13101 [handler.py:finish():869] shutting down handler +2024-08-12 06:40:50,442 INFO WriterThread:13101 [datastore.py:close():296] close: /project/wandb/run-20240812_063447-whqmtxyq/run-whqmtxyq.wandb +2024-08-12 06:40:50,610 INFO SenderThread:13101 [sender.py:finish():1572] shutting down sender +2024-08-12 06:40:50,610 INFO SenderThread:13101 [file_pusher.py:finish():172] shutting down file pusher +2024-08-12 06:40:50,610 INFO SenderThread:13101 [file_pusher.py:join():178] waiting for file pusher diff --git a/wandb/run-20240812_063447-whqmtxyq/logs/debug.log b/wandb/run-20240812_063447-whqmtxyq/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e6b029b2cbc6a9bcc195363d4cd9ea12ea55a121 --- /dev/null +++ b/wandb/run-20240812_063447-whqmtxyq/logs/debug.log @@ -0,0 +1,30 @@ +2024-08-12 06:34:47,351 INFO MainThread:13030 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-12 06:34:47,352 INFO MainThread:13030 [wandb_setup.py:_flush():76] Configure stats pid to 13030 +2024-08-12 06:34:47,352 INFO MainThread:13030 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-12 06:34:47,352 INFO MainThread:13030 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-12 06:34:47,352 INFO MainThread:13030 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train sample'} +2024-08-12 06:34:47,352 INFO MainThread:13030 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-12 06:34:47,352 INFO MainThread:13030 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-12 06:34:47,352 INFO MainThread:13030 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240812_063447-whqmtxyq/logs/debug.log +2024-08-12 06:34:47,352 INFO MainThread:13030 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240812_063447-whqmtxyq/logs/debug-internal.log +2024-08-12 06:34:47,352 INFO MainThread:13030 [wandb_init.py:init():566] calling init triggers +2024-08-12 06:34:47,352 INFO MainThread:13030 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document'], 'valid_data_path': ['235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document'], 'test_data_path': ['235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 1021, 'num_workers': 2, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/google/gemma-2-2b', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'yans-sample-gemma-2-2b_train_2024-08-12-06:34:36', 'wandb_project': 'llm_tutorial', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/yans-sample-gemma-2-2b', 'save': '/work/llm_recipes/models/yans-sample-gemma-2-2b', 'base_model': '/share/pretrained_lm/google/gemma-2-2b', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 3, 'save_interval': 3, 'eval_iters': 10, 'optimizer': 'anyprecision', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 20000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 20000, 'train_samples': None, 'global_batch_size': 320, 'micro_batch_size': 1, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 4096, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/yans-sample-gemma-2-2b', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 256000, 'gradient_accumulation_steps': 320} +2024-08-12 06:34:47,353 INFO MainThread:13030 [wandb_init.py:init():616] starting backend +2024-08-12 06:34:47,353 INFO MainThread:13030 [wandb_init.py:init():620] setting up manager +2024-08-12 06:34:47,357 INFO MainThread:13030 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-12 06:34:47,358 INFO MainThread:13030 [wandb_init.py:init():628] backend started and connected +2024-08-12 06:34:47,363 INFO MainThread:13030 [wandb_init.py:init():720] updated telemetry +2024-08-12 06:34:47,374 INFO MainThread:13030 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-12 06:34:47,834 INFO MainThread:13030 [wandb_run.py:_on_init():2262] communicating current version +2024-08-12 06:34:47,915 INFO MainThread:13030 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.6 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-12 06:34:47,915 INFO MainThread:13030 [wandb_init.py:init():804] starting run threads in backend +2024-08-12 06:34:47,973 INFO MainThread:13030 [wandb_run.py:_console_start():2241] atexit reg +2024-08-12 06:34:47,973 INFO MainThread:13030 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-12 06:34:47,973 INFO MainThread:13030 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-12 06:34:47,974 INFO MainThread:13030 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-12 06:34:47,975 INFO MainThread:13030 [wandb_init.py:init():847] run started, returning control to user process +2024-08-12 06:36:09,914 INFO MainThread:13030 [wandb_run.py:_config_callback():1343] config_cb None None {'model_architecture': 'Gemma2ForCausalLM', 'activation_function': 'gelu_pytorch_tanh', 'hidden_size': 2304, 'model_type': 'gemma2', 'max_position_embeddings': 1021, 'num_attention_heads': 8, 'num_hidden_layers': 26} +2024-08-12 06:36:09,915 INFO MainThread:13030 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1} +2024-08-12 06:40:50,612 WARNING MsgRouterThr:13030 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20240812_063447-whqmtxyq/run-whqmtxyq.wandb b/wandb/run-20240812_063447-whqmtxyq/run-whqmtxyq.wandb new file mode 100644 index 0000000000000000000000000000000000000000..69e45cdb71fee16383818720fd7a3da537670a14 Binary files /dev/null and b/wandb/run-20240812_063447-whqmtxyq/run-whqmtxyq.wandb differ diff --git a/wandb/run-20240815_031216-0szn78ph/files/config.yaml b/wandb/run-20240815_031216-0szn78ph/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c35ecf3066b15054ea00fc86923800023679b10 --- /dev/null +++ b/wandb/run-20240815_031216-0szn78ph/files/config.yaml @@ -0,0 +1,335 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: FULL_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '304771887' + - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document +valid_data_path: + desc: null + value: + - '304771887' + - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document +test_data_path: + desc: null + value: + - '304771887' + - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 4096 +num_workers: + desc: null + value: 2 +tokenizer_type: + desc: null + value: HFPreTrainedTokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: yans-qwen2-0.5B_train_2024-08-15-03:11:59 +wandb_project: + desc: null + value: llm_tutorial +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/yans-qwen2-0.5B +save: + desc: null + value: /work/llm_recipes/models/yans-qwen2-0.5B +base_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 10 +save_interval: + desc: null + value: 10 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: adam +lr: + desc: null + value: 2.0e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 20000 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 1.0e-06 +train_iters: + desc: null + value: 20000 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 320 +micro_batch_size: + desc: null + value: 1 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 4096 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-06 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/yans-qwen2-0.5B +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: false +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 151680 +gradient_accumulation_steps: + desc: null + value: 320 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1723659136.24386 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +model_architecture: + desc: null + value: Qwen2ForCausalLM +activation_function: + desc: null + value: silu +hidden_size: + desc: null + value: 896 +model_type: + desc: null + value: qwen2 +max_position_embeddings: + desc: null + value: 4096 +num_attention_heads: + desc: null + value: 14 +num_hidden_layers: + desc: null + value: 24 diff --git a/wandb/run-20240815_031216-0szn78ph/files/output.log b/wandb/run-20240815_031216-0szn78ph/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..5b45b2d8a896086aee28a536293b24a9eb69cc5a --- /dev/null +++ b/wandb/run-20240815_031216-0szn78ph/files/output.log @@ -0,0 +1,92 @@ +Created Hugging Face repository with ID koichi12/yans-qwen2-0.5B. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +Loading model state dict from /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001160/model.pt +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +Loaded model state dict from /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001160/model.pt +--> Model /share/pretrained_lm/Qwen/Qwen2-0.5B +--> /share/pretrained_lm/Qwen/Qwen2-0.5B has 494.032768 Million params +BFloat16 enabled for mixed precision - using bfSixteen policy +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1. + warnings.warn( +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 6400000 + validation: 6403200 + test: 3200 +> building train, validation, and test datasets for GPT ... +> finished creating GPT datasets ... +Loading optimizer state dict from /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001160/optimizer.pt +Let split = None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Loaded optimizer state dict from /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001160/optimizer.pt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): Qwen2ForCausalLM( + (model): Qwen2Model( + (embed_tokens): Embedding(151936, 896) + (layers): ModuleList( + (0-23): 24 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): Qwen2DecoderLayer( + (self_attn): Qwen2FlashAttention2( + (q_proj): Linear(in_features=896, out_features=896, bias=True) + (k_proj): Linear(in_features=896, out_features=128, bias=True) + (v_proj): Linear(in_features=896, out_features=128, bias=True) + (o_proj): Linear(in_features=896, out_features=896, bias=False) + (rotary_emb): Qwen2RotaryEmbedding() + ) + (mlp): Qwen2MLP( + (gate_proj): Linear(in_features=896, out_features=4864, bias=False) + (up_proj): Linear(in_features=896, out_features=4864, bias=False) + (down_proj): Linear(in_features=4864, out_features=896, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): Qwen2RMSNorm() + (post_attention_layernorm): Qwen2RMSNorm() + ) + ) + ) + ) + (norm): Qwen2RMSNorm() + ) + (lm_head): Linear(in_features=896, out_features=151936, bias=False) + ) +) +model config: Qwen2Config { + "_name_or_path": "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "label_smoothing": 0.0, + "max_position_embeddings": 4096, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-06, + "rope_theta": 1000000.0, + "sliding_window": null, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.43.3", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} +[rank0]:[2024-08-15 03:12:42,940] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _flatten_optim_state_dict() profiling: defaultdict(, {}) +------------------------------------------------------------------ +iteration: 1161 , TFLOPS: 67.46644597716896, Tokens per sec: 16778.56616965974, Loss: 2.442603349685669 +------------------------------------------------------------------ \ No newline at end of file diff --git a/wandb/run-20240815_031216-0szn78ph/files/requirements.txt b/wandb/run-20240815_031216-0szn78ph/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..182ea92f1c353b184cb113e6088d68947f9289a8 --- /dev/null +++ b/wandb/run-20240815_031216-0szn78ph/files/requirements.txt @@ -0,0 +1,293 @@ +absl-py==2.1.0 +accelerate==0.33.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bleach==6.1.0 +blis==0.7.11 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +dataproperty==1.0.1 +datasets==2.20.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +dill==0.3.8 +distributed==2023.11.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +einops==0.7.0 +evaluate==0.4.2 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +grpcio==1.60.1 +huggingface-hub==0.24.5 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +jedi==0.19.1 +jinja2==3.1.3 +joblib==1.3.2 +json5==0.9.14 +jsonlines==4.0.0 +jsonnet==0.19.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +kiwisolver==1.4.5 +langcodes==3.3.0 +lazy-loader==0.3 +librosa==0.10.1 +llvmlite==0.40.1 +lm-eval==0.4.3 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mbstrdecoder==1.1.3 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +multiprocess==0.70.16 +murmurhash==1.0.10 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numexpr==2.10.1 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +onnx==1.15.0rc2 +opencv==4.7.0 +optree==0.10.0 +packaging==23.2 +pandas==1.5.3 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +pathvalidate==3.2.0 +peft==0.11.1 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pynvml==11.4.1 +pyparsing==3.1.1 +pytablewriter==1.2.0 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests==2.32.3 +rich==13.7.0 +rmm==23.12.0 +rouge-score==0.1.2 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.0 +safetensors==0.4.3 +scikit-learn==1.2.0 +scipy==1.12.0 +send2trash==1.8.2 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +sqlitedict==2.1.0 +srsly==2.4.8 +stack-data==0.6.3 +sympy==1.12 +tabledata==1.3.3 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tcolorpy==0.1.6 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm-multiprocess==0.0.11 +tqdm==4.66.5 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +typepy==1.3.2 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.9.0 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +urllib3==1.26.18 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +word2number==1.1 +xdoctest==1.0.2 +xgboost==1.7.6 +xxhash==3.4.1 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 +zstandard==0.23.0 \ No newline at end of file diff --git a/wandb/run-20240815_031216-0szn78ph/files/wandb-metadata.json b/wandb/run-20240815_031216-0szn78ph/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..0f5e87306e87988d91463fcff742bf13f55ade08 --- /dev/null +++ b/wandb/run-20240815_031216-0szn78ph/files/wandb-metadata.json @@ -0,0 +1,215 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-14T18:12:16.980997", + "startedAt": "2024-08-14T18:12:16.230100", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "4096", + "--sliding-window-size", + "4096", + "--micro-batch-size", + "1", + "--global-batch-size", + "320", + "--train-iters", + "20000", + "--tokenizer-type", + "HFPreTrainedTokenizer", + "--tokenizer-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--train-data-path", + "304771887", + "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document", + "--valid-data-path", + "304771887", + "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document", + "--test-data-path", + "304771887", + "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document", + "--lr", + "2e-5", + "--min-lr", + "1e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "20000", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "adam", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-6", + "--save-interval", + "10", + "--eval-interval", + "10", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--save", + "/work/llm_recipes/models/yans-qwen2-0.5B", + "--load", + "/work/llm_recipes/models/yans-qwen2-0.5B", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "FULL_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/yans-qwen2-0.5B", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "llm_tutorial", + "--wandb-name", + "yans-qwen2-0.5B_train_2024-08-15-03:11:59" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "6da01327e78c302bc0cfdb335f3ca297e2a19c8c" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.0389999999993, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.48782730102539 + } +} diff --git a/wandb/run-20240815_031216-0szn78ph/files/wandb-summary.json b/wandb/run-20240815_031216-0szn78ph/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..40e33a233ec0237fd7f33726311694f0b4ee1ac3 --- /dev/null +++ b/wandb/run-20240815_031216-0szn78ph/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 168}, "training/loss": 2.442603349685669, "training/perplexity": 11.502947992429535, "utils/batch_size": 1, "utils/global_batch_size": 320, "utils/seq_len": 4097, "utils/gradient_accumulation_steps": 320, "utils/iteration": 1161, "optimizer/lr": 1.9946184158325198e-05, "optimizer/variance_l2": 0.004682497095771901, "optimizer/variance_sqrt_l2": 0.5343142380105511, "optimizer/momentum_l2": 0.12459250428605805, "optimizer/weight_l2": 825.0639369164065, "optimizer/variance_l1": 0.2849578857421875, "optimizer/variance_sqrt_l1": 4625.0, "optimizer/momentum_l1": 977.875, "optimizer/weight_l1": 6918144.0, "optimizer/variance_abs_max": 0.0030059814453125, "optimizer/variance_sqrt_abs_max": 0.054931640625, "optimizer/momentum_abs_max": 0.0108642578125, "optimizer/weight_abs_max": 175.0, "stats/1_iteration_time": 78.13778523999986, "stats/tokens_per_sec": 16778.56616965974, "stats/tokens_per_sec_per_gpu": 16778.56616965974, "stats/tflops": 67.46644597716896, "_timestamp": 1723659241.8232834, "_runtime": 105.57942342758179, "_step": 1161} \ No newline at end of file diff --git a/wandb/run-20240815_031216-0szn78ph/logs/debug-internal.log b/wandb/run-20240815_031216-0szn78ph/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..46fb74916711fab1a4daccc466ab819843d82722 --- /dev/null +++ b/wandb/run-20240815_031216-0szn78ph/logs/debug-internal.log @@ -0,0 +1,260 @@ +2024-08-15 03:12:16,244 INFO StreamThr :10026 [internal.py:wandb_internal():86] W&B internal server running at pid: 10026, started at: 2024-08-15 03:12:16.243481 +2024-08-15 03:12:16,245 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status +2024-08-15 03:12:16,248 INFO WriterThread:10026 [datastore.py:open_for_write():87] open: /project/wandb/run-20240815_031216-0szn78ph/run-0szn78ph.wandb +2024-08-15 03:12:16,249 DEBUG SenderThread:10026 [sender.py:send():382] send: header +2024-08-15 03:12:16,409 DEBUG SenderThread:10026 [sender.py:send():382] send: run +2024-08-15 03:12:16,887 INFO SenderThread:10026 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240815_031216-0szn78ph/files +2024-08-15 03:12:16,887 INFO SenderThread:10026 [sender.py:_start_run_threads():1136] run started: 0szn78ph with start time 1723659136.24386 +2024-08-15 03:12:16,892 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: check_version +2024-08-15 03:12:16,892 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: check_version +2024-08-15 03:12:16,962 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: run_start +2024-08-15 03:12:16,969 DEBUG HandlerThread:10026 [system_info.py:__init__():27] System info init +2024-08-15 03:12:16,969 DEBUG HandlerThread:10026 [system_info.py:__init__():42] System info init done +2024-08-15 03:12:16,969 INFO HandlerThread:10026 [system_monitor.py:start():194] Starting system monitor +2024-08-15 03:12:16,969 INFO SystemMonitor:10026 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-15 03:12:16,969 INFO HandlerThread:10026 [system_monitor.py:probe():214] Collecting system info +2024-08-15 03:12:16,969 INFO SystemMonitor:10026 [interfaces.py:start():190] Started cpu monitoring +2024-08-15 03:12:16,970 INFO SystemMonitor:10026 [interfaces.py:start():190] Started disk monitoring +2024-08-15 03:12:16,971 INFO SystemMonitor:10026 [interfaces.py:start():190] Started gpu monitoring +2024-08-15 03:12:16,972 INFO SystemMonitor:10026 [interfaces.py:start():190] Started memory monitoring +2024-08-15 03:12:16,972 INFO SystemMonitor:10026 [interfaces.py:start():190] Started network monitoring +2024-08-15 03:12:16,980 DEBUG HandlerThread:10026 [system_info.py:probe():151] Probing system +2024-08-15 03:12:16,983 DEBUG HandlerThread:10026 [system_info.py:_probe_git():136] Probing git +2024-08-15 03:12:16,995 DEBUG HandlerThread:10026 [system_info.py:_probe_git():144] Probing git done +2024-08-15 03:12:16,995 DEBUG HandlerThread:10026 [system_info.py:probe():199] Probing system done +2024-08-15 03:12:16,995 DEBUG HandlerThread:10026 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-14T18:12:16.980997', 'startedAt': '2024-08-14T18:12:16.230100', 'docker': None, 'cuda': None, 'args': ('--seq-length', '4096', '--sliding-window-size', '4096', '--micro-batch-size', '1', '--global-batch-size', '320', '--train-iters', '20000', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--train-data-path', '304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document', '--valid-data-path', '304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document', '--test-data-path', '304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '20000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'adam', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '10', '--eval-interval', '10', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--save', '/work/llm_recipes/models/yans-qwen2-0.5B', '--load', '/work/llm_recipes/models/yans-qwen2-0.5B', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/yans-qwen2-0.5B', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial', '--wandb-name', 'yans-qwen2-0.5B_train_2024-08-15-03:11:59'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '6da01327e78c302bc0cfdb335f3ca297e2a19c8c'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.0389999999993, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.48782730102539}} +2024-08-15 03:12:16,995 INFO HandlerThread:10026 [system_monitor.py:probe():224] Finished collecting system info +2024-08-15 03:12:16,995 INFO HandlerThread:10026 [system_monitor.py:probe():227] Publishing system info +2024-08-15 03:12:16,997 INFO HandlerThread:10026 [system_monitor.py:probe():229] Finished publishing system info +2024-08-15 03:12:17,023 DEBUG SenderThread:10026 [sender.py:send():382] send: files +2024-08-15 03:12:17,024 INFO SenderThread:10026 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-15 03:12:17,033 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: python_packages +2024-08-15 03:12:17,034 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-15 03:12:17,034 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: python_packages +2024-08-15 03:12:17,034 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: stop_status +2024-08-15 03:12:17,036 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: stop_status +2024-08-15 03:12:17,320 DEBUG SenderThread:10026 [sender.py:send():382] send: telemetry +2024-08-15 03:12:17,786 INFO wandb-upload_0:10026 [upload_job.py:push():131] Uploaded file /tmp/tmp2lpzau9swandb/2fbn8bzg-wandb-metadata.json +2024-08-15 03:12:17,889 INFO Thread-12 :10026 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240815_031216-0szn78ph/files/output.log +2024-08-15 03:12:17,889 INFO Thread-12 :10026 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240815_031216-0szn78ph/files/wandb-metadata.json +2024-08-15 03:12:17,889 INFO Thread-12 :10026 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240815_031216-0szn78ph/files/requirements.txt +2024-08-15 03:12:19,889 INFO Thread-12 :10026 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240815_031216-0szn78ph/files/output.log +2024-08-15 03:12:21,867 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:12:21,890 INFO Thread-12 :10026 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240815_031216-0szn78ph/files/output.log +2024-08-15 03:12:22,891 INFO Thread-12 :10026 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240815_031216-0szn78ph/files/output.log +2024-08-15 03:12:26,867 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:12:31,868 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:12:32,032 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: stop_status +2024-08-15 03:12:32,033 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: stop_status +2024-08-15 03:12:32,033 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-15 03:12:37,282 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:12:37,900 INFO Thread-12 :10026 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240815_031216-0szn78ph/files/output.log +2024-08-15 03:12:38,901 INFO Thread-12 :10026 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240815_031216-0szn78ph/files/output.log +2024-08-15 03:12:39,901 INFO Thread-12 :10026 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240815_031216-0szn78ph/files/output.log +2024-08-15 03:12:40,902 INFO Thread-12 :10026 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240815_031216-0szn78ph/files/output.log +2024-08-15 03:12:42,647 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:12:43,260 DEBUG SenderThread:10026 [sender.py:send():382] send: config +2024-08-15 03:12:43,261 DEBUG SenderThread:10026 [sender.py:send():382] send: config +2024-08-15 03:12:43,904 INFO Thread-12 :10026 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240815_031216-0szn78ph/files/output.log +2024-08-15 03:12:44,904 INFO Thread-12 :10026 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240815_031216-0szn78ph/files/output.log +2024-08-15 03:12:47,034 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-15 03:12:47,034 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: stop_status +2024-08-15 03:12:47,035 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: stop_status +2024-08-15 03:12:48,218 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:12:48,907 INFO Thread-12 :10026 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240815_031216-0szn78ph/files/config.yaml +2024-08-15 03:12:53,411 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:12:58,411 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:13:02,035 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-15 03:13:02,035 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: stop_status +2024-08-15 03:13:02,035 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: stop_status +2024-08-15 03:13:04,284 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:13:09,285 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:13:14,285 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:13:16,973 DEBUG SystemMonitor:10026 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-15 03:13:16,974 DEBUG SenderThread:10026 [sender.py:send():382] send: stats +2024-08-15 03:13:17,034 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: stop_status +2024-08-15 03:13:17,034 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: stop_status +2024-08-15 03:13:17,078 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-15 03:13:19,286 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:13:24,287 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:13:29,288 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:13:32,034 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: stop_status +2024-08-15 03:13:32,034 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: stop_status +2024-08-15 03:13:32,078 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-15 03:13:35,214 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:13:40,215 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:13:45,216 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:13:46,975 DEBUG SenderThread:10026 [sender.py:send():382] send: stats +2024-08-15 03:13:47,034 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-15 03:13:47,034 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: stop_status +2024-08-15 03:13:47,035 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: stop_status +2024-08-15 03:13:50,291 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:13:55,292 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:14:00,292 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:14:01,824 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: partial_history +2024-08-15 03:14:01,949 INFO Thread-12 :10026 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240815_031216-0szn78ph/files/output.log +2024-08-15 03:14:02,034 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-15 03:14:02,034 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: stop_status +2024-08-15 03:14:02,035 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: stop_status +2024-08-15 03:14:06,235 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:14:11,236 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:14:16,236 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:14:16,976 DEBUG SenderThread:10026 [sender.py:send():382] send: stats +2024-08-15 03:14:17,034 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-15 03:14:17,034 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: stop_status +2024-08-15 03:14:17,035 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: stop_status +2024-08-15 03:14:21,267 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:14:26,267 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:14:31,268 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:14:32,034 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-15 03:14:32,035 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: stop_status +2024-08-15 03:14:32,035 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: stop_status +2024-08-15 03:14:37,220 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:14:42,221 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:14:46,977 DEBUG SenderThread:10026 [sender.py:send():382] send: stats +2024-08-15 03:14:47,034 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-15 03:14:47,035 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: stop_status +2024-08-15 03:14:47,035 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: stop_status +2024-08-15 03:14:48,220 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:14:53,220 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:14:58,221 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:15:02,034 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-15 03:15:02,035 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: stop_status +2024-08-15 03:15:02,035 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: stop_status +2024-08-15 03:15:03,261 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:15:05,634 DEBUG SenderThread:10026 [sender.py:send():382] send: exit +2024-08-15 03:15:05,634 INFO SenderThread:10026 [sender.py:send_exit():589] handling exit code: 255 +2024-08-15 03:15:05,634 INFO SenderThread:10026 [sender.py:send_exit():591] handling runtime: 168 +2024-08-15 03:15:05,636 INFO SenderThread:10026 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-15 03:15:05,636 INFO SenderThread:10026 [sender.py:send_exit():597] send defer +2024-08-15 03:15:05,636 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: defer +2024-08-15 03:15:05,636 INFO HandlerThread:10026 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-15 03:15:05,636 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: defer +2024-08-15 03:15:05,636 INFO SenderThread:10026 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-15 03:15:05,636 INFO SenderThread:10026 [sender.py:transition_state():617] send defer: 1 +2024-08-15 03:15:05,636 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: defer +2024-08-15 03:15:05,637 INFO HandlerThread:10026 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-15 03:15:05,637 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: defer +2024-08-15 03:15:05,637 INFO SenderThread:10026 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-15 03:15:05,637 INFO SenderThread:10026 [sender.py:transition_state():617] send defer: 2 +2024-08-15 03:15:05,637 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: defer +2024-08-15 03:15:05,637 INFO HandlerThread:10026 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-15 03:15:05,637 INFO HandlerThread:10026 [system_monitor.py:finish():203] Stopping system monitor +2024-08-15 03:15:05,637 DEBUG SystemMonitor:10026 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-15 03:15:05,637 DEBUG SystemMonitor:10026 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-15 03:15:05,637 INFO HandlerThread:10026 [interfaces.py:finish():202] Joined cpu monitor +2024-08-15 03:15:05,639 INFO HandlerThread:10026 [interfaces.py:finish():202] Joined disk monitor +2024-08-15 03:15:05,671 INFO HandlerThread:10026 [interfaces.py:finish():202] Joined gpu monitor +2024-08-15 03:15:05,672 INFO HandlerThread:10026 [interfaces.py:finish():202] Joined memory monitor +2024-08-15 03:15:05,672 INFO HandlerThread:10026 [interfaces.py:finish():202] Joined network monitor +2024-08-15 03:15:05,672 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: defer +2024-08-15 03:15:05,672 INFO SenderThread:10026 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-15 03:15:05,672 INFO SenderThread:10026 [sender.py:transition_state():617] send defer: 3 +2024-08-15 03:15:05,672 DEBUG SenderThread:10026 [sender.py:send():382] send: stats +2024-08-15 03:15:05,673 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: defer +2024-08-15 03:15:05,673 INFO HandlerThread:10026 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-15 03:15:05,676 DEBUG SenderThread:10026 [sender.py:send():382] send: history +2024-08-15 03:15:05,676 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: summary_record +2024-08-15 03:15:05,677 INFO SenderThread:10026 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-15 03:15:05,677 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: defer +2024-08-15 03:15:05,677 INFO SenderThread:10026 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-15 03:15:05,677 INFO SenderThread:10026 [sender.py:transition_state():617] send defer: 4 +2024-08-15 03:15:05,677 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: defer +2024-08-15 03:15:05,677 INFO HandlerThread:10026 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-15 03:15:05,677 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: defer +2024-08-15 03:15:05,677 INFO SenderThread:10026 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-15 03:15:05,677 INFO SenderThread:10026 [sender.py:transition_state():617] send defer: 5 +2024-08-15 03:15:05,677 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: defer +2024-08-15 03:15:05,677 INFO HandlerThread:10026 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-15 03:15:05,678 DEBUG SenderThread:10026 [sender.py:send():382] send: summary +2024-08-15 03:15:05,679 INFO SenderThread:10026 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-15 03:15:05,679 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: defer +2024-08-15 03:15:05,679 INFO SenderThread:10026 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-15 03:15:05,679 INFO SenderThread:10026 [sender.py:transition_state():617] send defer: 6 +2024-08-15 03:15:05,679 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: defer +2024-08-15 03:15:05,679 INFO HandlerThread:10026 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-15 03:15:05,680 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: defer +2024-08-15 03:15:05,680 INFO SenderThread:10026 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-15 03:15:05,680 INFO SenderThread:10026 [sender.py:transition_state():617] send defer: 7 +2024-08-15 03:15:05,680 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:15:05,680 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: defer +2024-08-15 03:15:05,680 INFO HandlerThread:10026 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-15 03:15:05,680 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: defer +2024-08-15 03:15:05,680 INFO SenderThread:10026 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-15 03:15:05,984 INFO Thread-12 :10026 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240815_031216-0szn78ph/files/wandb-summary.json +2024-08-15 03:15:06,481 INFO SenderThread:10026 [sender.py:transition_state():617] send defer: 8 +2024-08-15 03:15:06,481 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: defer +2024-08-15 03:15:06,481 INFO HandlerThread:10026 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-15 03:15:06,481 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: defer +2024-08-15 03:15:06,481 INFO SenderThread:10026 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-15 03:15:06,481 INFO SenderThread:10026 [job_builder.py:build():296] Attempting to build job artifact +2024-08-15 03:15:06,482 INFO SenderThread:10026 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-15 03:15:06,507 INFO SenderThread:10026 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-15 03:15:06,516 INFO SenderThread:10026 [sender.py:transition_state():617] send defer: 9 +2024-08-15 03:15:06,517 DEBUG SenderThread:10026 [sender.py:send():382] send: artifact +2024-08-15 03:15:06,517 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: defer +2024-08-15 03:15:06,518 INFO HandlerThread:10026 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-15 03:15:06,633 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-15 03:15:06,985 INFO Thread-12 :10026 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240815_031216-0szn78ph/files/output.log +2024-08-15 03:15:08,040 INFO wandb-upload_0:10026 [upload_job.py:push():89] Uploaded file /singularity_home/.local/share/wandb/artifacts/staging/tmpb932s___ +2024-08-15 03:15:08,047 INFO wandb-upload_1:10026 [upload_job.py:push():89] Uploaded file /singularity_home/.local/share/wandb/artifacts/staging/tmpl85vnluw +2024-08-15 03:15:09,160 INFO SenderThread:10026 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTE1MDEyMDEwMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjM2MjY3MjMzNA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTE0MDA5NDY1MQ==', 'versionIndex': 9}}} +2024-08-15 03:15:09,160 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: defer +2024-08-15 03:15:09,160 INFO SenderThread:10026 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-15 03:15:09,160 INFO SenderThread:10026 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-15 03:15:09,986 INFO SenderThread:10026 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240815_031216-0szn78ph/files +2024-08-15 03:15:09,987 INFO SenderThread:10026 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240815_031216-0szn78ph/files/requirements.txt requirements.txt +2024-08-15 03:15:09,987 INFO SenderThread:10026 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240815_031216-0szn78ph/files/config.yaml config.yaml +2024-08-15 03:15:09,988 INFO SenderThread:10026 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240815_031216-0szn78ph/files/wandb-metadata.json wandb-metadata.json +2024-08-15 03:15:09,989 INFO SenderThread:10026 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240815_031216-0szn78ph/files/wandb-summary.json wandb-summary.json +2024-08-15 03:15:09,990 INFO SenderThread:10026 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240815_031216-0szn78ph/files/output.log output.log +2024-08-15 03:15:09,992 INFO SenderThread:10026 [sender.py:transition_state():617] send defer: 10 +2024-08-15 03:15:09,992 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: poll_exit +2024-08-15 03:15:09,992 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: defer +2024-08-15 03:15:09,993 INFO HandlerThread:10026 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-15 03:15:09,994 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: defer +2024-08-15 03:15:09,994 INFO SenderThread:10026 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-15 03:15:09,994 INFO SenderThread:10026 [file_pusher.py:finish():172] shutting down file pusher +2024-08-15 03:15:10,399 INFO wandb-upload_1:10026 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240815_031216-0szn78ph/files/config.yaml +2024-08-15 03:15:10,439 INFO wandb-upload_0:10026 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240815_031216-0szn78ph/files/requirements.txt +2024-08-15 03:15:10,453 INFO wandb-upload_2:10026 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240815_031216-0szn78ph/files/wandb-summary.json +2024-08-15 03:15:10,537 INFO wandb-upload_3:10026 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240815_031216-0szn78ph/files/output.log +2024-08-15 03:15:10,635 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-15 03:15:10,635 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: poll_exit +2024-08-15 03:15:10,737 INFO Thread-11 (_thread_body):10026 [sender.py:transition_state():617] send defer: 11 +2024-08-15 03:15:10,738 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: defer +2024-08-15 03:15:10,738 INFO HandlerThread:10026 [handler.py:handle_request_defer():172] handle defer: 11 +2024-08-15 03:15:10,738 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: defer +2024-08-15 03:15:10,738 INFO SenderThread:10026 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-08-15 03:15:10,738 INFO SenderThread:10026 [file_pusher.py:join():178] waiting for file pusher +2024-08-15 03:15:10,738 INFO SenderThread:10026 [sender.py:transition_state():617] send defer: 12 +2024-08-15 03:15:10,738 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: defer +2024-08-15 03:15:10,738 INFO HandlerThread:10026 [handler.py:handle_request_defer():172] handle defer: 12 +2024-08-15 03:15:10,738 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: defer +2024-08-15 03:15:10,738 INFO SenderThread:10026 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-08-15 03:15:10,738 INFO SenderThread:10026 [file_stream.py:finish():595] file stream finish called +2024-08-15 03:15:11,367 INFO SenderThread:10026 [file_stream.py:finish():599] file stream finish is done +2024-08-15 03:15:11,368 INFO SenderThread:10026 [sender.py:transition_state():617] send defer: 13 +2024-08-15 03:15:11,368 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: defer +2024-08-15 03:15:11,368 INFO HandlerThread:10026 [handler.py:handle_request_defer():172] handle defer: 13 +2024-08-15 03:15:11,368 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: defer +2024-08-15 03:15:11,368 INFO SenderThread:10026 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-08-15 03:15:11,368 INFO SenderThread:10026 [sender.py:transition_state():617] send defer: 14 +2024-08-15 03:15:11,369 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: defer +2024-08-15 03:15:11,369 DEBUG SenderThread:10026 [sender.py:send():382] send: final +2024-08-15 03:15:11,369 INFO HandlerThread:10026 [handler.py:handle_request_defer():172] handle defer: 14 +2024-08-15 03:15:11,369 DEBUG SenderThread:10026 [sender.py:send():382] send: footer +2024-08-15 03:15:11,369 DEBUG SenderThread:10026 [sender.py:send_request():409] send_request: defer +2024-08-15 03:15:11,369 INFO SenderThread:10026 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-08-15 03:15:14,370 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:15:19,370 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:15:24,371 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:15:29,371 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:15:34,372 DEBUG HandlerThread:10026 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 03:15:37,452 WARNING StreamThr :10026 [internal.py:is_dead():414] Internal process exiting, parent pid 9957 disappeared +2024-08-15 03:15:37,452 ERROR StreamThr :10026 [internal.py:wandb_internal():152] Internal process shutdown. +2024-08-15 03:15:38,372 INFO SenderThread:10026 [sender.py:finish():1572] shutting down sender +2024-08-15 03:15:38,372 INFO SenderThread:10026 [file_pusher.py:finish():172] shutting down file pusher +2024-08-15 03:15:38,372 INFO SenderThread:10026 [file_pusher.py:join():178] waiting for file pusher +2024-08-15 03:15:38,372 INFO WriterThread:10026 [datastore.py:close():296] close: /project/wandb/run-20240815_031216-0szn78ph/run-0szn78ph.wandb +2024-08-15 03:15:38,373 INFO HandlerThread:10026 [handler.py:finish():869] shutting down handler diff --git a/wandb/run-20240815_031216-0szn78ph/logs/debug.log b/wandb/run-20240815_031216-0szn78ph/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..304c3696adb9151a3de0e2e304aa5b4da715baf5 --- /dev/null +++ b/wandb/run-20240815_031216-0szn78ph/logs/debug.log @@ -0,0 +1,29 @@ +2024-08-15 03:12:16,236 INFO MainThread:9957 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-15 03:12:16,236 INFO MainThread:9957 [wandb_setup.py:_flush():76] Configure stats pid to 9957 +2024-08-15 03:12:16,236 INFO MainThread:9957 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-15 03:12:16,236 INFO MainThread:9957 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-15 03:12:16,236 INFO MainThread:9957 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train Qwen2'} +2024-08-15 03:12:16,236 INFO MainThread:9957 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-15 03:12:16,236 INFO MainThread:9957 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-15 03:12:16,236 INFO MainThread:9957 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240815_031216-0szn78ph/logs/debug.log +2024-08-15 03:12:16,236 INFO MainThread:9957 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240815_031216-0szn78ph/logs/debug-internal.log +2024-08-15 03:12:16,236 INFO MainThread:9957 [wandb_init.py:init():566] calling init triggers +2024-08-15 03:12:16,237 INFO MainThread:9957 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document'], 'valid_data_path': ['304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document'], 'test_data_path': ['304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 4096, 'num_workers': 2, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'yans-qwen2-0.5B_train_2024-08-15-03:11:59', 'wandb_project': 'llm_tutorial', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/yans-qwen2-0.5B', 'save': '/work/llm_recipes/models/yans-qwen2-0.5B', 'base_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 10, 'save_interval': 10, 'eval_iters': 10, 'optimizer': 'adam', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 20000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 20000, 'train_samples': None, 'global_batch_size': 320, 'micro_batch_size': 1, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 4096, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/yans-qwen2-0.5B', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 151680, 'gradient_accumulation_steps': 320} +2024-08-15 03:12:16,237 INFO MainThread:9957 [wandb_init.py:init():616] starting backend +2024-08-15 03:12:16,237 INFO MainThread:9957 [wandb_init.py:init():620] setting up manager +2024-08-15 03:12:16,241 INFO MainThread:9957 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-15 03:12:16,243 INFO MainThread:9957 [wandb_init.py:init():628] backend started and connected +2024-08-15 03:12:16,248 INFO MainThread:9957 [wandb_init.py:init():720] updated telemetry +2024-08-15 03:12:16,405 INFO MainThread:9957 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-15 03:12:16,892 INFO MainThread:9957 [wandb_run.py:_on_init():2262] communicating current version +2024-08-15 03:12:16,915 INFO MainThread:9957 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.6 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-15 03:12:16,915 INFO MainThread:9957 [wandb_init.py:init():804] starting run threads in backend +2024-08-15 03:12:17,032 INFO MainThread:9957 [wandb_run.py:_console_start():2241] atexit reg +2024-08-15 03:12:17,033 INFO MainThread:9957 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-15 03:12:17,033 INFO MainThread:9957 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-15 03:12:17,033 INFO MainThread:9957 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-15 03:12:17,034 INFO MainThread:9957 [wandb_init.py:init():847] run started, returning control to user process +2024-08-15 03:12:43,259 INFO MainThread:9957 [wandb_run.py:_config_callback():1343] config_cb None None {'model_architecture': 'Qwen2ForCausalLM', 'activation_function': 'silu', 'hidden_size': 896, 'model_type': 'qwen2', 'max_position_embeddings': 4096, 'num_attention_heads': 14, 'num_hidden_layers': 24} +2024-08-15 03:12:43,260 INFO MainThread:9957 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1} diff --git a/wandb/run-20240815_031216-0szn78ph/run-0szn78ph.wandb b/wandb/run-20240815_031216-0szn78ph/run-0szn78ph.wandb new file mode 100644 index 0000000000000000000000000000000000000000..c8a1c4658dd8dfc7ac958dab5d2156b9a27da8aa Binary files /dev/null and b/wandb/run-20240815_031216-0szn78ph/run-0szn78ph.wandb differ diff --git a/wandb/run-20240823_162543-eroprw00/files/config.yaml b/wandb/run-20240823_162543-eroprw00/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cefea98b1cc1440efae24ea65fa155e9ea8cbfb1 --- /dev/null +++ b/wandb/run-20240823_162543-eroprw00/files/config.yaml @@ -0,0 +1,342 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: FULL_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '1754785366' + - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document + - '28623823675' + - /project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document +valid_data_path: + desc: null + value: + - '1754785366' + - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document +test_data_path: + desc: null + value: + - '1754785366' + - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 4096 +num_workers: + desc: null + value: 2 +tokenizer_type: + desc: null + value: HFPreTrainedTokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: Qwen2-0.5b-0.2_train_2024-08-23-16:25:30 +wandb_project: + desc: null + value: llm_tutorial-0.2 +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/Qwen2-0.5b-0.2 +save: + desc: null + value: /work/llm_recipes/models/Qwen2-0.5b-0.2 +base_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 10 +save_interval: + desc: null + value: 10 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: anyprecision +lr: + desc: null + value: 2.0e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 7500 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 1.0e-06 +train_iters: + desc: null + value: 7500 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 640 +micro_batch_size: + desc: null + value: 5 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 131072 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-06 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/Qwen2-0.5b-0.2 +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: true +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +valid_micro_batch_size: + desc: null + value: 1 +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 151680 +gradient_accumulation_steps: + desc: null + value: 128 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1724397943.202675 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +model_architecture: + desc: null + value: Qwen2ForCausalLM +activation_function: + desc: null + value: silu +hidden_size: + desc: null + value: 896 +model_type: + desc: null + value: qwen2 +max_position_embeddings: + desc: null + value: 4096 +num_attention_heads: + desc: null + value: 14 +num_hidden_layers: + desc: null + value: 24 diff --git a/wandb/run-20240823_162543-eroprw00/files/output.log b/wandb/run-20240823_162543-eroprw00/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..583d4517d0c43603db9e3902c1f8d76647b25ad3 --- /dev/null +++ b/wandb/run-20240823_162543-eroprw00/files/output.log @@ -0,0 +1,116 @@ +Created Hugging Face repository with ID koichi12/Qwen2-0.5b-0.2. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +Loading model state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/model.pt +Loaded model state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/model.pt +--> Model /share/pretrained_lm/Qwen/Qwen2-0.5B +--> /share/pretrained_lm/Qwen/Qwen2-0.5B has 494.032768 Million params +BFloat16 enabled for mixed precision - using bfSixteen policy +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1. + warnings.warn( +Let split = None +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 4800000 + validation: 4806400 + test: 6400 +> building train, validation, and test datasets for GPT ... +Unable to save the indexes because path_to_cache is None +> finished creating GPT datasets ... +Loading optimizer state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/optimizer.pt +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Loaded optimizer state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/optimizer.pt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): Qwen2ForCausalLM( + (model): Qwen2Model( + (embed_tokens): Embedding(151936, 896) + (layers): ModuleList( + (0-23): 24 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): Qwen2DecoderLayer( + (self_attn): Qwen2FlashAttention2( + (q_proj): Linear(in_features=896, out_features=896, bias=True) + (k_proj): Linear(in_features=896, out_features=128, bias=True) + (v_proj): Linear(in_features=896, out_features=128, bias=True) + (o_proj): Linear(in_features=896, out_features=896, bias=False) + (rotary_emb): Qwen2RotaryEmbedding() + ) + (mlp): Qwen2MLP( + (gate_proj): Linear(in_features=896, out_features=4864, bias=False) + (up_proj): Linear(in_features=896, out_features=4864, bias=False) + (down_proj): Linear(in_features=4864, out_features=896, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): Qwen2RMSNorm() + (post_attention_layernorm): Qwen2RMSNorm() + ) + ) + ) + ) + (norm): Qwen2RMSNorm() + ) + (lm_head): Linear(in_features=896, out_features=151936, bias=False) + ) +) +model config: Qwen2Config { + "_name_or_path": "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "label_smoothing": 0.0, + "max_position_embeddings": 4096, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-06, + "rope_theta": 1000000.0, + "sliding_window": 131072, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.43.3", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} +[rank0]:[2024-08-23 16:25:50,866] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _flatten_optim_state_dict() profiling: defaultdict(, {}) +Traceback (most recent call last): + File "/project/examples/finetuning.py", line 13, in + main() + File "/project/src/llama_recipes/finetuning.py", line 282, in main + train( + File "/project/src/llama_recipes/utils/train_utils.py", line 110, in train + loss: torch.Tensor = model(**batch).loss + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl + return forward_call(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py", line 849, in forward + output = self._fsdp_wrapped_module(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl + return forward_call(*args, **kwargs) + File "/project/lib/transformers/src/transformers/models/qwen2/modeling_qwen2.py", line 1082, in forward + loss = loss_fct(shift_logits, shift_labels) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl + return forward_call(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/loss.py", line 1179, in forward + return F.cross_entropy(input, target, weight=self.weight, + File "/usr/local/lib/python3.10/dist-packages/torch/nn/functional.py", line 3086, in cross_entropy + return torch._C._nn.cross_entropy_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index, label_smoothing) +torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 11.59 GiB. GPU 0 has a total capacity of 39.39 GiB of which 11.28 GiB is free. Including non-PyTorch memory, this process has 28.11 GiB memory in use. Of the allocated memory 26.94 GiB is allocated by PyTorch, and 363.22 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) \ No newline at end of file diff --git a/wandb/run-20240823_162543-eroprw00/files/requirements.txt b/wandb/run-20240823_162543-eroprw00/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..34a2774e444bdc395654ccf8ce6da6833c7bc1ee --- /dev/null +++ b/wandb/run-20240823_162543-eroprw00/files/requirements.txt @@ -0,0 +1,375 @@ +absl-py==2.1.0 +accelerate==0.23.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +astroid==3.2.4 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bert-score==0.3.13 +bleach==6.1.0 +blis==0.7.11 +build==1.2.1 +cachecontrol==0.14.0 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +cleo==2.1.0 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cramjam==2.8.3 +crashtest==0.4.1 +cryptography==43.0.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +dataclasses-json==0.6.7 +dataproperty==1.0.1 +datasets==2.20.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +dill==0.3.8 +distlib==0.3.8 +distributed==2023.11.0 +distro==1.9.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +dulwich==0.21.7 +einops==0.7.0 +emoji==2.12.1 +entmax==1.3 +evaluate==0.4.2 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastparquet==2023.10.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +fugashi==1.3.2 +fuzzywuzzy==0.18.0 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +greenlet==3.0.3 +grpcio==1.60.1 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.24.5 +hydra-core==1.3.2 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +installer==0.7.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +isort==5.13.2 +jaraco.classes==3.4.0 +jedi==0.19.1 +jeepney==0.8.0 +jinja2==3.1.3 +jiter==0.5.0 +joblib==1.3.2 +json5==0.9.14 +jsonargparse==3.13.1 +jsonlines==4.0.0 +jsonnet==0.19.1 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +keyring==24.3.1 +kiwisolver==1.4.5 +langchain-community==0.2.12 +langchain-core==0.2.31 +langchain-huggingface==0.0.2 +langchain-openai==0.1.21 +langchain-text-splitters==0.2.2 +langchain==0.2.13 +langcodes==3.3.0 +langsmith==0.1.99 +lazy-loader==0.3 +levenshtein==0.25.1 +librosa==0.10.1 +lightning-utilities==0.11.6 +llm-jp-eval==1.4.0 +llvmlite==0.40.1 +lm-eval==0.3.0 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +marshmallow==3.21.3 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mbstrdecoder==1.1.3 +mccabe==0.7.0 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +mojimoji==0.0.13 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +multiprocess==0.70.16 +murmurhash==1.0.10 +mypy-extensions==1.0.0 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +neologdn==0.5.3 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numexpr==2.10.1 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +omegaconf==2.3.0 +onnx==1.15.0rc2 +openai==1.40.6 +opencv==4.7.0 +optree==0.10.0 +orjson==3.10.7 +packaging==23.2 +pandas==2.2.2 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +pathvalidate==3.2.0 +peft==0.5.0 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +pkginfo==1.11.1 +plac==1.4.3 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +poetry-core==1.9.0 +poetry-plugin-export==1.8.0 +poetry==1.8.3 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow-hotfix==0.6 +pyarrow==15.0.2 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycountry==24.6.1 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pylint==3.2.6 +pynvml==11.4.1 +pyparsing==3.1.1 +pyproject-hooks==1.1.0 +pytablewriter==1.2.0 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +python-levenshtein==0.25.1 +pytorch-lightning==2.4.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapidfuzz==3.9.6 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests-toolbelt==1.0.0 +requests==2.32.3 +rhoknp==1.7.0 +rich==13.7.0 +rmm==23.12.0 +rouge-score==0.1.2 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.2 +safetensors==0.4.3 +scikit-learn==1.5.1 +scipy==1.12.0 +secretstorage==3.3.3 +send2trash==1.8.2 +sentence-transformers==3.0.1 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +shellingham==1.5.4 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +sqlalchemy==2.0.32 +sqlitedict==2.1.0 +srsly==2.4.8 +stack-data==0.6.3 +sumeval==0.2.2 +sympy==1.12 +tabledata==1.3.3 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tcolorpy==0.1.6 +tenacity==8.5.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +text-generation==0.7.0 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tiktoken==0.7.0 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +tomlkit==0.13.2 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchmetrics==0.10.3 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm-multiprocess==0.0.11 +tqdm==4.66.5 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +trove-classifiers==2024.7.2 +typepy==1.3.2 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.12.2 +typing-inspect==0.9.0 +tzdata==2024.1 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +unbabel-comet==2.2.2 +unidic-lite==1.0.8 +urllib3==1.26.18 +virtualenv==20.26.3 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +word2number==1.1 +xdoctest==1.0.2 +xgboost==1.7.6 +xmltodict==0.13.0 +xxhash==3.4.1 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 +zstandard==0.23.0 \ No newline at end of file diff --git a/wandb/run-20240823_162543-eroprw00/files/wandb-metadata.json b/wandb/run-20240823_162543-eroprw00/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..ed024780147682de1c07b9ce7fcb991551161e77 --- /dev/null +++ b/wandb/run-20240823_162543-eroprw00/files/wandb-metadata.json @@ -0,0 +1,220 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-23T07:25:43.758914", + "startedAt": "2024-08-23T07:25:43.187250", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "4096", + "--sliding-window-size", + "131072", + "--micro-batch-size", + "5", + "--valid_micro_batch_size", + "1", + "--global-batch-size", + "640", + "--train-iters", + "7500", + "--tokenizer-type", + "HFPreTrainedTokenizer", + "--tokenizer-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--train-data-path", + "1754785366", + "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document", + "28623823675", + "/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document", + "--valid-data-path", + "1754785366", + "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document", + "--test-data-path", + "1754785366", + "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document", + "--lr", + "2e-5", + "--min-lr", + "1e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "7500", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "anyprecision", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-6", + "--save-interval", + "10", + "--eval-interval", + "10", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--save", + "/work/llm_recipes/models/Qwen2-0.5b-0.2", + "--load", + "/work/llm_recipes/models/Qwen2-0.5b-0.2", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "FULL_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--upload-all-checkpoints-to-hf", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/Qwen2-0.5b-0.2", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "llm_tutorial-0.2", + "--wandb-name", + "Qwen2-0.5b-0.2_train_2024-08-23-16:25:30" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "887a2cc5d104c10264701f95cbbb0a6a116768d6" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.0389999999993, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.487831115722656 + } +} diff --git a/wandb/run-20240823_162543-eroprw00/files/wandb-summary.json b/wandb/run-20240823_162543-eroprw00/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a594e25ada167c5ad54a828e8a5b1a6223620b44 --- /dev/null +++ b/wandb/run-20240823_162543-eroprw00/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 8}} \ No newline at end of file diff --git a/wandb/run-20240823_162543-eroprw00/logs/debug-internal.log b/wandb/run-20240823_162543-eroprw00/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..8c133d5d25fa0ac05d4e988a4e0c7ca1f73996b7 --- /dev/null +++ b/wandb/run-20240823_162543-eroprw00/logs/debug-internal.log @@ -0,0 +1,188 @@ +2024-08-23 16:25:43,204 INFO StreamThr :11284 [internal.py:wandb_internal():86] W&B internal server running at pid: 11284, started at: 2024-08-23 16:25:43.204013 +2024-08-23 16:25:43,206 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: status +2024-08-23 16:25:43,207 INFO WriterThread:11284 [datastore.py:open_for_write():87] open: /project/wandb/run-20240823_162543-eroprw00/run-eroprw00.wandb +2024-08-23 16:25:43,208 DEBUG SenderThread:11284 [sender.py:send():382] send: header +2024-08-23 16:25:43,222 DEBUG SenderThread:11284 [sender.py:send():382] send: run +2024-08-23 16:25:43,662 INFO SenderThread:11284 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240823_162543-eroprw00/files +2024-08-23 16:25:43,662 INFO SenderThread:11284 [sender.py:_start_run_threads():1136] run started: eroprw00 with start time 1724397943.202675 +2024-08-23 16:25:43,667 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: check_version +2024-08-23 16:25:43,668 DEBUG SenderThread:11284 [sender.py:send_request():409] send_request: check_version +2024-08-23 16:25:43,739 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: run_start +2024-08-23 16:25:43,746 DEBUG HandlerThread:11284 [system_info.py:__init__():27] System info init +2024-08-23 16:25:43,746 DEBUG HandlerThread:11284 [system_info.py:__init__():42] System info init done +2024-08-23 16:25:43,746 INFO HandlerThread:11284 [system_monitor.py:start():194] Starting system monitor +2024-08-23 16:25:43,746 INFO SystemMonitor:11284 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-23 16:25:43,746 INFO HandlerThread:11284 [system_monitor.py:probe():214] Collecting system info +2024-08-23 16:25:43,746 INFO SystemMonitor:11284 [interfaces.py:start():190] Started cpu monitoring +2024-08-23 16:25:43,747 INFO SystemMonitor:11284 [interfaces.py:start():190] Started disk monitoring +2024-08-23 16:25:43,747 INFO SystemMonitor:11284 [interfaces.py:start():190] Started gpu monitoring +2024-08-23 16:25:43,748 INFO SystemMonitor:11284 [interfaces.py:start():190] Started memory monitoring +2024-08-23 16:25:43,749 INFO SystemMonitor:11284 [interfaces.py:start():190] Started network monitoring +2024-08-23 16:25:43,758 DEBUG HandlerThread:11284 [system_info.py:probe():151] Probing system +2024-08-23 16:25:43,760 DEBUG HandlerThread:11284 [system_info.py:_probe_git():136] Probing git +2024-08-23 16:25:43,773 DEBUG HandlerThread:11284 [system_info.py:_probe_git():144] Probing git done +2024-08-23 16:25:43,773 DEBUG HandlerThread:11284 [system_info.py:probe():199] Probing system done +2024-08-23 16:25:43,773 DEBUG HandlerThread:11284 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-23T07:25:43.758914', 'startedAt': '2024-08-23T07:25:43.187250', 'docker': None, 'cuda': None, 'args': ('--seq-length', '4096', '--sliding-window-size', '131072', '--micro-batch-size', '5', '--valid_micro_batch_size', '1', '--global-batch-size', '640', '--train-iters', '7500', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--train-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document', '--valid-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '--test-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '7500', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'anyprecision', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '10', '--eval-interval', '10', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--save', '/work/llm_recipes/models/Qwen2-0.5b-0.2', '--load', '/work/llm_recipes/models/Qwen2-0.5b-0.2', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--upload-all-checkpoints-to-hf', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/Qwen2-0.5b-0.2', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial-0.2', '--wandb-name', 'Qwen2-0.5b-0.2_train_2024-08-23-16:25:30'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '887a2cc5d104c10264701f95cbbb0a6a116768d6'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.0389999999993, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.487831115722656}} +2024-08-23 16:25:43,773 INFO HandlerThread:11284 [system_monitor.py:probe():224] Finished collecting system info +2024-08-23 16:25:43,773 INFO HandlerThread:11284 [system_monitor.py:probe():227] Publishing system info +2024-08-23 16:25:43,774 INFO HandlerThread:11284 [system_monitor.py:probe():229] Finished publishing system info +2024-08-23 16:25:43,780 DEBUG SenderThread:11284 [sender.py:send():382] send: files +2024-08-23 16:25:43,780 INFO SenderThread:11284 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-23 16:25:43,791 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: python_packages +2024-08-23 16:25:43,791 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:25:43,792 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:25:43,792 DEBUG SenderThread:11284 [sender.py:send_request():409] send_request: python_packages +2024-08-23 16:25:43,794 DEBUG SenderThread:11284 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:25:44,074 DEBUG SenderThread:11284 [sender.py:send():382] send: telemetry +2024-08-23 16:25:44,478 INFO wandb-upload_0:11284 [upload_job.py:push():131] Uploaded file /tmp/tmpn8dztdufwandb/9bfyl56b-wandb-metadata.json +2024-08-23 16:25:44,664 INFO Thread-12 :11284 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_162543-eroprw00/files/wandb-metadata.json +2024-08-23 16:25:44,664 INFO Thread-12 :11284 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_162543-eroprw00/files/requirements.txt +2024-08-23 16:25:44,664 INFO Thread-12 :11284 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_162543-eroprw00/files/output.log +2024-08-23 16:25:46,664 INFO Thread-12 :11284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162543-eroprw00/files/output.log +2024-08-23 16:25:48,665 INFO Thread-12 :11284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162543-eroprw00/files/output.log +2024-08-23 16:25:49,201 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:25:50,667 INFO Thread-12 :11284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162543-eroprw00/files/output.log +2024-08-23 16:25:51,139 DEBUG SenderThread:11284 [sender.py:send():382] send: config +2024-08-23 16:25:51,140 DEBUG SenderThread:11284 [sender.py:send():382] send: config +2024-08-23 16:25:52,592 DEBUG SenderThread:11284 [sender.py:send():382] send: exit +2024-08-23 16:25:52,592 INFO SenderThread:11284 [sender.py:send_exit():589] handling exit code: 1 +2024-08-23 16:25:52,592 INFO SenderThread:11284 [sender.py:send_exit():591] handling runtime: 8 +2024-08-23 16:25:52,593 INFO SenderThread:11284 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:25:52,594 INFO SenderThread:11284 [sender.py:send_exit():597] send defer +2024-08-23 16:25:52,594 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:25:52,594 INFO HandlerThread:11284 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-23 16:25:52,594 DEBUG SenderThread:11284 [sender.py:send_request():409] send_request: defer +2024-08-23 16:25:52,594 INFO SenderThread:11284 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-23 16:25:52,594 INFO SenderThread:11284 [sender.py:transition_state():617] send defer: 1 +2024-08-23 16:25:52,594 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:25:52,594 INFO HandlerThread:11284 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-23 16:25:52,595 DEBUG SenderThread:11284 [sender.py:send_request():409] send_request: defer +2024-08-23 16:25:52,595 INFO SenderThread:11284 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-23 16:25:52,595 INFO SenderThread:11284 [sender.py:transition_state():617] send defer: 2 +2024-08-23 16:25:52,595 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:25:52,595 INFO HandlerThread:11284 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-23 16:25:52,595 INFO HandlerThread:11284 [system_monitor.py:finish():203] Stopping system monitor +2024-08-23 16:25:52,595 DEBUG SystemMonitor:11284 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-23 16:25:52,595 INFO HandlerThread:11284 [interfaces.py:finish():202] Joined cpu monitor +2024-08-23 16:25:52,595 DEBUG SystemMonitor:11284 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-23 16:25:52,595 INFO HandlerThread:11284 [interfaces.py:finish():202] Joined disk monitor +2024-08-23 16:25:52,596 DEBUG SystemMonitor:11284 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-23 16:25:52,629 INFO HandlerThread:11284 [interfaces.py:finish():202] Joined gpu monitor +2024-08-23 16:25:52,629 INFO HandlerThread:11284 [interfaces.py:finish():202] Joined memory monitor +2024-08-23 16:25:52,629 INFO HandlerThread:11284 [interfaces.py:finish():202] Joined network monitor +2024-08-23 16:25:52,629 DEBUG SenderThread:11284 [sender.py:send_request():409] send_request: defer +2024-08-23 16:25:52,629 INFO SenderThread:11284 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-23 16:25:52,629 INFO SenderThread:11284 [sender.py:transition_state():617] send defer: 3 +2024-08-23 16:25:52,629 DEBUG SenderThread:11284 [sender.py:send():382] send: stats +2024-08-23 16:25:52,629 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:25:52,630 INFO HandlerThread:11284 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-23 16:25:52,630 DEBUG SenderThread:11284 [sender.py:send_request():409] send_request: defer +2024-08-23 16:25:52,630 INFO SenderThread:11284 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-23 16:25:52,630 INFO SenderThread:11284 [sender.py:transition_state():617] send defer: 4 +2024-08-23 16:25:52,630 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:25:52,630 INFO HandlerThread:11284 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-23 16:25:52,630 DEBUG SenderThread:11284 [sender.py:send_request():409] send_request: defer +2024-08-23 16:25:52,630 INFO SenderThread:11284 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-23 16:25:52,630 INFO SenderThread:11284 [sender.py:transition_state():617] send defer: 5 +2024-08-23 16:25:52,630 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:25:52,631 INFO HandlerThread:11284 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-23 16:25:52,631 DEBUG SenderThread:11284 [sender.py:send():382] send: summary +2024-08-23 16:25:52,632 INFO SenderThread:11284 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:25:52,632 DEBUG SenderThread:11284 [sender.py:send_request():409] send_request: defer +2024-08-23 16:25:52,632 INFO SenderThread:11284 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-23 16:25:52,632 INFO SenderThread:11284 [sender.py:transition_state():617] send defer: 6 +2024-08-23 16:25:52,632 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:25:52,632 INFO HandlerThread:11284 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-23 16:25:52,632 DEBUG SenderThread:11284 [sender.py:send_request():409] send_request: defer +2024-08-23 16:25:52,632 INFO SenderThread:11284 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-23 16:25:52,635 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:25:52,668 INFO Thread-12 :11284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162543-eroprw00/files/output.log +2024-08-23 16:25:52,668 INFO Thread-12 :11284 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_162543-eroprw00/files/wandb-summary.json +2024-08-23 16:25:52,831 INFO SenderThread:11284 [sender.py:transition_state():617] send defer: 7 +2024-08-23 16:25:52,831 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:25:52,831 INFO HandlerThread:11284 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-23 16:25:52,831 DEBUG SenderThread:11284 [sender.py:send_request():409] send_request: defer +2024-08-23 16:25:52,831 INFO SenderThread:11284 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-23 16:25:53,592 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:25:53,669 INFO Thread-12 :11284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162543-eroprw00/files/config.yaml +2024-08-23 16:25:54,373 INFO SenderThread:11284 [sender.py:transition_state():617] send defer: 8 +2024-08-23 16:25:54,374 DEBUG SenderThread:11284 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:25:54,374 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:25:54,374 INFO HandlerThread:11284 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-23 16:25:54,374 DEBUG SenderThread:11284 [sender.py:send_request():409] send_request: defer +2024-08-23 16:25:54,374 INFO SenderThread:11284 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-23 16:25:54,374 INFO SenderThread:11284 [job_builder.py:build():296] Attempting to build job artifact +2024-08-23 16:25:54,375 INFO SenderThread:11284 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-23 16:25:54,389 INFO SenderThread:11284 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-23 16:25:54,398 INFO SenderThread:11284 [sender.py:transition_state():617] send defer: 9 +2024-08-23 16:25:54,398 DEBUG SenderThread:11284 [sender.py:send():382] send: artifact +2024-08-23 16:25:54,398 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:25:54,399 INFO HandlerThread:11284 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-23 16:25:54,593 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:25:54,670 INFO Thread-12 :11284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162543-eroprw00/files/output.log +2024-08-23 16:25:55,372 INFO SenderThread:11284 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTE2MTk5MDU4OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjQxNjQ1ODQ1MA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTE2MTk5MDU4OQ==', 'versionIndex': 2}}} +2024-08-23 16:25:55,372 DEBUG SenderThread:11284 [sender.py:send_request():409] send_request: defer +2024-08-23 16:25:55,372 INFO SenderThread:11284 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-23 16:25:55,372 INFO SenderThread:11284 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-23 16:25:55,671 INFO SenderThread:11284 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240823_162543-eroprw00/files +2024-08-23 16:25:55,671 INFO SenderThread:11284 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_162543-eroprw00/files/requirements.txt requirements.txt +2024-08-23 16:25:55,671 INFO SenderThread:11284 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_162543-eroprw00/files/config.yaml config.yaml +2024-08-23 16:25:55,673 INFO SenderThread:11284 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_162543-eroprw00/files/wandb-metadata.json wandb-metadata.json +2024-08-23 16:25:55,673 INFO SenderThread:11284 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_162543-eroprw00/files/wandb-summary.json wandb-summary.json +2024-08-23 16:25:55,674 INFO SenderThread:11284 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_162543-eroprw00/files/output.log output.log +2024-08-23 16:25:55,676 INFO SenderThread:11284 [sender.py:transition_state():617] send defer: 10 +2024-08-23 16:25:55,676 DEBUG SenderThread:11284 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:25:55,676 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:25:55,677 INFO HandlerThread:11284 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-23 16:25:55,678 DEBUG SenderThread:11284 [sender.py:send_request():409] send_request: defer +2024-08-23 16:25:55,678 INFO SenderThread:11284 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-23 16:25:55,678 INFO SenderThread:11284 [file_pusher.py:finish():172] shutting down file pusher +2024-08-23 16:25:56,071 INFO wandb-upload_0:11284 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_162543-eroprw00/files/requirements.txt +2024-08-23 16:25:56,117 INFO wandb-upload_1:11284 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_162543-eroprw00/files/config.yaml +2024-08-23 16:25:56,151 INFO wandb-upload_3:11284 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_162543-eroprw00/files/output.log +2024-08-23 16:25:56,152 INFO wandb-upload_2:11284 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_162543-eroprw00/files/wandb-summary.json +2024-08-23 16:25:56,353 INFO Thread-11 (_thread_body):11284 [sender.py:transition_state():617] send defer: 11 +2024-08-23 16:25:56,353 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:25:56,353 INFO HandlerThread:11284 [handler.py:handle_request_defer():172] handle defer: 11 +2024-08-23 16:25:56,353 DEBUG SenderThread:11284 [sender.py:send_request():409] send_request: defer +2024-08-23 16:25:56,353 INFO SenderThread:11284 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-08-23 16:25:56,353 INFO SenderThread:11284 [file_pusher.py:join():178] waiting for file pusher +2024-08-23 16:25:56,353 INFO SenderThread:11284 [sender.py:transition_state():617] send defer: 12 +2024-08-23 16:25:56,354 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:25:56,354 INFO HandlerThread:11284 [handler.py:handle_request_defer():172] handle defer: 12 +2024-08-23 16:25:56,354 DEBUG SenderThread:11284 [sender.py:send_request():409] send_request: defer +2024-08-23 16:25:56,354 INFO SenderThread:11284 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-08-23 16:25:56,354 INFO SenderThread:11284 [file_stream.py:finish():595] file stream finish called +2024-08-23 16:25:56,522 INFO SenderThread:11284 [file_stream.py:finish():599] file stream finish is done +2024-08-23 16:25:56,522 INFO SenderThread:11284 [sender.py:transition_state():617] send defer: 13 +2024-08-23 16:25:56,523 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:25:56,523 INFO HandlerThread:11284 [handler.py:handle_request_defer():172] handle defer: 13 +2024-08-23 16:25:56,523 DEBUG SenderThread:11284 [sender.py:send_request():409] send_request: defer +2024-08-23 16:25:56,523 INFO SenderThread:11284 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-08-23 16:25:56,523 INFO SenderThread:11284 [sender.py:transition_state():617] send defer: 14 +2024-08-23 16:25:56,523 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:25:56,523 DEBUG SenderThread:11284 [sender.py:send():382] send: final +2024-08-23 16:25:56,523 INFO HandlerThread:11284 [handler.py:handle_request_defer():172] handle defer: 14 +2024-08-23 16:25:56,523 DEBUG SenderThread:11284 [sender.py:send():382] send: footer +2024-08-23 16:25:56,524 DEBUG SenderThread:11284 [sender.py:send_request():409] send_request: defer +2024-08-23 16:25:56,524 INFO SenderThread:11284 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-08-23 16:25:56,524 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:25:56,524 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:25:56,524 DEBUG SenderThread:11284 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:25:56,525 DEBUG SenderThread:11284 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:25:56,525 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: server_info +2024-08-23 16:25:56,525 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: get_summary +2024-08-23 16:25:56,525 DEBUG SenderThread:11284 [sender.py:send_request():409] send_request: server_info +2024-08-23 16:25:56,527 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: sampled_history +2024-08-23 16:25:56,527 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:25:56,527 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: job_info +2024-08-23 16:25:56,684 DEBUG SenderThread:11284 [sender.py:send_request():409] send_request: job_info +2024-08-23 16:25:56,684 INFO MainThread:11284 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-08-23 16:25:56,685 INFO MainThread:11284 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-08-23 16:25:56,685 INFO MainThread:11284 [wandb_run.py:_footer_sync_info():3825] logging synced files +2024-08-23 16:25:56,685 DEBUG HandlerThread:11284 [handler.py:handle_request():146] handle_request: shutdown +2024-08-23 16:25:56,685 INFO HandlerThread:11284 [handler.py:finish():869] shutting down handler +2024-08-23 16:25:57,528 INFO WriterThread:11284 [datastore.py:close():296] close: /project/wandb/run-20240823_162543-eroprw00/run-eroprw00.wandb +2024-08-23 16:25:57,685 INFO SenderThread:11284 [sender.py:finish():1572] shutting down sender +2024-08-23 16:25:57,685 INFO SenderThread:11284 [file_pusher.py:finish():172] shutting down file pusher +2024-08-23 16:25:57,685 INFO SenderThread:11284 [file_pusher.py:join():178] waiting for file pusher diff --git a/wandb/run-20240823_162543-eroprw00/logs/debug.log b/wandb/run-20240823_162543-eroprw00/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..91699756d728ffe6231dfc41e140d28987387460 --- /dev/null +++ b/wandb/run-20240823_162543-eroprw00/logs/debug.log @@ -0,0 +1,30 @@ +2024-08-23 16:25:43,196 INFO MainThread:11213 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-23 16:25:43,196 INFO MainThread:11213 [wandb_setup.py:_flush():76] Configure stats pid to 11213 +2024-08-23 16:25:43,196 INFO MainThread:11213 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-23 16:25:43,196 INFO MainThread:11213 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-23 16:25:43,196 INFO MainThread:11213 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train sample'} +2024-08-23 16:25:43,196 INFO MainThread:11213 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-23 16:25:43,196 INFO MainThread:11213 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-23 16:25:43,196 INFO MainThread:11213 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240823_162543-eroprw00/logs/debug.log +2024-08-23 16:25:43,196 INFO MainThread:11213 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240823_162543-eroprw00/logs/debug-internal.log +2024-08-23 16:25:43,196 INFO MainThread:11213 [wandb_init.py:init():566] calling init triggers +2024-08-23 16:25:43,197 INFO MainThread:11213 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document'], 'valid_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document'], 'test_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 4096, 'num_workers': 2, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'Qwen2-0.5b-0.2_train_2024-08-23-16:25:30', 'wandb_project': 'llm_tutorial-0.2', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/Qwen2-0.5b-0.2', 'save': '/work/llm_recipes/models/Qwen2-0.5b-0.2', 'base_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 10, 'save_interval': 10, 'eval_iters': 10, 'optimizer': 'anyprecision', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 7500, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 7500, 'train_samples': None, 'global_batch_size': 640, 'micro_batch_size': 5, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 131072, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/Qwen2-0.5b-0.2', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': True, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'valid_micro_batch_size': 1, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 151680, 'gradient_accumulation_steps': 128} +2024-08-23 16:25:43,197 INFO MainThread:11213 [wandb_init.py:init():616] starting backend +2024-08-23 16:25:43,197 INFO MainThread:11213 [wandb_init.py:init():620] setting up manager +2024-08-23 16:25:43,201 INFO MainThread:11213 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-23 16:25:43,202 INFO MainThread:11213 [wandb_init.py:init():628] backend started and connected +2024-08-23 16:25:43,207 INFO MainThread:11213 [wandb_init.py:init():720] updated telemetry +2024-08-23 16:25:43,218 INFO MainThread:11213 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-23 16:25:43,667 INFO MainThread:11213 [wandb_run.py:_on_init():2262] communicating current version +2024-08-23 16:25:43,692 INFO MainThread:11213 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.7 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-23 16:25:43,692 INFO MainThread:11213 [wandb_init.py:init():804] starting run threads in backend +2024-08-23 16:25:43,790 INFO MainThread:11213 [wandb_run.py:_console_start():2241] atexit reg +2024-08-23 16:25:43,790 INFO MainThread:11213 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-23 16:25:43,790 INFO MainThread:11213 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-23 16:25:43,790 INFO MainThread:11213 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-23 16:25:43,791 INFO MainThread:11213 [wandb_init.py:init():847] run started, returning control to user process +2024-08-23 16:25:51,139 INFO MainThread:11213 [wandb_run.py:_config_callback():1343] config_cb None None {'model_architecture': 'Qwen2ForCausalLM', 'activation_function': 'silu', 'hidden_size': 896, 'model_type': 'qwen2', 'max_position_embeddings': 4096, 'num_attention_heads': 14, 'num_hidden_layers': 24} +2024-08-23 16:25:51,139 INFO MainThread:11213 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1} +2024-08-23 16:25:57,685 WARNING MsgRouterThr:11213 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20240823_162543-eroprw00/run-eroprw00.wandb b/wandb/run-20240823_162543-eroprw00/run-eroprw00.wandb new file mode 100644 index 0000000000000000000000000000000000000000..b4f470914e778dd8dfa0c59a3f62f7c9985cf30e Binary files /dev/null and b/wandb/run-20240823_162543-eroprw00/run-eroprw00.wandb differ diff --git a/wandb/run-20240823_162802-bqibep1u/files/config.yaml b/wandb/run-20240823_162802-bqibep1u/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6df4b18b1f0f79e1dbbe86b6d852f505280a3b15 --- /dev/null +++ b/wandb/run-20240823_162802-bqibep1u/files/config.yaml @@ -0,0 +1,342 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: FULL_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '1754785366' + - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document + - '28623823675' + - /project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document +valid_data_path: + desc: null + value: + - '1754785366' + - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document +test_data_path: + desc: null + value: + - '1754785366' + - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 4096 +num_workers: + desc: null + value: 2 +tokenizer_type: + desc: null + value: HFPreTrainedTokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: Qwen2-0.5b-0.2_train_2024-08-23-16:27:49 +wandb_project: + desc: null + value: llm_tutorial-0.2 +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/Qwen2-0.5b-0.2 +save: + desc: null + value: /work/llm_recipes/models/Qwen2-0.5b-0.2 +base_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 10 +save_interval: + desc: null + value: 10 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: anyprecision +lr: + desc: null + value: 2.0e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 7500 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 1.0e-06 +train_iters: + desc: null + value: 7500 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 320 +micro_batch_size: + desc: null + value: 5 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 131072 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-06 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/Qwen2-0.5b-0.2 +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: true +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +valid_micro_batch_size: + desc: null + value: 1 +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 151680 +gradient_accumulation_steps: + desc: null + value: 64 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1724398082.664982 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +model_architecture: + desc: null + value: Qwen2ForCausalLM +activation_function: + desc: null + value: silu +hidden_size: + desc: null + value: 896 +model_type: + desc: null + value: qwen2 +max_position_embeddings: + desc: null + value: 4096 +num_attention_heads: + desc: null + value: 14 +num_hidden_layers: + desc: null + value: 24 diff --git a/wandb/run-20240823_162802-bqibep1u/files/output.log b/wandb/run-20240823_162802-bqibep1u/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..1b13fd9e397a672c40cbce92c97df00f1eac1833 --- /dev/null +++ b/wandb/run-20240823_162802-bqibep1u/files/output.log @@ -0,0 +1,116 @@ +Created Hugging Face repository with ID koichi12/Qwen2-0.5b-0.2. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +Loading model state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/model.pt +Loaded model state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/model.pt +--> Model /share/pretrained_lm/Qwen/Qwen2-0.5B +--> /share/pretrained_lm/Qwen/Qwen2-0.5B has 494.032768 Million params +BFloat16 enabled for mixed precision - using bfSixteen policy +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1. + warnings.warn( +Let split = None +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 2400000 + validation: 2403200 + test: 3200 +> building train, validation, and test datasets for GPT ... +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +> finished creating GPT datasets ... +Loading optimizer state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/optimizer.pt +[rank0]:[2024-08-23 16:28:10,071] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _flatten_optim_state_dict() profiling: defaultdict(, {}) +Loaded optimizer state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/optimizer.pt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): Qwen2ForCausalLM( + (model): Qwen2Model( + (embed_tokens): Embedding(151936, 896) + (layers): ModuleList( + (0-23): 24 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): Qwen2DecoderLayer( + (self_attn): Qwen2FlashAttention2( + (q_proj): Linear(in_features=896, out_features=896, bias=True) + (k_proj): Linear(in_features=896, out_features=128, bias=True) + (v_proj): Linear(in_features=896, out_features=128, bias=True) + (o_proj): Linear(in_features=896, out_features=896, bias=False) + (rotary_emb): Qwen2RotaryEmbedding() + ) + (mlp): Qwen2MLP( + (gate_proj): Linear(in_features=896, out_features=4864, bias=False) + (up_proj): Linear(in_features=896, out_features=4864, bias=False) + (down_proj): Linear(in_features=4864, out_features=896, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): Qwen2RMSNorm() + (post_attention_layernorm): Qwen2RMSNorm() + ) + ) + ) + ) + (norm): Qwen2RMSNorm() + ) + (lm_head): Linear(in_features=896, out_features=151936, bias=False) + ) +) +model config: Qwen2Config { + "_name_or_path": "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "label_smoothing": 0.0, + "max_position_embeddings": 4096, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-06, + "rope_theta": 1000000.0, + "sliding_window": 131072, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.43.3", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} +Traceback (most recent call last): + File "/project/examples/finetuning.py", line 13, in + main() + File "/project/src/llama_recipes/finetuning.py", line 282, in main + train( + File "/project/src/llama_recipes/utils/train_utils.py", line 110, in train + loss: torch.Tensor = model(**batch).loss + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl + return forward_call(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py", line 849, in forward + output = self._fsdp_wrapped_module(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl + return forward_call(*args, **kwargs) + File "/project/lib/transformers/src/transformers/models/qwen2/modeling_qwen2.py", line 1082, in forward + loss = loss_fct(shift_logits, shift_labels) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl + return forward_call(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/loss.py", line 1179, in forward + return F.cross_entropy(input, target, weight=self.weight, + File "/usr/local/lib/python3.10/dist-packages/torch/nn/functional.py", line 3086, in cross_entropy + return torch._C._nn.cross_entropy_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index, label_smoothing) +torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 11.59 GiB. GPU 0 has a total capacity of 39.39 GiB of which 11.28 GiB is free. Including non-PyTorch memory, this process has 28.11 GiB memory in use. Of the allocated memory 26.94 GiB is allocated by PyTorch, and 363.22 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) \ No newline at end of file diff --git a/wandb/run-20240823_162802-bqibep1u/files/requirements.txt b/wandb/run-20240823_162802-bqibep1u/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..34a2774e444bdc395654ccf8ce6da6833c7bc1ee --- /dev/null +++ b/wandb/run-20240823_162802-bqibep1u/files/requirements.txt @@ -0,0 +1,375 @@ +absl-py==2.1.0 +accelerate==0.23.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +astroid==3.2.4 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bert-score==0.3.13 +bleach==6.1.0 +blis==0.7.11 +build==1.2.1 +cachecontrol==0.14.0 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +cleo==2.1.0 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cramjam==2.8.3 +crashtest==0.4.1 +cryptography==43.0.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +dataclasses-json==0.6.7 +dataproperty==1.0.1 +datasets==2.20.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +dill==0.3.8 +distlib==0.3.8 +distributed==2023.11.0 +distro==1.9.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +dulwich==0.21.7 +einops==0.7.0 +emoji==2.12.1 +entmax==1.3 +evaluate==0.4.2 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastparquet==2023.10.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +fugashi==1.3.2 +fuzzywuzzy==0.18.0 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +greenlet==3.0.3 +grpcio==1.60.1 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.24.5 +hydra-core==1.3.2 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +installer==0.7.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +isort==5.13.2 +jaraco.classes==3.4.0 +jedi==0.19.1 +jeepney==0.8.0 +jinja2==3.1.3 +jiter==0.5.0 +joblib==1.3.2 +json5==0.9.14 +jsonargparse==3.13.1 +jsonlines==4.0.0 +jsonnet==0.19.1 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +keyring==24.3.1 +kiwisolver==1.4.5 +langchain-community==0.2.12 +langchain-core==0.2.31 +langchain-huggingface==0.0.2 +langchain-openai==0.1.21 +langchain-text-splitters==0.2.2 +langchain==0.2.13 +langcodes==3.3.0 +langsmith==0.1.99 +lazy-loader==0.3 +levenshtein==0.25.1 +librosa==0.10.1 +lightning-utilities==0.11.6 +llm-jp-eval==1.4.0 +llvmlite==0.40.1 +lm-eval==0.3.0 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +marshmallow==3.21.3 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mbstrdecoder==1.1.3 +mccabe==0.7.0 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +mojimoji==0.0.13 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +multiprocess==0.70.16 +murmurhash==1.0.10 +mypy-extensions==1.0.0 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +neologdn==0.5.3 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numexpr==2.10.1 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +omegaconf==2.3.0 +onnx==1.15.0rc2 +openai==1.40.6 +opencv==4.7.0 +optree==0.10.0 +orjson==3.10.7 +packaging==23.2 +pandas==2.2.2 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +pathvalidate==3.2.0 +peft==0.5.0 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +pkginfo==1.11.1 +plac==1.4.3 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +poetry-core==1.9.0 +poetry-plugin-export==1.8.0 +poetry==1.8.3 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow-hotfix==0.6 +pyarrow==15.0.2 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycountry==24.6.1 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pylint==3.2.6 +pynvml==11.4.1 +pyparsing==3.1.1 +pyproject-hooks==1.1.0 +pytablewriter==1.2.0 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +python-levenshtein==0.25.1 +pytorch-lightning==2.4.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapidfuzz==3.9.6 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests-toolbelt==1.0.0 +requests==2.32.3 +rhoknp==1.7.0 +rich==13.7.0 +rmm==23.12.0 +rouge-score==0.1.2 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.2 +safetensors==0.4.3 +scikit-learn==1.5.1 +scipy==1.12.0 +secretstorage==3.3.3 +send2trash==1.8.2 +sentence-transformers==3.0.1 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +shellingham==1.5.4 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +sqlalchemy==2.0.32 +sqlitedict==2.1.0 +srsly==2.4.8 +stack-data==0.6.3 +sumeval==0.2.2 +sympy==1.12 +tabledata==1.3.3 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tcolorpy==0.1.6 +tenacity==8.5.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +text-generation==0.7.0 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tiktoken==0.7.0 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +tomlkit==0.13.2 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchmetrics==0.10.3 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm-multiprocess==0.0.11 +tqdm==4.66.5 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +trove-classifiers==2024.7.2 +typepy==1.3.2 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.12.2 +typing-inspect==0.9.0 +tzdata==2024.1 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +unbabel-comet==2.2.2 +unidic-lite==1.0.8 +urllib3==1.26.18 +virtualenv==20.26.3 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +word2number==1.1 +xdoctest==1.0.2 +xgboost==1.7.6 +xmltodict==0.13.0 +xxhash==3.4.1 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 +zstandard==0.23.0 \ No newline at end of file diff --git a/wandb/run-20240823_162802-bqibep1u/files/wandb-metadata.json b/wandb/run-20240823_162802-bqibep1u/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..ca65dafd26eef583317603e0d01cd0e38fb1f88d --- /dev/null +++ b/wandb/run-20240823_162802-bqibep1u/files/wandb-metadata.json @@ -0,0 +1,220 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-23T07:28:04.137804", + "startedAt": "2024-08-23T07:28:02.652941", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "4096", + "--sliding-window-size", + "131072", + "--micro-batch-size", + "5", + "--valid_micro_batch_size", + "1", + "--global-batch-size", + "320", + "--train-iters", + "7500", + "--tokenizer-type", + "HFPreTrainedTokenizer", + "--tokenizer-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--train-data-path", + "1754785366", + "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document", + "28623823675", + "/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document", + "--valid-data-path", + "1754785366", + "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document", + "--test-data-path", + "1754785366", + "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document", + "--lr", + "2e-5", + "--min-lr", + "1e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "7500", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "anyprecision", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-6", + "--save-interval", + "10", + "--eval-interval", + "10", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--save", + "/work/llm_recipes/models/Qwen2-0.5b-0.2", + "--load", + "/work/llm_recipes/models/Qwen2-0.5b-0.2", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "FULL_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--upload-all-checkpoints-to-hf", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/Qwen2-0.5b-0.2", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "llm_tutorial-0.2", + "--wandb-name", + "Qwen2-0.5b-0.2_train_2024-08-23-16:27:49" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "887a2cc5d104c10264701f95cbbb0a6a116768d6" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.0389999999993, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.487831115722656 + } +} diff --git a/wandb/run-20240823_162802-bqibep1u/files/wandb-summary.json b/wandb/run-20240823_162802-bqibep1u/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..13ed209ce59c23ca7857cb54916c6cd691e4764e --- /dev/null +++ b/wandb/run-20240823_162802-bqibep1u/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 7}} \ No newline at end of file diff --git a/wandb/run-20240823_162802-bqibep1u/logs/debug-internal.log b/wandb/run-20240823_162802-bqibep1u/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b70e2e1809e30b012b03a57bad0b7a57fa074dd0 --- /dev/null +++ b/wandb/run-20240823_162802-bqibep1u/logs/debug-internal.log @@ -0,0 +1,187 @@ +2024-08-23 16:28:02,667 INFO StreamThr :11596 [internal.py:wandb_internal():86] W&B internal server running at pid: 11596, started at: 2024-08-23 16:28:02.666245 +2024-08-23 16:28:02,668 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: status +2024-08-23 16:28:02,670 INFO WriterThread:11596 [datastore.py:open_for_write():87] open: /project/wandb/run-20240823_162802-bqibep1u/run-bqibep1u.wandb +2024-08-23 16:28:02,670 DEBUG SenderThread:11596 [sender.py:send():382] send: header +2024-08-23 16:28:02,684 DEBUG SenderThread:11596 [sender.py:send():382] send: run +2024-08-23 16:28:04,047 INFO SenderThread:11596 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240823_162802-bqibep1u/files +2024-08-23 16:28:04,047 INFO SenderThread:11596 [sender.py:_start_run_threads():1136] run started: bqibep1u with start time 1724398082.664982 +2024-08-23 16:28:04,052 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: check_version +2024-08-23 16:28:04,052 DEBUG SenderThread:11596 [sender.py:send_request():409] send_request: check_version +2024-08-23 16:28:04,119 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: run_start +2024-08-23 16:28:04,125 DEBUG HandlerThread:11596 [system_info.py:__init__():27] System info init +2024-08-23 16:28:04,126 DEBUG HandlerThread:11596 [system_info.py:__init__():42] System info init done +2024-08-23 16:28:04,126 INFO HandlerThread:11596 [system_monitor.py:start():194] Starting system monitor +2024-08-23 16:28:04,126 INFO SystemMonitor:11596 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-23 16:28:04,126 INFO HandlerThread:11596 [system_monitor.py:probe():214] Collecting system info +2024-08-23 16:28:04,126 INFO SystemMonitor:11596 [interfaces.py:start():190] Started cpu monitoring +2024-08-23 16:28:04,127 INFO SystemMonitor:11596 [interfaces.py:start():190] Started disk monitoring +2024-08-23 16:28:04,128 INFO SystemMonitor:11596 [interfaces.py:start():190] Started gpu monitoring +2024-08-23 16:28:04,129 INFO SystemMonitor:11596 [interfaces.py:start():190] Started memory monitoring +2024-08-23 16:28:04,129 INFO SystemMonitor:11596 [interfaces.py:start():190] Started network monitoring +2024-08-23 16:28:04,137 DEBUG HandlerThread:11596 [system_info.py:probe():151] Probing system +2024-08-23 16:28:04,139 DEBUG HandlerThread:11596 [system_info.py:_probe_git():136] Probing git +2024-08-23 16:28:04,152 DEBUG HandlerThread:11596 [system_info.py:_probe_git():144] Probing git done +2024-08-23 16:28:04,152 DEBUG HandlerThread:11596 [system_info.py:probe():199] Probing system done +2024-08-23 16:28:04,152 DEBUG HandlerThread:11596 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-23T07:28:04.137804', 'startedAt': '2024-08-23T07:28:02.652941', 'docker': None, 'cuda': None, 'args': ('--seq-length', '4096', '--sliding-window-size', '131072', '--micro-batch-size', '5', '--valid_micro_batch_size', '1', '--global-batch-size', '320', '--train-iters', '7500', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--train-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document', '--valid-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '--test-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '7500', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'anyprecision', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '10', '--eval-interval', '10', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--save', '/work/llm_recipes/models/Qwen2-0.5b-0.2', '--load', '/work/llm_recipes/models/Qwen2-0.5b-0.2', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--upload-all-checkpoints-to-hf', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/Qwen2-0.5b-0.2', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial-0.2', '--wandb-name', 'Qwen2-0.5b-0.2_train_2024-08-23-16:27:49'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '887a2cc5d104c10264701f95cbbb0a6a116768d6'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.0389999999993, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.487831115722656}} +2024-08-23 16:28:04,152 INFO HandlerThread:11596 [system_monitor.py:probe():224] Finished collecting system info +2024-08-23 16:28:04,152 INFO HandlerThread:11596 [system_monitor.py:probe():227] Publishing system info +2024-08-23 16:28:04,154 INFO HandlerThread:11596 [system_monitor.py:probe():229] Finished publishing system info +2024-08-23 16:28:04,159 DEBUG SenderThread:11596 [sender.py:send():382] send: files +2024-08-23 16:28:04,159 INFO SenderThread:11596 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-23 16:28:04,171 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: python_packages +2024-08-23 16:28:04,171 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:28:04,171 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:28:04,171 DEBUG SenderThread:11596 [sender.py:send_request():409] send_request: python_packages +2024-08-23 16:28:04,173 DEBUG SenderThread:11596 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:28:04,372 DEBUG SenderThread:11596 [sender.py:send():382] send: telemetry +2024-08-23 16:28:04,810 INFO wandb-upload_0:11596 [upload_job.py:push():131] Uploaded file /tmp/tmpghl9unwnwandb/s571np3i-wandb-metadata.json +2024-08-23 16:28:05,049 INFO Thread-12 :11596 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_162802-bqibep1u/files/output.log +2024-08-23 16:28:05,049 INFO Thread-12 :11596 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_162802-bqibep1u/files/requirements.txt +2024-08-23 16:28:05,049 INFO Thread-12 :11596 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_162802-bqibep1u/files/wandb-metadata.json +2024-08-23 16:28:07,049 INFO Thread-12 :11596 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162802-bqibep1u/files/output.log +2024-08-23 16:28:07,804 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:28:09,050 INFO Thread-12 :11596 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162802-bqibep1u/files/output.log +2024-08-23 16:28:10,362 DEBUG SenderThread:11596 [sender.py:send():382] send: config +2024-08-23 16:28:10,362 DEBUG SenderThread:11596 [sender.py:send():382] send: config +2024-08-23 16:28:11,051 INFO Thread-12 :11596 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162802-bqibep1u/files/output.log +2024-08-23 16:28:11,858 DEBUG SenderThread:11596 [sender.py:send():382] send: exit +2024-08-23 16:28:11,858 INFO SenderThread:11596 [sender.py:send_exit():589] handling exit code: 1 +2024-08-23 16:28:11,858 INFO SenderThread:11596 [sender.py:send_exit():591] handling runtime: 7 +2024-08-23 16:28:11,859 INFO SenderThread:11596 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:28:11,859 INFO SenderThread:11596 [sender.py:send_exit():597] send defer +2024-08-23 16:28:11,860 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:28:11,860 INFO HandlerThread:11596 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-23 16:28:11,860 DEBUG SenderThread:11596 [sender.py:send_request():409] send_request: defer +2024-08-23 16:28:11,860 INFO SenderThread:11596 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-23 16:28:11,860 INFO SenderThread:11596 [sender.py:transition_state():617] send defer: 1 +2024-08-23 16:28:11,860 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:28:11,860 INFO HandlerThread:11596 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-23 16:28:11,860 DEBUG SenderThread:11596 [sender.py:send_request():409] send_request: defer +2024-08-23 16:28:11,860 INFO SenderThread:11596 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-23 16:28:11,860 INFO SenderThread:11596 [sender.py:transition_state():617] send defer: 2 +2024-08-23 16:28:11,860 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:28:11,860 INFO HandlerThread:11596 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-23 16:28:11,860 INFO HandlerThread:11596 [system_monitor.py:finish():203] Stopping system monitor +2024-08-23 16:28:11,861 DEBUG SystemMonitor:11596 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-23 16:28:11,861 INFO HandlerThread:11596 [interfaces.py:finish():202] Joined cpu monitor +2024-08-23 16:28:11,861 DEBUG SystemMonitor:11596 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-23 16:28:11,861 INFO HandlerThread:11596 [interfaces.py:finish():202] Joined disk monitor +2024-08-23 16:28:11,861 DEBUG SystemMonitor:11596 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-23 16:28:11,894 INFO HandlerThread:11596 [interfaces.py:finish():202] Joined gpu monitor +2024-08-23 16:28:11,894 INFO HandlerThread:11596 [interfaces.py:finish():202] Joined memory monitor +2024-08-23 16:28:11,894 INFO HandlerThread:11596 [interfaces.py:finish():202] Joined network monitor +2024-08-23 16:28:11,895 DEBUG SenderThread:11596 [sender.py:send_request():409] send_request: defer +2024-08-23 16:28:11,895 INFO SenderThread:11596 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-23 16:28:11,895 INFO SenderThread:11596 [sender.py:transition_state():617] send defer: 3 +2024-08-23 16:28:11,895 DEBUG SenderThread:11596 [sender.py:send():382] send: stats +2024-08-23 16:28:11,895 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:28:11,896 INFO HandlerThread:11596 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-23 16:28:11,896 DEBUG SenderThread:11596 [sender.py:send_request():409] send_request: defer +2024-08-23 16:28:11,896 INFO SenderThread:11596 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-23 16:28:11,896 INFO SenderThread:11596 [sender.py:transition_state():617] send defer: 4 +2024-08-23 16:28:11,896 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:28:11,896 INFO HandlerThread:11596 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-23 16:28:11,896 DEBUG SenderThread:11596 [sender.py:send_request():409] send_request: defer +2024-08-23 16:28:11,896 INFO SenderThread:11596 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-23 16:28:11,896 INFO SenderThread:11596 [sender.py:transition_state():617] send defer: 5 +2024-08-23 16:28:11,896 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:28:11,896 INFO HandlerThread:11596 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-23 16:28:11,896 DEBUG SenderThread:11596 [sender.py:send():382] send: summary +2024-08-23 16:28:11,897 INFO SenderThread:11596 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:28:11,898 DEBUG SenderThread:11596 [sender.py:send_request():409] send_request: defer +2024-08-23 16:28:11,898 INFO SenderThread:11596 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-23 16:28:11,898 INFO SenderThread:11596 [sender.py:transition_state():617] send defer: 6 +2024-08-23 16:28:11,898 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:28:11,898 INFO HandlerThread:11596 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-23 16:28:11,898 DEBUG SenderThread:11596 [sender.py:send_request():409] send_request: defer +2024-08-23 16:28:11,898 INFO SenderThread:11596 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-23 16:28:11,901 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:28:12,052 INFO Thread-12 :11596 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_162802-bqibep1u/files/wandb-summary.json +2024-08-23 16:28:12,101 INFO SenderThread:11596 [sender.py:transition_state():617] send defer: 7 +2024-08-23 16:28:12,101 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:28:12,101 INFO HandlerThread:11596 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-23 16:28:12,101 DEBUG SenderThread:11596 [sender.py:send_request():409] send_request: defer +2024-08-23 16:28:12,101 INFO SenderThread:11596 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-23 16:28:12,748 INFO SenderThread:11596 [sender.py:transition_state():617] send defer: 8 +2024-08-23 16:28:12,748 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:28:12,748 INFO HandlerThread:11596 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-23 16:28:12,749 DEBUG SenderThread:11596 [sender.py:send_request():409] send_request: defer +2024-08-23 16:28:12,749 INFO SenderThread:11596 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-23 16:28:12,749 INFO SenderThread:11596 [job_builder.py:build():296] Attempting to build job artifact +2024-08-23 16:28:12,750 INFO SenderThread:11596 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-23 16:28:12,764 INFO SenderThread:11596 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-23 16:28:12,772 INFO SenderThread:11596 [sender.py:transition_state():617] send defer: 9 +2024-08-23 16:28:12,773 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:28:12,773 DEBUG SenderThread:11596 [sender.py:send():382] send: artifact +2024-08-23 16:28:12,773 INFO HandlerThread:11596 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-23 16:28:12,858 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:28:13,053 INFO Thread-12 :11596 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162802-bqibep1u/files/output.log +2024-08-23 16:28:13,054 INFO Thread-12 :11596 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162802-bqibep1u/files/config.yaml +2024-08-23 16:28:13,612 INFO SenderThread:11596 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTE2MTk5MDU4OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjQxNjQ1ODQ1MA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTE2MTk5MDU4OQ==', 'versionIndex': 2}}} +2024-08-23 16:28:13,612 DEBUG SenderThread:11596 [sender.py:send_request():409] send_request: defer +2024-08-23 16:28:13,612 INFO SenderThread:11596 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-23 16:28:13,612 INFO SenderThread:11596 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-23 16:28:14,054 INFO SenderThread:11596 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240823_162802-bqibep1u/files +2024-08-23 16:28:14,055 INFO SenderThread:11596 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_162802-bqibep1u/files/requirements.txt requirements.txt +2024-08-23 16:28:14,055 INFO SenderThread:11596 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_162802-bqibep1u/files/config.yaml config.yaml +2024-08-23 16:28:14,056 INFO SenderThread:11596 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_162802-bqibep1u/files/wandb-metadata.json wandb-metadata.json +2024-08-23 16:28:14,058 INFO SenderThread:11596 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_162802-bqibep1u/files/wandb-summary.json wandb-summary.json +2024-08-23 16:28:14,058 INFO SenderThread:11596 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_162802-bqibep1u/files/output.log output.log +2024-08-23 16:28:14,059 INFO SenderThread:11596 [sender.py:transition_state():617] send defer: 10 +2024-08-23 16:28:14,060 DEBUG SenderThread:11596 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:28:14,060 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:28:14,061 INFO HandlerThread:11596 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-23 16:28:14,061 DEBUG SenderThread:11596 [sender.py:send_request():409] send_request: defer +2024-08-23 16:28:14,061 INFO SenderThread:11596 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-23 16:28:14,061 INFO SenderThread:11596 [file_pusher.py:finish():172] shutting down file pusher +2024-08-23 16:28:14,556 INFO wandb-upload_2:11596 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_162802-bqibep1u/files/wandb-summary.json +2024-08-23 16:28:14,617 INFO wandb-upload_3:11596 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_162802-bqibep1u/files/output.log +2024-08-23 16:28:14,858 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:28:14,859 DEBUG SenderThread:11596 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:28:14,918 INFO wandb-upload_0:11596 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_162802-bqibep1u/files/requirements.txt +2024-08-23 16:28:14,975 INFO wandb-upload_1:11596 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_162802-bqibep1u/files/config.yaml +2024-08-23 16:28:15,175 INFO Thread-11 (_thread_body):11596 [sender.py:transition_state():617] send defer: 11 +2024-08-23 16:28:15,175 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:28:15,175 INFO HandlerThread:11596 [handler.py:handle_request_defer():172] handle defer: 11 +2024-08-23 16:28:15,176 DEBUG SenderThread:11596 [sender.py:send_request():409] send_request: defer +2024-08-23 16:28:15,176 INFO SenderThread:11596 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-08-23 16:28:15,176 INFO SenderThread:11596 [file_pusher.py:join():178] waiting for file pusher +2024-08-23 16:28:15,176 INFO SenderThread:11596 [sender.py:transition_state():617] send defer: 12 +2024-08-23 16:28:15,176 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:28:15,176 INFO HandlerThread:11596 [handler.py:handle_request_defer():172] handle defer: 12 +2024-08-23 16:28:15,176 DEBUG SenderThread:11596 [sender.py:send_request():409] send_request: defer +2024-08-23 16:28:15,176 INFO SenderThread:11596 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-08-23 16:28:15,176 INFO SenderThread:11596 [file_stream.py:finish():595] file stream finish called +2024-08-23 16:28:15,375 INFO SenderThread:11596 [file_stream.py:finish():599] file stream finish is done +2024-08-23 16:28:15,375 INFO SenderThread:11596 [sender.py:transition_state():617] send defer: 13 +2024-08-23 16:28:15,375 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:28:15,375 INFO HandlerThread:11596 [handler.py:handle_request_defer():172] handle defer: 13 +2024-08-23 16:28:15,375 DEBUG SenderThread:11596 [sender.py:send_request():409] send_request: defer +2024-08-23 16:28:15,375 INFO SenderThread:11596 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-08-23 16:28:15,376 INFO SenderThread:11596 [sender.py:transition_state():617] send defer: 14 +2024-08-23 16:28:15,376 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:28:15,376 DEBUG SenderThread:11596 [sender.py:send():382] send: final +2024-08-23 16:28:15,376 INFO HandlerThread:11596 [handler.py:handle_request_defer():172] handle defer: 14 +2024-08-23 16:28:15,376 DEBUG SenderThread:11596 [sender.py:send():382] send: footer +2024-08-23 16:28:15,376 DEBUG SenderThread:11596 [sender.py:send_request():409] send_request: defer +2024-08-23 16:28:15,376 INFO SenderThread:11596 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-08-23 16:28:15,377 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:28:15,377 DEBUG SenderThread:11596 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:28:15,377 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:28:15,378 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: server_info +2024-08-23 16:28:15,378 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: get_summary +2024-08-23 16:28:15,378 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: sampled_history +2024-08-23 16:28:15,378 DEBUG SenderThread:11596 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:28:15,379 DEBUG SenderThread:11596 [sender.py:send_request():409] send_request: server_info +2024-08-23 16:28:15,379 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:28:15,382 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: job_info +2024-08-23 16:28:15,546 DEBUG SenderThread:11596 [sender.py:send_request():409] send_request: job_info +2024-08-23 16:28:15,546 INFO MainThread:11596 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-08-23 16:28:15,546 INFO MainThread:11596 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-08-23 16:28:15,547 INFO MainThread:11596 [wandb_run.py:_footer_sync_info():3825] logging synced files +2024-08-23 16:28:15,547 DEBUG HandlerThread:11596 [handler.py:handle_request():146] handle_request: shutdown +2024-08-23 16:28:15,547 INFO HandlerThread:11596 [handler.py:finish():869] shutting down handler +2024-08-23 16:28:16,383 INFO WriterThread:11596 [datastore.py:close():296] close: /project/wandb/run-20240823_162802-bqibep1u/run-bqibep1u.wandb +2024-08-23 16:28:16,546 INFO SenderThread:11596 [sender.py:finish():1572] shutting down sender +2024-08-23 16:28:16,547 INFO SenderThread:11596 [file_pusher.py:finish():172] shutting down file pusher +2024-08-23 16:28:16,547 INFO SenderThread:11596 [file_pusher.py:join():178] waiting for file pusher diff --git a/wandb/run-20240823_162802-bqibep1u/logs/debug.log b/wandb/run-20240823_162802-bqibep1u/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d2c3fc0b9bfc11666db5a0bed5046c03afef8d34 --- /dev/null +++ b/wandb/run-20240823_162802-bqibep1u/logs/debug.log @@ -0,0 +1,30 @@ +2024-08-23 16:28:02,658 INFO MainThread:11525 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-23 16:28:02,659 INFO MainThread:11525 [wandb_setup.py:_flush():76] Configure stats pid to 11525 +2024-08-23 16:28:02,659 INFO MainThread:11525 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-23 16:28:02,659 INFO MainThread:11525 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-23 16:28:02,659 INFO MainThread:11525 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train sample'} +2024-08-23 16:28:02,659 INFO MainThread:11525 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-23 16:28:02,659 INFO MainThread:11525 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-23 16:28:02,659 INFO MainThread:11525 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240823_162802-bqibep1u/logs/debug.log +2024-08-23 16:28:02,659 INFO MainThread:11525 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240823_162802-bqibep1u/logs/debug-internal.log +2024-08-23 16:28:02,659 INFO MainThread:11525 [wandb_init.py:init():566] calling init triggers +2024-08-23 16:28:02,659 INFO MainThread:11525 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document'], 'valid_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document'], 'test_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 4096, 'num_workers': 2, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'Qwen2-0.5b-0.2_train_2024-08-23-16:27:49', 'wandb_project': 'llm_tutorial-0.2', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/Qwen2-0.5b-0.2', 'save': '/work/llm_recipes/models/Qwen2-0.5b-0.2', 'base_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 10, 'save_interval': 10, 'eval_iters': 10, 'optimizer': 'anyprecision', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 7500, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 7500, 'train_samples': None, 'global_batch_size': 320, 'micro_batch_size': 5, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 131072, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/Qwen2-0.5b-0.2', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': True, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'valid_micro_batch_size': 1, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 151680, 'gradient_accumulation_steps': 64} +2024-08-23 16:28:02,659 INFO MainThread:11525 [wandb_init.py:init():616] starting backend +2024-08-23 16:28:02,659 INFO MainThread:11525 [wandb_init.py:init():620] setting up manager +2024-08-23 16:28:02,664 INFO MainThread:11525 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-23 16:28:02,664 INFO MainThread:11525 [wandb_init.py:init():628] backend started and connected +2024-08-23 16:28:02,669 INFO MainThread:11525 [wandb_init.py:init():720] updated telemetry +2024-08-23 16:28:02,680 INFO MainThread:11525 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-23 16:28:04,052 INFO MainThread:11525 [wandb_run.py:_on_init():2262] communicating current version +2024-08-23 16:28:04,074 INFO MainThread:11525 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.7 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-23 16:28:04,074 INFO MainThread:11525 [wandb_init.py:init():804] starting run threads in backend +2024-08-23 16:28:04,169 INFO MainThread:11525 [wandb_run.py:_console_start():2241] atexit reg +2024-08-23 16:28:04,170 INFO MainThread:11525 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-23 16:28:04,170 INFO MainThread:11525 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-23 16:28:04,170 INFO MainThread:11525 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-23 16:28:04,171 INFO MainThread:11525 [wandb_init.py:init():847] run started, returning control to user process +2024-08-23 16:28:10,361 INFO MainThread:11525 [wandb_run.py:_config_callback():1343] config_cb None None {'model_architecture': 'Qwen2ForCausalLM', 'activation_function': 'silu', 'hidden_size': 896, 'model_type': 'qwen2', 'max_position_embeddings': 4096, 'num_attention_heads': 14, 'num_hidden_layers': 24} +2024-08-23 16:28:10,361 INFO MainThread:11525 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1} +2024-08-23 16:28:16,547 WARNING MsgRouterThr:11525 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20240823_162802-bqibep1u/run-bqibep1u.wandb b/wandb/run-20240823_162802-bqibep1u/run-bqibep1u.wandb new file mode 100644 index 0000000000000000000000000000000000000000..9eec19f5c31660fba99bccd0e3657840d28ed450 Binary files /dev/null and b/wandb/run-20240823_162802-bqibep1u/run-bqibep1u.wandb differ diff --git a/wandb/run-20240826_221726-7jzdp89j/logs/debug.log b/wandb/run-20240826_221726-7jzdp89j/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b2fc59de0dc49ca416d52152ae038d67a0189406 --- /dev/null +++ b/wandb/run-20240826_221726-7jzdp89j/logs/debug.log @@ -0,0 +1,29 @@ +2024-08-26 22:17:26,988 INFO MainThread:18635 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-26 22:17:26,989 INFO MainThread:18635 [wandb_setup.py:_flush():76] Configure stats pid to 18635 +2024-08-26 22:17:26,989 INFO MainThread:18635 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-26 22:17:26,989 INFO MainThread:18635 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-26 22:17:26,989 INFO MainThread:18635 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train sample'} +2024-08-26 22:17:26,989 INFO MainThread:18635 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-26 22:17:26,989 INFO MainThread:18635 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-26 22:17:26,989 INFO MainThread:18635 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240826_221726-7jzdp89j/logs/debug.log +2024-08-26 22:17:26,989 INFO MainThread:18635 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240826_221726-7jzdp89j/logs/debug-internal.log +2024-08-26 22:17:26,989 INFO MainThread:18635 [wandb_init.py:init():566] calling init triggers +2024-08-26 22:17:26,989 INFO MainThread:18635 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document'], 'valid_data_path': ['1205770', '/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document'], 'test_data_path': ['1205770', '/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 1024, 'num_workers': 4, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/Qwen/Qwen2-1.5B', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'yans-baseline-qwen2-1.5B-3.5e-5_train_2024-08-26-22:17:00', 'wandb_project': 'yans_experiment', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5', 'save': '/work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5', 'base_model': '/share/pretrained_lm/Qwen/Qwen2-1.5B', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 200, 'save_interval': 200, 'eval_iters': 10, 'optimizer': 'anyprecision', 'lr': 3.5e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 23178, 'lr_warmup_iters': 500, 'min_lr': 3.5e-06, 'train_iters': 23178, 'train_samples': None, 'global_batch_size': 1280, 'micro_batch_size': 16, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 131072, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-08, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/yans-baseline-qwen2-1.5B-3.5e-5', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': True, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'valid_micro_batch_size': 1, 'rank': 0, 'world_size': 8, 'padded_vocab_size': 151680, 'gradient_accumulation_steps': 10} +2024-08-26 22:17:26,989 INFO MainThread:18635 [wandb_init.py:init():616] starting backend +2024-08-26 22:17:26,989 INFO MainThread:18635 [wandb_init.py:init():620] setting up manager +2024-08-26 22:17:26,994 INFO MainThread:18635 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-26 22:17:26,995 INFO MainThread:18635 [wandb_init.py:init():628] backend started and connected +2024-08-26 22:17:27,002 INFO MainThread:18635 [wandb_init.py:init():720] updated telemetry +2024-08-26 22:17:27,100 INFO MainThread:18635 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-26 22:17:27,580 INFO MainThread:18635 [wandb_run.py:_on_init():2262] communicating current version +2024-08-26 22:17:27,604 INFO MainThread:18635 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.7 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-26 22:17:27,604 INFO MainThread:18635 [wandb_init.py:init():804] starting run threads in backend +2024-08-26 22:17:27,755 INFO MainThread:18635 [wandb_run.py:_console_start():2241] atexit reg +2024-08-26 22:17:27,756 INFO MainThread:18635 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-26 22:17:27,756 INFO MainThread:18635 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-26 22:17:27,756 INFO MainThread:18635 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-26 22:17:27,757 INFO MainThread:18635 [wandb_init.py:init():847] run started, returning control to user process +2024-08-26 22:18:08,207 INFO MainThread:18635 [wandb_run.py:_config_callback():1343] config_cb None None {'model_architecture': 'Qwen2ForCausalLM', 'activation_function': 'silu', 'hidden_size': 1536, 'model_type': 'qwen2', 'max_position_embeddings': 1024, 'num_attention_heads': 12, 'num_hidden_layers': 28} +2024-08-26 22:18:08,207 INFO MainThread:18635 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 8} diff --git a/wandb/run-20241031_224656-75hrquae/files/config.yaml b/wandb/run-20241031_224656-75hrquae/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2eabe67fac0dfbf67b164cbf0f2827a03ec21a93 --- /dev/null +++ b/wandb/run-20241031_224656-75hrquae/files/config.yaml @@ -0,0 +1,319 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: NO_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '183190553' + - /project/trans/datasets/train/documents_text_document +valid_data_path: + desc: null + value: + - '61085876' + - /project/trans/datasets/valid/documents_text_document +test_data_path: + desc: null + value: + - '61085876' + - /project/trans/datasets/valid/documents_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 1024 +num_workers: + desc: null + value: 4 +tokenizer_type: + desc: null + value: HFPreTrainedTokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/llm-jp/llm-jp-v3-3.7b-instruct +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: llm-jp-v3-3.7b_ja-en_3M-pairs_train_2024-10-31-22:46:29 +wandb_project: + desc: null + value: trans_experiment +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs +save: + desc: null + value: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs +base_model: + desc: null + value: /share/pretrained_lm/llm-jp/llm-jp-v3-3.7b-instruct +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 200 +save_interval: + desc: null + value: 200 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: anyprecision +lr: + desc: null + value: 0.00015 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 50 +lr_warmup_iters: + desc: null + value: 5 +min_lr: + desc: null + value: 1.5e-05 +train_iters: + desc: null + value: 50 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 256 +micro_batch_size: + desc: null + value: 1 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 131072 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-08 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/llm-jp-v3-3.7b_ja-en_3M-pairs +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: true +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +valid_micro_batch_size: + desc: null + value: 1 +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 99584 +gradient_accumulation_steps: + desc: null + value: 256 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1730382416.312415 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 diff --git a/wandb/run-20241031_224656-75hrquae/files/output.log b/wandb/run-20241031_224656-75hrquae/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..dd1b62a5332f09974ce6c5340de277cb3dfa6bb4 --- /dev/null +++ b/wandb/run-20241031_224656-75hrquae/files/output.log @@ -0,0 +1,8 @@ +Created Hugging Face repository with ID koichi12/llm-jp-v3-3.7b_ja-en_3M-pairs. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +You are attempting to use Flash Attention 2.0 without specifying a torch dtype. This might lead to unexpected behaviour +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +Flash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in LlamaForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `torch_dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", torch_dtype=torch.float16)` +Flash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in LlamaModel is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `torch_dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", torch_dtype=torch.float16)` +Loading checkpoint shards: 0%| | 0/2 [00:00 Running with torch torch_distributed debug set to detail +File not found: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs/latest_iteration.txt +File not found: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs/latest_iteration.txt +You are attempting to use Flash Attention 2.0 without specifying a torch dtype. This might lead to unexpected behaviour +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +Flash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in LlamaForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `torch_dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", torch_dtype=torch.float16)` +Flash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in LlamaModel is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `torch_dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", torch_dtype=torch.float16)` + + +Loading checkpoint shards: 100%|██████████| 2/2 [00:07<00:00, 3.91s/it] +File not found: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs, skipping model loading +--> Model /share/pretrained_lm/llm-jp/llm-jp-v3-3.7b +--> /share/pretrained_lm/llm-jp/llm-jp-v3-3.7b has 3782.913024 Million params +BFloat16 enabled for mixed precision - using bfSixteen policy +Let split = None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 178688 + validation: 89600 + test: 2560 +> building train, validation, and test datasets for GPT ... +> finished creating GPT datasets ... +File not found: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs, skipping optimizer loading +File not found: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs/latest_iteration.txt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): LlamaForCausalLM( + (model): LlamaModel( + (embed_tokens): Embedding(99584, 3072) + (layers): ModuleList( + (0-27): 28 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): LlamaDecoderLayer( + (self_attn): LlamaFlashAttention2( + (q_proj): Linear(in_features=3072, out_features=3072, bias=False) + (k_proj): Linear(in_features=3072, out_features=3072, bias=False) + (v_proj): Linear(in_features=3072, out_features=3072, bias=False) + (o_proj): Linear(in_features=3072, out_features=3072, bias=False) + (rotary_emb): LlamaRotaryEmbedding() + ) + (mlp): LlamaMLP( + (gate_proj): Linear(in_features=3072, out_features=8192, bias=False) + (up_proj): Linear(in_features=3072, out_features=8192, bias=False) + (down_proj): Linear(in_features=8192, out_features=3072, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): LlamaRMSNorm() + (post_attention_layernorm): LlamaRMSNorm() + ) + ) + ) + ) + (norm): LlamaRMSNorm() + (rotary_emb): LlamaRotaryEmbedding() + ) + (lm_head): Linear(in_features=3072, out_features=99584, bias=False) + ) +) +model config: LlamaConfig { + "_name_or_path": "/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b", + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 1, + "eos_token_id": 2, + "hidden_act": "silu", + "hidden_size": 3072, + "initializer_range": 0.02, + "intermediate_size": 8192, + "label_smoothing": 0.0, + "max_position_embeddings": 4096, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 24, + "num_hidden_layers": 28, + "num_key_value_heads": 24, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 10000, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.43.3", + "use_cache": false, + "vocab_size": 99584 +} +------------------------------------------------------------------ +iteration: 1 , TFLOPS: 24.557547351612122, Tokens per sec: 2240.7653923163525, Loss: 3.053647518157959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2 , TFLOPS: 24.688628115712138, Tokens per sec: 2252.725920604794, Loss: 2.9236981868743896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3 , TFLOPS: 24.747876957219642, Tokens per sec: 2258.1321100619075, Loss: 2.7249813079833984 +------------------------------------------------------------------ \ No newline at end of file diff --git a/wandb/run-20241101_032853-4fl9vo96/files/requirements.txt b/wandb/run-20241101_032853-4fl9vo96/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..34a2774e444bdc395654ccf8ce6da6833c7bc1ee --- /dev/null +++ b/wandb/run-20241101_032853-4fl9vo96/files/requirements.txt @@ -0,0 +1,375 @@ +absl-py==2.1.0 +accelerate==0.23.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +astroid==3.2.4 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bert-score==0.3.13 +bleach==6.1.0 +blis==0.7.11 +build==1.2.1 +cachecontrol==0.14.0 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +cleo==2.1.0 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cramjam==2.8.3 +crashtest==0.4.1 +cryptography==43.0.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +dataclasses-json==0.6.7 +dataproperty==1.0.1 +datasets==2.20.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +dill==0.3.8 +distlib==0.3.8 +distributed==2023.11.0 +distro==1.9.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +dulwich==0.21.7 +einops==0.7.0 +emoji==2.12.1 +entmax==1.3 +evaluate==0.4.2 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastparquet==2023.10.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +fugashi==1.3.2 +fuzzywuzzy==0.18.0 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +greenlet==3.0.3 +grpcio==1.60.1 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.24.5 +hydra-core==1.3.2 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +installer==0.7.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +isort==5.13.2 +jaraco.classes==3.4.0 +jedi==0.19.1 +jeepney==0.8.0 +jinja2==3.1.3 +jiter==0.5.0 +joblib==1.3.2 +json5==0.9.14 +jsonargparse==3.13.1 +jsonlines==4.0.0 +jsonnet==0.19.1 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +keyring==24.3.1 +kiwisolver==1.4.5 +langchain-community==0.2.12 +langchain-core==0.2.31 +langchain-huggingface==0.0.2 +langchain-openai==0.1.21 +langchain-text-splitters==0.2.2 +langchain==0.2.13 +langcodes==3.3.0 +langsmith==0.1.99 +lazy-loader==0.3 +levenshtein==0.25.1 +librosa==0.10.1 +lightning-utilities==0.11.6 +llm-jp-eval==1.4.0 +llvmlite==0.40.1 +lm-eval==0.3.0 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +marshmallow==3.21.3 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mbstrdecoder==1.1.3 +mccabe==0.7.0 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +mojimoji==0.0.13 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +multiprocess==0.70.16 +murmurhash==1.0.10 +mypy-extensions==1.0.0 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +neologdn==0.5.3 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numexpr==2.10.1 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +omegaconf==2.3.0 +onnx==1.15.0rc2 +openai==1.40.6 +opencv==4.7.0 +optree==0.10.0 +orjson==3.10.7 +packaging==23.2 +pandas==2.2.2 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +pathvalidate==3.2.0 +peft==0.5.0 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +pkginfo==1.11.1 +plac==1.4.3 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +poetry-core==1.9.0 +poetry-plugin-export==1.8.0 +poetry==1.8.3 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow-hotfix==0.6 +pyarrow==15.0.2 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycountry==24.6.1 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pylint==3.2.6 +pynvml==11.4.1 +pyparsing==3.1.1 +pyproject-hooks==1.1.0 +pytablewriter==1.2.0 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +python-levenshtein==0.25.1 +pytorch-lightning==2.4.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapidfuzz==3.9.6 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests-toolbelt==1.0.0 +requests==2.32.3 +rhoknp==1.7.0 +rich==13.7.0 +rmm==23.12.0 +rouge-score==0.1.2 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.2 +safetensors==0.4.3 +scikit-learn==1.5.1 +scipy==1.12.0 +secretstorage==3.3.3 +send2trash==1.8.2 +sentence-transformers==3.0.1 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +shellingham==1.5.4 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +sqlalchemy==2.0.32 +sqlitedict==2.1.0 +srsly==2.4.8 +stack-data==0.6.3 +sumeval==0.2.2 +sympy==1.12 +tabledata==1.3.3 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tcolorpy==0.1.6 +tenacity==8.5.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +text-generation==0.7.0 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tiktoken==0.7.0 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +tomlkit==0.13.2 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchmetrics==0.10.3 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm-multiprocess==0.0.11 +tqdm==4.66.5 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +trove-classifiers==2024.7.2 +typepy==1.3.2 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.12.2 +typing-inspect==0.9.0 +tzdata==2024.1 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +unbabel-comet==2.2.2 +unidic-lite==1.0.8 +urllib3==1.26.18 +virtualenv==20.26.3 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +word2number==1.1 +xdoctest==1.0.2 +xgboost==1.7.6 +xmltodict==0.13.0 +xxhash==3.4.1 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 +zstandard==0.23.0 \ No newline at end of file diff --git a/wandb/run-20241101_032853-4fl9vo96/files/wandb-metadata.json b/wandb/run-20241101_032853-4fl9vo96/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..ed0deda3bdf1b0993f2cb96d33222fbb7c2f36f3 --- /dev/null +++ b/wandb/run-20241101_032853-4fl9vo96/files/wandb-metadata.json @@ -0,0 +1,314 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-10-31T18:28:53.835149", + "startedAt": "2024-10-31T18:28:53.249894", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "1024", + "--sliding-window-size", + "131072", + "--micro-batch-size", + "1", + "--valid_micro_batch_size", + "1", + "--global-batch-size", + "256", + "--train-iters", + "698", + "--tokenizer-type", + "HFPreTrainedTokenizer", + "--tokenizer-model", + "/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b", + "--train-data-path", + "183224836", + "/project/trans/datasets/processed_ja-en/train/documents_text_document", + "--valid-data-path", + "61084836", + "/project/trans/datasets/processed_ja-en/valid/documents_text_document", + "--test-data-path", + "61084836", + "/project/trans/datasets/processed_ja-en/valid/documents_text_document", + "--lr", + "1.5e-4", + "--min-lr", + "1.5e-5", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "70", + "--lr-decay-iters", + "698", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "anyprecision", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-8", + "--save-interval", + "50", + "--eval-interval", + "20", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b", + "--save", + "/work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs", + "--load", + "/work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs", + "--num-workers", + "4", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "NO_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "3", + "--upload-all-checkpoints-to-hf", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/llm-jp-v3-3.7b_ja-en_actual_3M-pairs", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "trans_experiment", + "--wandb-name", + "llm-jp-v3-3.7b_ja-en_actual_3M-pairs_train_2024-11-01-03:28:39" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "3b2976faebe2228c39adb20194a29b785a37defe" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 36, + "cpu_count_logical": 36, + "cpu_freq": { + "current": 2400.032000000001, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 2, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 113.18244934082031 + } +} diff --git a/wandb/run-20241101_032853-4fl9vo96/files/wandb-summary.json b/wandb/run-20241101_032853-4fl9vo96/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..c414e5306440647f3c866fda677a404ef468c66d --- /dev/null +++ b/wandb/run-20241101_032853-4fl9vo96/files/wandb-summary.json @@ -0,0 +1 @@ +{"training/loss": 2.7249813079833984, "training/perplexity": 15.25612875638197, "utils/batch_size": 1, "utils/global_batch_size": 256, "utils/seq_len": 1025, "utils/gradient_accumulation_steps": 128, "utils/iteration": 3, "optimizer/lr": 2.0785714285714284e-05, "optimizer/variance_l2": 0.0068593084794657914, "optimizer/variance_sqrt_l2": 0.3780638976422062, "optimizer/momentum_l2": 0.19476604310138873, "optimizer/weight_l2": 1145.7966660799812, "optimizer/variance_l1": 0.14287757873535156, "optimizer/variance_sqrt_l1": 3317.5, "optimizer/momentum_l1": 1475.5, "optimizer/weight_l1": 51896320.0, "optimizer/variance_abs_max": 0.002716064453125, "optimizer/variance_sqrt_abs_max": 0.052001953125, "optimizer/momentum_abs_max": 0.032470703125, "optimizer/weight_abs_max": 6.5, "stats/1_iteration_time": 116.20223583500001, "stats/tokens_per_sec": 2258.1321100619075, "stats/tokens_per_sec_per_gpu": 1129.0660550309537, "stats/tflops": 24.747876957219642, "_timestamp": 1730399698.4356222, "_runtime": 365.172217130661, "_step": 3, "_wandb": {"runtime": 468}} \ No newline at end of file diff --git a/wandb/run-20241101_032853-4fl9vo96/logs/debug-internal.log b/wandb/run-20241101_032853-4fl9vo96/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..e4c2ba1beb36340902551efc6b204dd4f3c7859c --- /dev/null +++ b/wandb/run-20241101_032853-4fl9vo96/logs/debug-internal.log @@ -0,0 +1,396 @@ +2024-11-01 03:28:53,265 INFO StreamThr :11187 [internal.py:wandb_internal():86] W&B internal server running at pid: 11187, started at: 2024-11-01 03:28:53.264657 +2024-11-01 03:28:53,267 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status +2024-11-01 03:28:53,269 INFO WriterThread:11187 [datastore.py:open_for_write():87] open: /project/wandb/run-20241101_032853-4fl9vo96/run-4fl9vo96.wandb +2024-11-01 03:28:53,270 DEBUG SenderThread:11187 [sender.py:send():382] send: header +2024-11-01 03:28:53,284 DEBUG SenderThread:11187 [sender.py:send():382] send: run +2024-11-01 03:28:53,733 INFO SenderThread:11187 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20241101_032853-4fl9vo96/files +2024-11-01 03:28:53,733 INFO SenderThread:11187 [sender.py:_start_run_threads():1136] run started: 4fl9vo96 with start time 1730399333.263405 +2024-11-01 03:28:53,737 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: check_version +2024-11-01 03:28:53,738 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: check_version +2024-11-01 03:28:53,807 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: run_start +2024-11-01 03:28:53,814 DEBUG HandlerThread:11187 [system_info.py:__init__():27] System info init +2024-11-01 03:28:53,814 DEBUG HandlerThread:11187 [system_info.py:__init__():42] System info init done +2024-11-01 03:28:53,814 INFO HandlerThread:11187 [system_monitor.py:start():194] Starting system monitor +2024-11-01 03:28:53,814 INFO SystemMonitor:11187 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-11-01 03:28:53,814 INFO HandlerThread:11187 [system_monitor.py:probe():214] Collecting system info +2024-11-01 03:28:53,814 INFO SystemMonitor:11187 [interfaces.py:start():190] Started cpu monitoring +2024-11-01 03:28:53,815 INFO SystemMonitor:11187 [interfaces.py:start():190] Started disk monitoring +2024-11-01 03:28:53,815 INFO SystemMonitor:11187 [interfaces.py:start():190] Started gpu monitoring +2024-11-01 03:28:53,816 INFO SystemMonitor:11187 [interfaces.py:start():190] Started memory monitoring +2024-11-01 03:28:53,817 INFO SystemMonitor:11187 [interfaces.py:start():190] Started network monitoring +2024-11-01 03:28:53,835 DEBUG HandlerThread:11187 [system_info.py:probe():151] Probing system +2024-11-01 03:28:53,837 DEBUG HandlerThread:11187 [system_info.py:_probe_git():136] Probing git +2024-11-01 03:28:53,850 DEBUG HandlerThread:11187 [system_info.py:_probe_git():144] Probing git done +2024-11-01 03:28:53,850 DEBUG HandlerThread:11187 [system_info.py:probe():199] Probing system done +2024-11-01 03:28:53,850 DEBUG HandlerThread:11187 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-10-31T18:28:53.835149', 'startedAt': '2024-10-31T18:28:53.249894', 'docker': None, 'cuda': None, 'args': ('--seq-length', '1024', '--sliding-window-size', '131072', '--micro-batch-size', '1', '--valid_micro_batch_size', '1', '--global-batch-size', '256', '--train-iters', '698', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b', '--train-data-path', '183224836', '/project/trans/datasets/processed_ja-en/train/documents_text_document', '--valid-data-path', '61084836', '/project/trans/datasets/processed_ja-en/valid/documents_text_document', '--test-data-path', '61084836', '/project/trans/datasets/processed_ja-en/valid/documents_text_document', '--lr', '1.5e-4', '--min-lr', '1.5e-5', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '70', '--lr-decay-iters', '698', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'anyprecision', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-8', '--save-interval', '50', '--eval-interval', '20', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b', '--save', '/work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs', '--load', '/work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs', '--num-workers', '4', '--fsdp-activation-checkpointing', '--sharding-strategy', 'NO_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '3', '--upload-all-checkpoints-to-hf', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/llm-jp-v3-3.7b_ja-en_actual_3M-pairs', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'trans_experiment', '--wandb-name', 'llm-jp-v3-3.7b_ja-en_actual_3M-pairs_train_2024-11-01-03:28:39'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '3b2976faebe2228c39adb20194a29b785a37defe'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 36, 'cpu_count_logical': 36, 'cpu_freq': {'current': 2400.032000000001, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 2, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 113.18244934082031}} +2024-11-01 03:28:53,850 INFO HandlerThread:11187 [system_monitor.py:probe():224] Finished collecting system info +2024-11-01 03:28:53,850 INFO HandlerThread:11187 [system_monitor.py:probe():227] Publishing system info +2024-11-01 03:28:53,852 INFO HandlerThread:11187 [system_monitor.py:probe():229] Finished publishing system info +2024-11-01 03:28:53,857 DEBUG SenderThread:11187 [sender.py:send():382] send: files +2024-11-01 03:28:53,857 INFO SenderThread:11187 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-11-01 03:28:53,869 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: python_packages +2024-11-01 03:28:53,870 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:28:53,870 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:28:53,870 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: python_packages +2024-11-01 03:28:53,872 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:28:54,102 DEBUG SenderThread:11187 [sender.py:send():382] send: telemetry +2024-11-01 03:28:54,504 INFO wandb-upload_0:11187 [upload_job.py:push():131] Uploaded file /tmp/tmp_k080z1mwandb/90j6uifg-wandb-metadata.json +2024-11-01 03:28:54,735 INFO Thread-12 :11187 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20241101_032853-4fl9vo96/files/requirements.txt +2024-11-01 03:28:54,735 INFO Thread-12 :11187 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20241101_032853-4fl9vo96/files/wandb-metadata.json +2024-11-01 03:28:54,735 INFO Thread-12 :11187 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20241101_032853-4fl9vo96/files/output.log +2024-11-01 03:28:56,735 INFO Thread-12 :11187 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_032853-4fl9vo96/files/output.log +2024-11-01 03:28:58,637 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:29:00,738 INFO Thread-12 :11187 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_032853-4fl9vo96/files/output.log +2024-11-01 03:29:02,739 INFO Thread-12 :11187 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_032853-4fl9vo96/files/output.log +2024-11-01 03:29:04,478 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:29:04,741 INFO Thread-12 :11187 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_032853-4fl9vo96/files/output.log +2024-11-01 03:29:08,188 DEBUG SenderThread:11187 [sender.py:send():382] send: config +2024-11-01 03:29:08,188 DEBUG SenderThread:11187 [sender.py:send():382] send: config +2024-11-01 03:29:08,744 INFO Thread-12 :11187 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_032853-4fl9vo96/files/output.log +2024-11-01 03:29:08,871 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:29:08,872 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:29:08,872 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:29:10,124 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:29:10,745 INFO Thread-12 :11187 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_032853-4fl9vo96/files/output.log +2024-11-01 03:29:15,125 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:29:20,125 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:29:23,868 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:29:23,868 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:29:23,910 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:29:25,129 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:29:25,755 INFO Thread-12 :11187 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_032853-4fl9vo96/files/config.yaml +2024-11-01 03:29:30,753 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:29:35,754 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:29:38,868 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:29:38,869 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:29:38,910 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:29:41,120 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:29:46,121 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:29:51,122 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:29:53,817 DEBUG SystemMonitor:11187 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-11-01 03:29:53,820 DEBUG SenderThread:11187 [sender.py:send():382] send: stats +2024-11-01 03:29:53,868 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:29:53,869 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:29:53,910 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:29:57,060 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:30:02,061 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:30:07,062 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:30:08,868 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:30:08,869 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:30:08,910 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:30:12,123 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:30:17,123 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:30:22,124 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:30:23,821 DEBUG SenderThread:11187 [sender.py:send():382] send: stats +2024-11-01 03:30:23,868 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:30:23,869 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:30:23,910 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:30:28,104 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:30:33,105 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:30:38,106 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:30:38,869 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:30:38,869 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:30:38,910 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:30:44,062 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:30:49,062 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:30:53,822 DEBUG SenderThread:11187 [sender.py:send():382] send: stats +2024-11-01 03:30:53,869 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:30:53,869 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:30:53,910 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:30:54,093 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:30:59,094 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:31:04,095 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:31:05,748 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:31:08,820 INFO Thread-12 :11187 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_032853-4fl9vo96/files/output.log +2024-11-01 03:31:08,869 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:31:08,869 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:31:08,871 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:31:09,121 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:31:14,122 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:31:19,123 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:31:23,823 DEBUG SenderThread:11187 [sender.py:send():382] send: stats +2024-11-01 03:31:23,869 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:31:23,869 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:31:23,910 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:31:24,130 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:31:29,131 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:31:34,132 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:31:38,869 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:31:38,869 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:31:38,910 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:31:40,047 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:31:45,048 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:31:50,049 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:31:53,825 DEBUG SenderThread:11187 [sender.py:send():382] send: stats +2024-11-01 03:31:53,869 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:31:53,869 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:31:53,910 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:31:55,102 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:32:00,103 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:32:05,104 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:32:08,869 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:32:08,870 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:32:08,910 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:32:10,151 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:32:15,152 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:32:20,153 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:32:23,826 DEBUG SenderThread:11187 [sender.py:send():382] send: stats +2024-11-01 03:32:23,869 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:32:23,870 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:32:23,910 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:32:26,128 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:32:31,129 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:32:36,130 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:32:38,869 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:32:38,870 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:32:38,910 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:32:42,104 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:32:47,105 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:32:52,106 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:32:53,827 DEBUG SenderThread:11187 [sender.py:send():382] send: stats +2024-11-01 03:32:53,870 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:32:53,870 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:32:53,910 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:32:58,106 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:33:02,232 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:33:02,235 DEBUG SenderThread:11187 [sender.py:send():382] send: history +2024-11-01 03:33:02,235 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:33:02,237 INFO SenderThread:11187 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:33:02,896 INFO Thread-12 :11187 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20241101_032853-4fl9vo96/files/wandb-summary.json +2024-11-01 03:33:03,276 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:33:04,897 INFO Thread-12 :11187 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_032853-4fl9vo96/files/output.log +2024-11-01 03:33:08,276 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:33:08,870 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:33:08,870 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:33:08,872 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:33:14,066 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:33:19,067 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:33:23,832 DEBUG SenderThread:11187 [sender.py:send():382] send: stats +2024-11-01 03:33:23,870 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:33:23,870 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:33:23,910 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:33:24,096 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:33:29,097 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:33:34,098 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:33:38,870 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:33:38,870 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:33:38,910 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:33:39,137 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:33:44,138 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:33:49,139 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:33:53,830 DEBUG SenderThread:11187 [sender.py:send():382] send: stats +2024-11-01 03:33:53,870 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:33:53,870 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:33:53,910 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:33:55,046 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:34:00,047 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:34:05,047 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:34:08,870 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:34:08,871 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:34:08,910 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:34:10,126 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:34:15,127 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:34:20,128 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:34:23,832 DEBUG SenderThread:11187 [sender.py:send():382] send: stats +2024-11-01 03:34:23,870 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:34:23,871 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:34:23,910 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:34:26,051 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:34:31,051 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:34:36,052 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:34:38,870 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:34:38,871 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:34:38,918 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:34:41,130 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:34:46,130 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:34:51,131 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:34:53,833 DEBUG SenderThread:11187 [sender.py:send():382] send: stats +2024-11-01 03:34:53,870 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:34:53,871 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:34:53,914 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:34:57,053 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:34:58,437 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:34:58,438 DEBUG SenderThread:11187 [sender.py:send():382] send: history +2024-11-01 03:34:58,439 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:34:58,440 INFO SenderThread:11187 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:34:58,971 INFO Thread-12 :11187 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_032853-4fl9vo96/files/wandb-summary.json +2024-11-01 03:35:00,973 INFO Thread-12 :11187 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_032853-4fl9vo96/files/output.log +2024-11-01 03:35:02,480 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:35:07,481 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:35:08,871 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:35:08,871 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:35:08,873 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:35:13,105 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:35:18,106 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:35:23,107 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:35:23,834 DEBUG SenderThread:11187 [sender.py:send():382] send: stats +2024-11-01 03:35:23,871 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:35:23,871 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:35:23,914 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:35:28,149 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:35:33,149 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:35:38,150 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:35:38,871 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:35:38,871 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:35:38,914 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:35:44,099 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:35:49,100 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:35:53,836 DEBUG SenderThread:11187 [sender.py:send():382] send: stats +2024-11-01 03:35:53,871 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:35:53,871 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:35:53,914 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:35:54,142 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:35:59,143 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:36:04,144 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:36:08,871 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:36:08,872 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:36:08,914 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:36:10,050 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:36:15,051 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:36:20,051 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:36:23,837 DEBUG SenderThread:11187 [sender.py:send():382] send: stats +2024-11-01 03:36:23,872 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:36:23,872 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:36:23,914 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:36:25,093 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:36:30,094 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:36:35,095 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:36:38,871 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:36:38,872 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:36:38,914 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:36:40,116 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:36:42,274 DEBUG SenderThread:11187 [sender.py:send():382] send: exit +2024-11-01 03:36:42,275 INFO SenderThread:11187 [sender.py:send_exit():589] handling exit code: 255 +2024-11-01 03:36:42,275 INFO SenderThread:11187 [sender.py:send_exit():591] handling runtime: 468 +2024-11-01 03:36:42,276 INFO SenderThread:11187 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:36:42,276 INFO SenderThread:11187 [sender.py:send_exit():597] send defer +2024-11-01 03:36:42,276 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: defer +2024-11-01 03:36:42,276 INFO HandlerThread:11187 [handler.py:handle_request_defer():172] handle defer: 0 +2024-11-01 03:36:42,277 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: defer +2024-11-01 03:36:42,277 INFO SenderThread:11187 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-11-01 03:36:42,277 INFO SenderThread:11187 [sender.py:transition_state():617] send defer: 1 +2024-11-01 03:36:42,277 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: defer +2024-11-01 03:36:42,277 INFO HandlerThread:11187 [handler.py:handle_request_defer():172] handle defer: 1 +2024-11-01 03:36:42,277 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: defer +2024-11-01 03:36:42,277 INFO SenderThread:11187 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-11-01 03:36:42,277 INFO SenderThread:11187 [sender.py:transition_state():617] send defer: 2 +2024-11-01 03:36:42,277 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: defer +2024-11-01 03:36:42,277 INFO HandlerThread:11187 [handler.py:handle_request_defer():172] handle defer: 2 +2024-11-01 03:36:42,277 INFO HandlerThread:11187 [system_monitor.py:finish():203] Stopping system monitor +2024-11-01 03:36:42,277 DEBUG SystemMonitor:11187 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-11-01 03:36:42,278 INFO HandlerThread:11187 [interfaces.py:finish():202] Joined cpu monitor +2024-11-01 03:36:42,278 DEBUG SystemMonitor:11187 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-11-01 03:36:42,278 INFO HandlerThread:11187 [interfaces.py:finish():202] Joined disk monitor +2024-11-01 03:36:42,379 INFO HandlerThread:11187 [interfaces.py:finish():202] Joined gpu monitor +2024-11-01 03:36:42,379 INFO HandlerThread:11187 [interfaces.py:finish():202] Joined memory monitor +2024-11-01 03:36:42,379 INFO HandlerThread:11187 [interfaces.py:finish():202] Joined network monitor +2024-11-01 03:36:42,380 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: defer +2024-11-01 03:36:42,380 INFO SenderThread:11187 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-11-01 03:36:42,380 INFO SenderThread:11187 [sender.py:transition_state():617] send defer: 3 +2024-11-01 03:36:42,380 DEBUG SenderThread:11187 [sender.py:send():382] send: stats +2024-11-01 03:36:42,380 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: defer +2024-11-01 03:36:42,381 INFO HandlerThread:11187 [handler.py:handle_request_defer():172] handle defer: 3 +2024-11-01 03:36:42,382 DEBUG SenderThread:11187 [sender.py:send():382] send: history +2024-11-01 03:36:42,382 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:36:42,383 INFO SenderThread:11187 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:36:42,384 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: defer +2024-11-01 03:36:42,384 INFO SenderThread:11187 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-11-01 03:36:42,384 INFO SenderThread:11187 [sender.py:transition_state():617] send defer: 4 +2024-11-01 03:36:42,384 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: defer +2024-11-01 03:36:42,384 INFO HandlerThread:11187 [handler.py:handle_request_defer():172] handle defer: 4 +2024-11-01 03:36:42,384 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: defer +2024-11-01 03:36:42,384 INFO SenderThread:11187 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-11-01 03:36:42,384 INFO SenderThread:11187 [sender.py:transition_state():617] send defer: 5 +2024-11-01 03:36:42,384 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: defer +2024-11-01 03:36:42,384 INFO HandlerThread:11187 [handler.py:handle_request_defer():172] handle defer: 5 +2024-11-01 03:36:42,385 DEBUG SenderThread:11187 [sender.py:send():382] send: summary +2024-11-01 03:36:42,386 INFO SenderThread:11187 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:36:42,386 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: defer +2024-11-01 03:36:42,386 INFO SenderThread:11187 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-11-01 03:36:42,386 INFO SenderThread:11187 [sender.py:transition_state():617] send defer: 6 +2024-11-01 03:36:42,386 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: defer +2024-11-01 03:36:42,386 INFO HandlerThread:11187 [handler.py:handle_request_defer():172] handle defer: 6 +2024-11-01 03:36:42,387 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: defer +2024-11-01 03:36:42,387 INFO SenderThread:11187 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-11-01 03:36:42,387 INFO SenderThread:11187 [sender.py:transition_state():617] send defer: 7 +2024-11-01 03:36:42,387 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:36:42,387 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: defer +2024-11-01 03:36:42,387 INFO HandlerThread:11187 [handler.py:handle_request_defer():172] handle defer: 7 +2024-11-01 03:36:42,387 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: defer +2024-11-01 03:36:42,387 INFO SenderThread:11187 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-11-01 03:36:43,041 INFO Thread-12 :11187 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_032853-4fl9vo96/files/wandb-summary.json +2024-11-01 03:36:43,122 INFO SenderThread:11187 [sender.py:transition_state():617] send defer: 8 +2024-11-01 03:36:43,122 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: defer +2024-11-01 03:36:43,122 INFO HandlerThread:11187 [handler.py:handle_request_defer():172] handle defer: 8 +2024-11-01 03:36:43,122 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: defer +2024-11-01 03:36:43,122 INFO SenderThread:11187 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-11-01 03:36:43,123 INFO SenderThread:11187 [job_builder.py:build():296] Attempting to build job artifact +2024-11-01 03:36:43,123 INFO SenderThread:11187 [job_builder.py:_get_source_type():426] is repo sourced job +2024-11-01 03:36:43,139 INFO SenderThread:11187 [job_builder.py:build():402] adding wandb-job metadata file +2024-11-01 03:36:43,172 INFO SenderThread:11187 [sender.py:transition_state():617] send defer: 9 +2024-11-01 03:36:43,172 DEBUG SenderThread:11187 [sender.py:send():382] send: artifact +2024-11-01 03:36:43,173 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: defer +2024-11-01 03:36:43,174 INFO HandlerThread:11187 [handler.py:handle_request_defer():172] handle defer: 9 +2024-11-01 03:36:43,274 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: poll_exit +2024-11-01 03:36:44,041 INFO SenderThread:11187 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTMwMjQ5MDY3NQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjQ4MzQ0MjQxNg==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTMwMjQ5MDY3NQ==', 'versionIndex': 1}}} +2024-11-01 03:36:44,041 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: defer +2024-11-01 03:36:44,041 INFO SenderThread:11187 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-11-01 03:36:44,041 INFO SenderThread:11187 [dir_watcher.py:finish():358] shutting down directory watcher +2024-11-01 03:36:44,042 INFO SenderThread:11187 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_032853-4fl9vo96/files/output.log +2024-11-01 03:36:44,042 INFO SenderThread:11187 [dir_watcher.py:finish():388] scan: /project/wandb/run-20241101_032853-4fl9vo96/files +2024-11-01 03:36:44,043 INFO SenderThread:11187 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20241101_032853-4fl9vo96/files/requirements.txt requirements.txt +2024-11-01 03:36:44,043 INFO SenderThread:11187 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20241101_032853-4fl9vo96/files/config.yaml config.yaml +2024-11-01 03:36:44,044 INFO SenderThread:11187 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20241101_032853-4fl9vo96/files/wandb-metadata.json wandb-metadata.json +2024-11-01 03:36:44,045 INFO SenderThread:11187 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20241101_032853-4fl9vo96/files/wandb-summary.json wandb-summary.json +2024-11-01 03:36:44,046 INFO SenderThread:11187 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20241101_032853-4fl9vo96/files/output.log output.log +2024-11-01 03:36:44,048 INFO SenderThread:11187 [sender.py:transition_state():617] send defer: 10 +2024-11-01 03:36:44,048 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: poll_exit +2024-11-01 03:36:44,048 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: defer +2024-11-01 03:36:44,049 INFO HandlerThread:11187 [handler.py:handle_request_defer():172] handle defer: 10 +2024-11-01 03:36:44,050 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: defer +2024-11-01 03:36:44,050 INFO SenderThread:11187 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-11-01 03:36:44,050 INFO SenderThread:11187 [file_pusher.py:finish():172] shutting down file pusher +2024-11-01 03:36:44,275 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: poll_exit +2024-11-01 03:36:44,275 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: poll_exit +2024-11-01 03:36:44,551 INFO wandb-upload_1:11187 [upload_job.py:push():131] Uploaded file /project/wandb/run-20241101_032853-4fl9vo96/files/config.yaml +2024-11-01 03:36:44,614 INFO wandb-upload_0:11187 [upload_job.py:push():131] Uploaded file /project/wandb/run-20241101_032853-4fl9vo96/files/requirements.txt +2024-11-01 03:36:44,634 INFO wandb-upload_3:11187 [upload_job.py:push():131] Uploaded file /project/wandb/run-20241101_032853-4fl9vo96/files/output.log +2024-11-01 03:36:44,954 INFO wandb-upload_2:11187 [upload_job.py:push():131] Uploaded file /project/wandb/run-20241101_032853-4fl9vo96/files/wandb-summary.json +2024-11-01 03:36:45,154 INFO Thread-11 (_thread_body):11187 [sender.py:transition_state():617] send defer: 11 +2024-11-01 03:36:45,155 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: defer +2024-11-01 03:36:45,155 INFO HandlerThread:11187 [handler.py:handle_request_defer():172] handle defer: 11 +2024-11-01 03:36:45,155 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: defer +2024-11-01 03:36:45,155 INFO SenderThread:11187 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-11-01 03:36:45,155 INFO SenderThread:11187 [file_pusher.py:join():178] waiting for file pusher +2024-11-01 03:36:45,155 INFO SenderThread:11187 [sender.py:transition_state():617] send defer: 12 +2024-11-01 03:36:45,155 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: defer +2024-11-01 03:36:45,155 INFO HandlerThread:11187 [handler.py:handle_request_defer():172] handle defer: 12 +2024-11-01 03:36:45,155 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: defer +2024-11-01 03:36:45,155 INFO SenderThread:11187 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-11-01 03:36:45,155 INFO SenderThread:11187 [file_stream.py:finish():595] file stream finish called +2024-11-01 03:36:45,859 INFO SenderThread:11187 [file_stream.py:finish():599] file stream finish is done +2024-11-01 03:36:45,859 INFO SenderThread:11187 [sender.py:transition_state():617] send defer: 13 +2024-11-01 03:36:45,859 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: defer +2024-11-01 03:36:45,859 INFO HandlerThread:11187 [handler.py:handle_request_defer():172] handle defer: 13 +2024-11-01 03:36:45,859 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: defer +2024-11-01 03:36:45,859 INFO SenderThread:11187 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-11-01 03:36:45,859 INFO SenderThread:11187 [sender.py:transition_state():617] send defer: 14 +2024-11-01 03:36:45,860 DEBUG SenderThread:11187 [sender.py:send():382] send: final +2024-11-01 03:36:45,860 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: defer +2024-11-01 03:36:45,860 DEBUG SenderThread:11187 [sender.py:send():382] send: footer +2024-11-01 03:36:45,860 INFO HandlerThread:11187 [handler.py:handle_request_defer():172] handle defer: 14 +2024-11-01 03:36:45,860 DEBUG SenderThread:11187 [sender.py:send_request():409] send_request: defer +2024-11-01 03:36:45,860 INFO SenderThread:11187 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-11-01 03:36:50,861 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:36:55,862 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:37:00,863 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:37:05,864 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:37:10,865 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:37:15,865 DEBUG HandlerThread:11187 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:37:18,808 WARNING StreamThr :11187 [internal.py:is_dead():414] Internal process exiting, parent pid 11023 disappeared +2024-11-01 03:37:18,809 ERROR StreamThr :11187 [internal.py:wandb_internal():152] Internal process shutdown. +2024-11-01 03:37:18,866 INFO SenderThread:11187 [sender.py:finish():1572] shutting down sender +2024-11-01 03:37:18,866 INFO SenderThread:11187 [file_pusher.py:finish():172] shutting down file pusher +2024-11-01 03:37:18,866 INFO SenderThread:11187 [file_pusher.py:join():178] waiting for file pusher +2024-11-01 03:37:18,866 INFO HandlerThread:11187 [handler.py:finish():869] shutting down handler +2024-11-01 03:37:18,866 INFO WriterThread:11187 [datastore.py:close():296] close: /project/wandb/run-20241101_032853-4fl9vo96/run-4fl9vo96.wandb diff --git a/wandb/run-20241101_032853-4fl9vo96/logs/debug.log b/wandb/run-20241101_032853-4fl9vo96/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..bbc7b8a9f91f716a3343d870d9ccecf272aa5406 --- /dev/null +++ b/wandb/run-20241101_032853-4fl9vo96/logs/debug.log @@ -0,0 +1,29 @@ +2024-11-01 03:28:53,256 INFO MainThread:11023 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-11-01 03:28:53,256 INFO MainThread:11023 [wandb_setup.py:_flush():76] Configure stats pid to 11023 +2024-11-01 03:28:53,256 INFO MainThread:11023 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-11-01 03:28:53,256 INFO MainThread:11023 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-11-01 03:28:53,256 INFO MainThread:11023 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'CPT en-ja'} +2024-11-01 03:28:53,256 INFO MainThread:11023 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-11-01 03:28:53,256 INFO MainThread:11023 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-11-01 03:28:53,257 INFO MainThread:11023 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20241101_032853-4fl9vo96/logs/debug.log +2024-11-01 03:28:53,257 INFO MainThread:11023 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20241101_032853-4fl9vo96/logs/debug-internal.log +2024-11-01 03:28:53,257 INFO MainThread:11023 [wandb_init.py:init():566] calling init triggers +2024-11-01 03:28:53,257 INFO MainThread:11023 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'NO_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['183224836', '/project/trans/datasets/processed_ja-en/train/documents_text_document'], 'valid_data_path': ['61084836', '/project/trans/datasets/processed_ja-en/valid/documents_text_document'], 'test_data_path': ['61084836', '/project/trans/datasets/processed_ja-en/valid/documents_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 1024, 'num_workers': 4, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'llm-jp-v3-3.7b_ja-en_actual_3M-pairs_train_2024-11-01-03:28:39', 'wandb_project': 'trans_experiment', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs', 'save': '/work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs', 'base_model': '/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 20, 'save_interval': 50, 'eval_iters': 10, 'optimizer': 'anyprecision', 'lr': 0.00015, 'lr_decay_style': 'cosine', 'lr_decay_iters': 698, 'lr_warmup_iters': 70, 'min_lr': 1.5e-05, 'train_iters': 698, 'train_samples': None, 'global_batch_size': 256, 'micro_batch_size': 1, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 131072, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-08, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 3, 'hf_repo_id': 'koichi12/llm-jp-v3-3.7b_ja-en_actual_3M-pairs', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': True, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'valid_micro_batch_size': 1, 'rank': 0, 'world_size': 2, 'padded_vocab_size': 99584, 'gradient_accumulation_steps': 128} +2024-11-01 03:28:53,257 INFO MainThread:11023 [wandb_init.py:init():616] starting backend +2024-11-01 03:28:53,257 INFO MainThread:11023 [wandb_init.py:init():620] setting up manager +2024-11-01 03:28:53,262 INFO MainThread:11023 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-01 03:28:53,263 INFO MainThread:11023 [wandb_init.py:init():628] backend started and connected +2024-11-01 03:28:53,268 INFO MainThread:11023 [wandb_init.py:init():720] updated telemetry +2024-11-01 03:28:53,280 INFO MainThread:11023 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-11-01 03:28:53,737 INFO MainThread:11023 [wandb_run.py:_on_init():2262] communicating current version +2024-11-01 03:28:53,761 INFO MainThread:11023 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.18.5 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-11-01 03:28:53,761 INFO MainThread:11023 [wandb_init.py:init():804] starting run threads in backend +2024-11-01 03:28:53,868 INFO MainThread:11023 [wandb_run.py:_console_start():2241] atexit reg +2024-11-01 03:28:53,868 INFO MainThread:11023 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-11-01 03:28:53,868 INFO MainThread:11023 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-11-01 03:28:53,868 INFO MainThread:11023 [wandb_run.py:_redirect():2186] Redirects installed. +2024-11-01 03:28:53,869 INFO MainThread:11023 [wandb_init.py:init():847] run started, returning control to user process +2024-11-01 03:29:08,187 INFO MainThread:11023 [wandb_run.py:_config_callback():1343] config_cb None None {'model_architecture': 'LlamaForCausalLM', 'activation_function': 'silu', 'hidden_size': 3072, 'model_type': 'llama', 'max_position_embeddings': 4096, 'num_attention_heads': 24, 'num_hidden_layers': 28} +2024-11-01 03:29:08,188 INFO MainThread:11023 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 2} diff --git a/wandb/run-20241101_032853-4fl9vo96/run-4fl9vo96.wandb b/wandb/run-20241101_032853-4fl9vo96/run-4fl9vo96.wandb new file mode 100644 index 0000000000000000000000000000000000000000..5f6ea0278ad5d04f9997523f3b49e5c58e73ebf2 Binary files /dev/null and b/wandb/run-20241101_032853-4fl9vo96/run-4fl9vo96.wandb differ