add pt tokenizer

Load data & train tokenizer.ipynb

@@ -44,7 +44,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 74,
+   "execution_count": 1,
    "id": "9241a429",
    "metadata": {},
    "outputs": [],
@@ -54,25 +54,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 75,
+   "execution_count": 2,
    "id": "348a4dd4",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "tokenizer = AutoTokenizer.from_pretrained(\"./\")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 81,
+   "execution_count": 4,
    "id": "595f318e",
    "metadata": {},
    "outputs": [
@@ -82,7 +74,7 @@
        "36450"
       ]
      },
-     "execution_count": 81,
+     "execution_count": 4,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -91,6 +83,58 @@
     "tokenizer.vocab_size"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 45,
+   "id": "e12a921a",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'input_ids': [4, 3620], 'attention_mask': [1, 1]}"
+      ]
+     },
+     "execution_count": 45,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tokenizer(\"<mask> word\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "441cd5af",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    ".mask_token = "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 46,
+   "id": "cbb3b0bc",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'input_ids': [4, 3620], 'attention_mask': [1, 1]}"
+      ]
+     },
+     "execution_count": 46,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tokenizer(\"<mask> word\")"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 3,

create_pt_tokenizer.py

@@ -1,3 +1,4 @@
-from transformers import AutoTokenizer
+from transformers import AutoTokenizer, AddedToken
 tokenizer = AutoTokenizer.from_pretrained("./")
+tokenizer.mask_token = AddedToken("<mask>", lstrip=True)
 tokenizer.save_pretrained("./")

special_tokens_map.json

@@ -1 +1 @@
-{"bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}, "unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}, "sep_token": {"content": "[SEP]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}, "pad_token": {"content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}, "cls_token": {"content": "[CLS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}, "mask_token": {"content": "[MASK]", "single_word": false, "lstrip": true, "rstrip": false, "normalized": false}}
+{"bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}, "unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}, "sep_token": {"content": "[SEP]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}, "pad_token": {"content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}, "cls_token": {"content": "[CLS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}, "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true}}

wandb/run-20210715_185845-dq8uirtg/files/output.log

@@ -3331,3 +3331,282 @@ Training...: 28805it [3:24:58,  1.09it/s]█████████████
 
 
 
+
+Training...: 28855it [3:25:18,  1.09it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 28905it [3:25:38,  1.09it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 28948it [3:25:45,  2.70it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 28998it [3:26:05,  2.67it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 29048it [3:26:25,  2.72it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 29098it [3:26:45,  2.68it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 29148it [3:27:05,  2.69it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+
+Training...: 29209it [3:27:39,  2.23it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 29261it [3:27:59,  3.04it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 29298it [3:28:05,  2.72it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 29348it [3:28:25,  2.68it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 29398it [3:28:45,  2.70it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 29448it [3:29:05,  2.69it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 29498it [3:29:25,  2.68it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 29563it [3:30:00,  4.11it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 29605it [3:30:19,  1.08it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 29657it [3:30:39,  1.59it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 29698it [3:30:46,  2.68it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 29748it [3:31:06,  2.71it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 29798it [3:31:26,  2.70it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 29848it [3:31:46,  2.72it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 29898it [3:32:06,  2.72it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 29948it [3:32:26,  2.69it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 29998it [3:32:46,  2.74it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 30048it [3:33:06,  2.68it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 30098it [3:33:26,  2.71it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 30148it [3:33:46,  2.69it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 30198it [3:34:06,  2.70it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 30248it [3:34:26,  2.72it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 30298it [3:34:46,  2.69it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 30348it [3:35:06,  2.72it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 30398it [3:35:26,  2.70it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 30448it [3:35:46,  2.69it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 30498it [3:36:06,  2.66it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 30548it [3:36:26,  2.67it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 30598it [3:36:46,  2.71it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 30648it [3:37:07,  2.69it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 30698it [3:37:27,  2.72it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 30748it [3:37:47,  2.68it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 30798it [3:38:07,  2.70it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 30848it [3:38:27,  2.68it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 30898it [3:38:47,  2.69it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 30948it [3:39:07,  2.73it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 30998it [3:39:27,  2.68it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 31048it [3:39:47,  2.71it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 31098it [3:40:07,  2.68it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 31148it [3:40:27,  2.70it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 31198it [3:40:47,  2.69it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 31248it [3:41:07,  2.74it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+

wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json

@@ -1 +1 @@
-{"training_step": 128800, "learning_rate": 2.4875864255591296e-05, "train_loss": 2.341568946838379, "_runtime": 12642, "_timestamp": 1626388167, "_step": 580, "eval_step": 126000, "eval_accuracy": 0.5787546038627625, "eval_loss": 2.237921953201294}
+{"training_step": 131250, "learning_rate": 2.4770188247202896e-05, "train_loss": 2.2838871479034424, "_runtime": 13625, "_timestamp": 1626389150, "_step": 629, "eval_step": 126000, "eval_accuracy": 0.5787546038627625, "eval_loss": 2.237921953201294}

wandb/run-20210715_185845-dq8uirtg/logs/debug-internal.log

@@ -7463,3 +7463,614 @@
 2021-07-15 22:29:36,104 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
 2021-07-15 22:29:36,105 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
 2021-07-15 22:29:37,880 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:29:48,034 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:29:48,035 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:29:48,035 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:29:48,825 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:29:48,885 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:29:49,886 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:29:50,886 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:29:51,235 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:29:51,235 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:29:51,887 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:29:53,888 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:29:55,889 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:29:57,890 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:30:06,376 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:30:06,377 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:30:08,103 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:30:08,103 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:30:08,104 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:30:08,894 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:30:09,894 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:30:10,895 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:30:11,895 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:30:13,896 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:30:15,897 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:30:17,898 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:30:18,905 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:30:21,509 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:30:21,510 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:30:28,142 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:30:28,142 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:30:28,143 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:30:28,902 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:30:28,903 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:30:29,903 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:30:31,903 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:30:33,904 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:30:35,905 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:30:36,640 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:30:36,640 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:30:37,906 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:30:48,194 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:30:48,195 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:30:48,195 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:30:48,910 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:30:48,910 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:30:48,986 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:30:49,911 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:30:51,772 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:30:51,773 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:30:51,911 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:30:53,912 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:30:55,913 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:30:57,913 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:31:06,907 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:31:06,908 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:31:08,233 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:31:08,233 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:31:08,234 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:31:08,917 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:31:08,917 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:31:09,918 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:31:11,918 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:31:13,919 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:31:15,920 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:31:17,921 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:31:19,064 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:31:22,040 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:31:22,040 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:31:28,269 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:31:28,269 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:31:28,270 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:31:28,925 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:31:28,925 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:31:29,925 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:31:31,926 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:31:33,927 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:31:35,928 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:31:37,206 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:31:37,206 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:31:37,929 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:31:48,313 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:31:48,313 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:31:48,314 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:31:48,933 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:31:48,933 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:31:49,142 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:31:49,933 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:31:51,934 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:31:52,339 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:31:52,339 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:31:53,935 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:31:55,936 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:31:57,937 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:32:07,475 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:32:07,475 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:32:08,395 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:32:08,396 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:32:08,396 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:32:08,941 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:32:09,942 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:32:10,942 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:32:11,942 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:32:13,943 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:32:15,944 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:32:19,220 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:32:22,607 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:32:22,608 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:32:28,432 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:32:28,433 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:32:28,433 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:32:28,949 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:32:29,950 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:32:30,950 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:32:31,950 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:32:33,951 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:32:35,952 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:32:37,739 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:32:37,739 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:32:37,953 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:32:48,483 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:32:48,483 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:32:48,483 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:32:48,957 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:32:48,957 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:32:49,298 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:32:49,957 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:32:51,958 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:32:52,870 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:32:52,870 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:32:53,959 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:32:55,960 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:32:57,961 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:33:08,001 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:33:08,002 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:33:08,521 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:33:08,521 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:33:08,521 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:33:08,966 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:33:08,966 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:33:09,966 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:33:11,967 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:33:13,968 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:33:15,969 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:33:17,970 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:33:19,374 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:33:23,135 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:33:23,136 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:33:28,559 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:33:28,559 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:33:28,560 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:33:28,974 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:33:28,974 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:33:29,975 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:33:31,975 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:33:33,976 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:33:35,977 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:33:37,978 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:33:38,266 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:33:38,267 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:33:48,613 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:33:48,614 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:33:48,614 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:33:48,982 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:33:48,982 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:33:49,452 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:33:49,983 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:33:51,984 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:33:53,401 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:33:53,401 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:33:53,985 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:33:55,985 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:33:57,986 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:34:08,531 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:34:08,532 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:34:08,663 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:34:08,663 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:34:08,664 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:34:08,990 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:34:08,991 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:34:09,991 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:34:11,992 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:34:13,992 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:34:15,993 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:34:19,530 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:34:23,703 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:34:23,703 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:34:28,699 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:34:28,699 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:34:28,700 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:34:28,998 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:34:29,999 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:34:30,999 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:34:32,000 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:34:34,000 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:34:36,001 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:34:38,002 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:34:38,833 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:34:38,833 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:34:48,735 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:34:48,736 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:34:48,736 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:34:49,006 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:34:49,607 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:34:50,007 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:34:52,008 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:34:53,962 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:34:53,963 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:34:54,009 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:34:56,009 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:34:58,010 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:35:08,783 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:35:08,783 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:35:08,783 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:35:09,015 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:35:09,095 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:35:09,095 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:35:10,015 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:35:12,016 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:35:14,017 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:35:16,018 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:35:18,018 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:35:19,686 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:35:24,233 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:35:24,234 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:35:28,818 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:35:28,819 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:35:28,819 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:35:29,023 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:35:30,023 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:35:32,024 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:35:34,025 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:35:36,026 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:35:38,027 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:35:39,364 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:35:39,364 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:35:48,891 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:35:48,891 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:35:48,891 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:35:49,031 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:35:49,763 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:35:50,031 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:35:52,032 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:35:54,033 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:35:54,530 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:35:54,531 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:35:56,034 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:35:58,034 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:36:08,929 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:36:08,929 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:36:08,930 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:36:09,039 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:36:09,740 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:36:09,740 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:36:10,040 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:36:12,040 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:36:14,041 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:36:16,042 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:36:19,841 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:36:24,904 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:36:24,904 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:36:28,966 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:36:28,966 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:36:28,967 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:36:29,047 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:36:30,048 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:36:32,049 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:36:34,049 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:36:36,050 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:36:40,034 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:36:40,035 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:36:49,003 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:36:49,003 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:36:49,004 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:36:49,056 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:36:49,912 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:36:50,056 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:36:52,057 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:36:54,058 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:36:55,165 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:36:55,165 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:36:56,058 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:37:09,039 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:37:09,040 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:37:09,040 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:37:09,064 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:37:10,064 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:37:10,310 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:37:10,310 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:37:12,065 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:37:14,066 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:37:16,066 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:37:19,980 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:37:25,452 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:37:25,453 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:37:29,079 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:37:29,080 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:37:29,080 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:37:30,072 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:37:30,072 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:37:32,073 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:37:34,073 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:37:37,075 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:37:39,076 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:37:40,584 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:37:40,585 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:37:49,119 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:37:49,119 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:37:49,119 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:37:50,051 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:37:50,081 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:37:50,081 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:37:51,081 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:37:53,082 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:37:55,083 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:37:55,719 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:37:55,719 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:37:57,084 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:37:59,084 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:38:09,161 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:38:09,162 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:38:09,162 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:38:10,088 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:38:10,089 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:38:10,879 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:38:10,880 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:38:11,089 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:38:13,090 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:38:15,091 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:38:17,092 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:38:19,092 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:38:20,126 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:38:26,024 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:38:26,024 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:38:29,201 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:38:29,201 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:38:29,202 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:38:30,097 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:38:30,097 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:38:31,097 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:38:33,098 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:38:35,099 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:38:37,100 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:38:39,101 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:38:41,155 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:38:41,155 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:38:49,246 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:38:49,246 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:38:49,247 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:38:50,106 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:38:50,106 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:38:50,203 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:38:51,106 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:38:53,107 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:38:55,108 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:38:56,287 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:38:56,287 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:38:57,108 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:39:09,286 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:39:09,287 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:39:09,287 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:39:10,114 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:39:10,114 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:39:11,114 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:39:11,422 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:39:11,422 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:39:13,115 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:39:15,116 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:39:17,117 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:39:20,282 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:39:26,559 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:39:26,560 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:39:29,325 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:39:29,325 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:39:29,326 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:39:30,122 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:39:31,123 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:39:33,124 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:39:35,125 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:39:37,126 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:39:41,700 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:39:41,700 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:39:49,363 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:39:49,364 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:39:49,364 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:39:50,131 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:39:50,365 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:39:51,132 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:39:53,132 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:39:55,133 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:39:56,832 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:39:56,833 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:39:57,134 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:39:59,135 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:40:09,402 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:40:09,403 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:40:09,403 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:40:10,139 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:40:11,140 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:40:11,970 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:40:11,970 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:40:13,141 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:40:15,142 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:40:17,143 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:40:19,144 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:40:20,463 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:40:27,103 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:40:27,103 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:40:29,440 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:40:29,440 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:40:29,441 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:40:30,149 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:40:31,149 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:40:33,150 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:40:35,151 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:40:37,152 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:40:39,153 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:40:42,307 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:40:42,308 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:40:49,519 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:40:49,519 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:40:49,520 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:40:50,157 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:40:50,539 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:40:51,158 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:40:53,159 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:40:55,159 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:40:57,160 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:40:57,494 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:40:57,494 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:40:59,161 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:41:09,616 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:41:09,617 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:41:09,618 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:41:10,165 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:41:11,165 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:41:12,624 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:41:12,624 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:41:13,166 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:41:15,167 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:41:17,168 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:41:19,169 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:41:20,618 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:41:27,754 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:41:27,755 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:41:29,669 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:41:29,669 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:41:29,670 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:41:30,173 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:41:31,173 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:41:33,174 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:41:35,175 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:41:37,176 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:41:39,177 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:41:42,886 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:41:42,886 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:41:49,725 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:41:49,725 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:41:49,726 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:41:50,181 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:41:50,696 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:41:51,182 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:41:53,183 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:41:55,184 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:41:57,184 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:41:58,017 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:41:58,017 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:42:09,776 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:42:09,776 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:42:09,777 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:42:10,189 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:42:11,190 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:42:13,150 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:42:13,150 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:42:13,191 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:42:15,191 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:42:17,192 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:42:20,774 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:42:28,282 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:42:28,282 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:42:29,826 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:42:29,826 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:42:29,826 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:42:30,197 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:42:31,197 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:42:33,198 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:42:35,199 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:42:37,200 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:42:39,201 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:42:43,418 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:42:43,418 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:42:49,861 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:42:49,862 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:42:49,862 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:42:50,205 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:42:50,851 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:42:51,206 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:42:53,207 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:42:55,207 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:42:57,208 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:42:58,552 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:42:58,552 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:42:59,209 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:43:09,897 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:43:09,898 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:43:09,898 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:43:10,214 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:43:11,214 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:43:13,215 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:43:13,686 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:43:13,686 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:43:15,216 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:43:17,217 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:43:19,218 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:43:20,930 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:43:28,817 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:43:28,818 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:43:29,934 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:43:29,934 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:43:29,935 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:43:30,222 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:43:31,223 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:43:33,223 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:43:35,224 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:43:37,225 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:43:39,226 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:43:43,951 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:43:43,952 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:43:49,971 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:43:49,971 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:43:49,972 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:43:50,230 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:43:51,008 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:43:51,231 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:43:53,232 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:43:55,233 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:43:57,233 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:43:59,091 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:43:59,091 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:43:59,234 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:44:10,006 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:44:10,007 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:44:10,007 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:44:10,239 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:44:11,239 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:44:13,240 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:44:14,220 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:44:14,221 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:44:15,241 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:44:17,242 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:44:19,243 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:44:21,085 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:44:29,354 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:44:29,354 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:44:30,044 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:44:30,045 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:44:30,046 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:44:30,248 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:44:31,248 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:44:33,249 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:44:35,250 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:44:37,251 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:44:44,487 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:44:44,487 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:44:50,085 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:44:50,086 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:44:50,086 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:44:50,256 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:44:51,163 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:44:51,256 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:44:53,257 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:44:55,258 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:44:57,259 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:44:59,618 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:44:59,618 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:45:10,131 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:45:10,132 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:45:10,132 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:45:10,263 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:45:11,263 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:45:13,264 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:45:14,750 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:45:14,750 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:45:15,265 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:45:17,266 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:45:21,242 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:45:29,881 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:45:29,881 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:45:30,179 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:45:30,179 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:45:30,180 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:45:30,271 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:45:31,272 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:45:32,272 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:45:34,273 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:45:36,274 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:45:38,275 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:45:45,014 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:45:45,015 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:45:50,227 DEBUG   SenderThread:735392 [sender.py:send():179] send: history
+2021-07-15 22:45:50,227 DEBUG   SenderThread:735392 [sender.py:send():179] send: summary
+2021-07-15 22:45:50,228 INFO    SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-15 22:45:50,280 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
+2021-07-15 22:45:51,281 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:45:51,319 DEBUG   SenderThread:735392 [sender.py:send():179] send: stats
+2021-07-15 22:45:52,281 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:45:54,282 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:45:56,283 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:45:58,284 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
+2021-07-15 22:46:00,146 DEBUG   HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-15 22:46:00,147 DEBUG   SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
+2021-07-15 22:46:00,285 INFO    Thread-8  :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log