updated run script

data/csv/default-ebd4ee978eb95743/0.0.0/6b34fb8fcf56f7c8ba51dc895bfa2bfbe43546f190a60fcf74bb5e8afdcc2317/cache-c1db679223cc6403.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:781a823f8569511e08d99f6dc2e3241476a7c17921ee61e20dacd2b0b41d4e8f
+size 38479112

data/text/default-e58e149459f95aee/0.0.0/21a506d1b2b34316b1e82d0bd79066905d846e5d7e619823c0dd338d6f1fa6ad/cache-8df22f3f486b2144.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b28b3502a59a0f2bd160e971796aa01c1b0012cbde2c5e1f0923e794544f60a0
+size 320928696

data/text/default-e58e149459f95aee/0.0.0/21a506d1b2b34316b1e82d0bd79066905d846e5d7e619823c0dd338d6f1fa6ad/dataset_info.json

@@ -0,0 +1 @@
+{"description": "", "citation": "", "homepage": "", "license": "", "features": {"text": {"dtype": "string", "_type": "Value"}}, "builder_name": "text", "config_name": "default", "version": {"version_str": "0.0.0", "major": 0, "minor": 0, "patch": 0}, "splits": {"train": {"name": "train", "num_bytes": 94417493, "num_examples": 1000000, "dataset_name": "text"}}, "download_checksums": {"/home/perk/models/SimCSE-test/data/nor_news_1998_2019_sentences_1M.txt": {"num_bytes": 91417493, "checksum": "a11b0487ea13419b7902c6890723469a3c540ec0143053e82312642509adfaae"}}, "download_size": 91417493, "dataset_size": 94417493, "size_in_bytes": 185834986}

data/text/default-e58e149459f95aee/0.0.0/21a506d1b2b34316b1e82d0bd79066905d846e5d7e619823c0dd338d6f1fa6ad/text-train.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c4d0f3761a50480d487ee5fd4c43ffc579d0158e0ada0823207dbad44e97d75
+size 94438104

result/unsup-simcse-nb-bert-bert-base-GPU/config.json

@@ -0,0 +1,30 @@
+{
+  "_name_or_path": "NbAiLab/nb-bert-base",
+  "architectures": [
+    "BertForCL"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "directionality": "bidi",
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "pooler_fc_size": 768,
+  "pooler_num_attention_heads": 12,
+  "pooler_num_fc_layers": 3,
+  "pooler_size_per_head": 128,
+  "pooler_type": "first_token_transform",
+  "position_embedding_type": "absolute",
+  "transformers_version": "4.2.1",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 119547
+}

result/unsup-simcse-nb-bert-bert-base-GPU/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0576260fed1ca8476b47283a315edd2e5abc367fb62f1c838854d607e245dc6
+size 1422952237

result/unsup-simcse-nb-bert-bert-base-GPU/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf76a994694e41be1fcbfc122885a70047addd40b472e9567f900ad7a62af982
+size 711502455

result/unsup-simcse-nb-bert-bert-base-GPU/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96ca550b8c9843603e40a470d979692c2d1273e7fe290623d06ab0a86483dd7a
+size 623

result/unsup-simcse-nb-bert-bert-base-GPU/special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

result/unsup-simcse-nb-bert-bert-base-GPU/tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": false, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "special_tokens_map_file": null, "name_or_path": "NbAiLab/nb-bert-base", "do_basic_tokenize": true, "never_split": null}

result/unsup-simcse-nb-bert-bert-base-GPU/trainer_state.json

@@ -0,0 +1,30 @@
+{
+  "best_metric": 0.6774975636337088,
+  "best_model_checkpoint": "result/unsup-simcse-nb-bert-bert-base-GPU",
+  "epoch": 0.031997952131063614,
+  "global_step": 250,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.02,
+      "eval_avg_sts": 0.6372576901841207,
+      "eval_sickr_spearman": 0.6285502192782981,
+      "eval_stsb_spearman": 0.6459651610899432,
+      "step": 125
+    },
+    {
+      "epoch": 0.03,
+      "eval_avg_sts": 0.6602932545653575,
+      "eval_sickr_spearman": 0.6430889454970062,
+      "eval_stsb_spearman": 0.6774975636337088,
+      "step": 250
+    }
+  ],
+  "max_steps": 7813,
+  "num_train_epochs": 1,
+  "total_flos": 0,
+  "trial_name": null,
+  "trial_params": null
+}

result/unsup-simcse-nb-bert-bert-base-GPU/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:797ade87b409af34b0f88b705fb64664b2dc569dfedec3ea53a78bbd0e1d29cb
+size 2095

result/unsup-simcse-nb-bert-bert-base-GPU/vocab.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe0fda7c425b48c516fc8f160d594c8022a0808447475c1a7c6d6479763f310c
+size 995526

run_sup_roberta.sh

@@ -6,7 +6,7 @@ export TPU_NUM_DEVICES=4
 export ALLOW_MULTIPLE_LIBTPU_LOAD=1
 
 python ../../SimCSE/train.py \
-    --model_name_or_path pere/roberta-base-scand-xlm \
+    --model_name_or_path xlm-roberta-base \
     --train_file data/mnli_no_for_simcse.csv \
     --output_dir result/sup-simcse-pere-roberta-scand-base \
     --num_train_epochs 3 \

run_sup_roberta_GPU.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+export ZONE=us-central2-b
+export XRT_TPU_CONFIG="localservice;0;localhost:51011"
+export TPU_NUM_DEVICES=4
+export ALLOW_MULTIPLE_LIBTPU_LOAD=1
+
+python ../../SimCSE/train.py \
+    --model_name_or_path pere/roberta-base-exp \
+    --train_file data/mnli_no_for_simcse.csv \
+    --output_dir result/sup-simcse-pere-roberta-scand-base-gpu \
+    --num_train_epochs 3 \
+    --per_device_train_batch_size 128 \
+    --learning_rate 5e-5 \
+    --max_seq_length 32 \
+    --evaluation_strategy steps \
+    --metric_for_best_model stsb_spearman \
+    --load_best_model_at_end \
+    --eval_steps 125 \
+    --pooler_type cls \
+    --overwrite_output_dir \
+    --temp 0.05 \
+    --do_train \
+    --do_eval \
+    --fp16 \
+    "$@"

run_unsup_GPU.sh

@@ -11,7 +11,7 @@ export ALLOW_MULTIPLE_LIBTPU_LOAD=1
 python3 ../../SimCSE/train.py \
     --model_name_or_path NbAiLab/nb-bert-base \
     --train_file data/nor_news_1998_2019_sentences_1M.txt \
-    --output_dir result/unsup-simcse-nb-bert-bert-base-GPU \
+    --output_dir result/unsup-simcse-nb-bert-bert-base-gpu \
     --num_train_epochs 1 \
     --per_device_train_batch_size 64 \
     --learning_rate 3e-5 \