Add files using upload-large-folder tool

scripts/yans/eval/lm-evaluation-harness/models/abeja-gpt-neox-japanese-2.7b/harness.jsquad-1.2.sh

@@ -0,0 +1,3 @@
+MODEL_ARGS="pretrained=abeja/gpt-neox-japanese-2.7b,device_map=auto,torch_dtype=auto"
+TASK="jsquad-1.2-0.2"
+python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "3" --device "cuda" --output_path "models/abeja-gpt-neox-japanese-2.7b/result.jsquad-1.2.json"

scripts/yans/eval/lm-evaluation-harness/models/abeja-gpt-neox-japanese-2.7b/harness.sh

@@ -0,0 +1,3 @@
+MODEL_ARGS="pretrained=abeja/gpt-neox-japanese-2.7b"
+TASK="jcommonsenseqa-1.1-0.2,jnli-1.1-0.2,marc_ja-1.1-0.2,jsquad-1.1-0.2,xlsum_ja"
+python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "2,3,3,3,1" --device "cuda" --output_path "models/abeja-gpt-neox-japanese-2.7b/result.json"

scripts/yans/eval/lm-evaluation-harness/models/abeja-gpt-neox-japanese-2.7b/result.json

@@ -0,0 +1,59 @@
+{
+  "results": {
+    "jcommonsenseqa-1.1-0.2": {
+      "acc": 0.20017873100983022,
+      "acc_stderr": 0.011966979264632673,
+      "acc_norm": 0.22609472743521,
+      "acc_norm_stderr": 0.012510314229861862
+    },
+    "jnli-1.1-0.2": {
+      "acc": 0.3972884141331142,
+      "acc_stderr": 0.009920570907906705,
+      "acc_norm": 0.34798685291700904,
+      "acc_norm_stderr": 0.009656917922100158
+    },
+    "marc_ja-1.1-0.2": {
+      "acc": 0.749912800837112,
+      "acc_stderr": 0.005719527388015089,
+      "acc_norm": 0.749912800837112,
+      "acc_norm_stderr": 0.005719527388015089
+    },
+    "jsquad-1.1-0.2": {
+      "exact_match": 13.665015758667266,
+      "f1": 22.909453892411364
+    },
+    "xlsum_ja": {
+      "rouge2": 6.149952794206885
+    },
+    "xwinograd_ja": {
+      "acc": 0.6037539103232534,
+      "acc_stderr": 0.01580264261655725
+    }
+  },
+  "versions": {
+    "jcommonsenseqa-1.1-0.2": 1.1,
+    "jnli-1.1-0.2": 1.1,
+    "jsquad-1.1-0.2": 1.1,
+    "marc_ja-1.1-0.2": 1.1,
+    "xlsum_ja": 1.0,
+    "xwinograd_ja": 1.0
+  },
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=abeja/gpt-neox-japanese-2.7b",
+    "num_fewshot": [
+      2,
+      3,
+      3,
+      3,
+      1,
+      0
+    ],
+    "batch_size": null,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

scripts/yans/eval/lm-evaluation-harness/models/abeja-gpt-neox-japanese-2.7b/result.jsquad-1.2.json

@@ -0,0 +1,22 @@
+{
+  "results": {
+    "jsquad-1.2-0.2": {
+      "exact_match": 15.803692030616839,
+      "f1": 25.18326978234071
+    }
+  },
+  "versions": {
+    "jsquad-1.2-0.2": 1.2
+  },
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=abeja/gpt-neox-japanese-2.7b,device_map=auto,torch_dtype=auto",
+    "num_fewshot": 3,
+    "batch_size": null,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

scripts/yans/eval/lm-evaluation-harness/models/community/cyberagent-open-calm-instruct-1b_1.3.0/README.md

@@ -0,0 +1,6 @@
+# cyberagent-open-calm-instruct-1b_1.3.0
+- This is a supervised finetuned version of the base model [`cyberagent/open-calm-1b`](https://huggingface.co/cyberagent/open-calm-1b).
+- The base model is trained on the datasets below by [Stability AI Japan](https://ja.stability.ai/).
+  - [japanese_hh-rlhf-49k](https://huggingface.co/datasets/fujiki/japanese_hh-rlhf-49k)
+  - [databricks-dolly-15k-ja](https://huggingface.co/datasets/kunishou/databricks-dolly-15k-ja)
+  - [japanese_alpaca_data](https://huggingface.co/datasets/fujiki/japanese_alpaca_data)

scripts/yans/eval/lm-evaluation-harness/models/community/cyberagent-open-calm-instruct-1b_1.3.0/harness.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -eu
+PROJECT_DIR=""
+MODEL_ARGS="pretrained=${PROJECT_DIR}/instruction_tuning/outputs/open-calm-instruct-1b_1.3.0,tokenizer=cyberagent/open-calm-1b"
+TASK="jsquad-1.1-0.3,jcommonsenseqa-1.1-0.3,jnli-1.1-0.3,marc_ja-1.1-0.3"
+python main.py \
+    --model hf-causal \
+    --model_args $MODEL_ARGS \
+    --tasks $TASK \
+    --num_fewshot "2,3,3,3" \
+    --device "cuda" \
+    --output_path "models/open-calm-instruct-1b_1.3.0/result.json"

scripts/yans/eval/lm-evaluation-harness/models/community/cyberagent-open-calm-instruct-1b_1.3.0/result.json

@@ -0,0 +1,48 @@
+{
+  "results": {
+    "jcommonsenseqa-1.1-0.3": {
+      "acc": 0.7015192135835567,
+      "acc_stderr": 0.013685386698397504,
+      "acc_norm": 0.6255585344057194,
+      "acc_norm_stderr": 0.014474549079455518
+    },
+    "jnli-1.1-0.3": {
+      "acc": 0.3011503697617091,
+      "acc_stderr": 0.00930063317508552,
+      "acc_norm": 0.25842235004108466,
+      "acc_norm_stderr": 0.008875080429298606
+    },
+    "marc_ja-1.1-0.3": {
+      "acc": 0.877431906614786,
+      "acc_stderr": 0.004361701432875794,
+      "acc_norm": 0.877431906614786,
+      "acc_norm_stderr": 0.004361701432875794
+    },
+    "jsquad-1.1-0.3": {
+      "exact_match": 35.929761368752814,
+      "f1": 45.27144783040928
+    }
+  },
+  "versions": {
+    "jcommonsenseqa-1.1-0.3": 1.1,
+    "jnli-1.1-0.3": 1.1,
+    "jsquad-1.1-0.3": 1.1,
+    "marc_ja-1.1-0.3": 1.1
+  },
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=${PROJECT_DIR}/instruction_tuning/outputs/open-calm-instruct-1b_1.3.0,tokenizer=cyberagent/open-calm-1b",
+    "num_fewshot": [
+      2,
+      3,
+      3,
+      3
+    ],
+    "batch_size": null,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

scripts/yans/eval/lm-evaluation-harness/models/community/cyberagent-open-calm-instruct-3b_1.3.0/README.md

@@ -0,0 +1,6 @@
+# cyberagent-open-calm-instruct-3b_1.3.0
+- This is a supervised finetuned version of the base model [`cyberagent/open-calm-3b`](https://huggingface.co/cyberagent/open-calm-3b).
+- The base model is trained on the datasets below by [Stability AI Japan](https://ja.stability.ai/).
+  - [japanese_hh-rlhf-49k](https://huggingface.co/datasets/fujiki/japanese_hh-rlhf-49k)
+  - [databricks-dolly-15k-ja](https://huggingface.co/datasets/kunishou/databricks-dolly-15k-ja)
+  - [japanese_alpaca_data](https://huggingface.co/datasets/fujiki/japanese_alpaca_data)

scripts/yans/eval/lm-evaluation-harness/models/community/cyberagent-open-calm-instruct-3b_1.3.0/harness.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -eu
+PROJECT_DIR=""
+MODEL_ARGS="pretrained=${PROJECT_DIR}/instruction_tuning/outputs/open-calm-instruct-3b_1.3.0,tokenizer=cyberagent/open-calm-3b"
+TASK="jcommonsenseqa-1.1-0.3,jnli-1.1-0.3,marc_ja-1.1-0.3,jsquad-1.1-0.3,jaqket_v2-0.1-0.3,xlsum_ja-1.0-0.3,xwinograd_ja,mgsm-1.0-0.3"
+python main.py \
+    --model hf-causal \
+    --model_args $MODEL_ARGS \
+    --tasks $TASK \
+    --num_fewshot "3,3,3,2,1,1,0,5" \
+    --device "cuda" \
+    --output_path "models/community/cyberagent-open-calm-instruct-3b_1.3.0/result.json"

scripts/yans/eval/lm-evaluation-harness/models/line-corporation/line-corporation-japanese-large-lm-1.7b-instruction-sft/harness.sh

@@ -0,0 +1,3 @@
+MODEL_ARGS="pretrained=line-corporation/japanese-large-lm-1.7b-instruction-sft,use_fast=False,device_map=auto,torch_dtype=auto"
+TASK="jcommonsenseqa-1.1-0.5,jnli-1.1-0.5,marc_ja-1.1-0.5,jsquad-1.1-0.5,jaqket_v2-0.2-0.5,xlsum_ja-1.0-0.5,xwinograd_ja,mgsm-1.0-0.5"
+python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "3,3,3,2,1,1,0,5" --device "cuda" --output_path "models/line-corporation/line-corporation-japanese-large-lm-1.7b-instruction-sft/result.json"

scripts/yans/eval/lm-evaluation-harness/models/line-corporation/line-corporation-japanese-large-lm-1.7b-instruction-sft/result.json

@@ -0,0 +1,71 @@
+{
+  "results": {
+    "jcommonsenseqa-1.1-0.5": {
+      "acc": 0.22430741733690795,
+      "acc_stderr": 0.012475148816050531,
+      "acc_norm": 0.23681858802502234,
+      "acc_norm_stderr": 0.01271454677969028
+    },
+    "jnli-1.1-0.5": {
+      "acc": 0.34346754313886607,
+      "acc_stderr": 0.009627197865307401,
+      "acc_norm": 0.3011503697617091,
+      "acc_norm_stderr": 0.009300633175085522
+    },
+    "marc_ja-1.1-0.5": {
+      "acc": 0.8036788114609126,
+      "acc_stderr": 0.005283057698929343,
+      "acc_norm": 0.8036788114609126,
+      "acc_norm_stderr": 0.005283057698929343
+    },
+    "xwinograd_ja": {
+      "acc": 0.6329509906152242,
+      "acc_stderr": 0.015572714283682185
+    },
+    "jsquad-1.1-0.5": {
+      "exact_match": 30.977037370553806,
+      "f1": 48.12415333506568
+    },
+    "jaqket_v2-0.2-0.5": {
+      "exact_match": 25.257731958762886,
+      "f1": 40.58191140665372
+    },
+    "xlsum_ja-1.0-0.5": {
+      "rouge2": 1.0385441084792033
+    },
+    "mgsm-1.0-0.5": {
+      "acc": 0.016,
+      "acc_stderr": 0.007951661188874354
+    }
+  },
+  "versions": {
+    "jcommonsenseqa-1.1-0.5": 1.1,
+    "jnli-1.1-0.5": 1.1,
+    "marc_ja-1.1-0.5": 1.1,
+    "jsquad-1.1-0.5": 1.1,
+    "jaqket_v2-0.2-0.5": 0.2,
+    "xlsum_ja-1.0-0.5": 1.0,
+    "xwinograd_ja": 1.0,
+    "mgsm-1.0-0.5": 1.0
+  },
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=line-corporation/japanese-large-lm-1.7b-instruction-sft,use_fast=False,device_map=auto,torch_dtype=auto",
+    "num_fewshot": [
+      3,
+      3,
+      3,
+      2,
+      1,
+      1,
+      0,
+      5
+    ],
+    "batch_size": null,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

scripts/yans/eval/lm-evaluation-harness/models/line-corporation/line-corporation-japanese-large-lm-1.7b/harness.sh

@@ -0,0 +1,3 @@
+MODEL_ARGS="pretrained=line-corporation/japanese-large-lm-1.7b,use_fast=False,device_map=auto,torch_dtype=auto"
+TASK="jcommonsenseqa-1.1-0.2,jnli-1.1-0.2,marc_ja-1.1-0.2,jsquad-1.1-0.2,jaqket_v2-0.2-0.2,xlsum_ja,xwinograd_ja,mgsm"
+python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "3,3,3,2,1,1,0,5" --device "cuda" --output_path "models/line-corporation/line-corporation-japanese-large-lm-1.7b/result.json"

scripts/yans/eval/lm-evaluation-harness/models/line-corporation/line-corporation-japanese-large-lm-1.7b/result.json

@@ -0,0 +1,71 @@
+{
+  "results": {
+    "jcommonsenseqa-1.1-0.2": {
+      "acc": 0.30831099195710454,
+      "acc_stderr": 0.013811124479483034,
+      "acc_norm": 0.26005361930294907,
+      "acc_norm_stderr": 0.013119300343161644
+    },
+    "jnli-1.1-0.2": {
+      "acc": 0.35949055053410023,
+      "acc_stderr": 0.009728266419780814,
+      "acc_norm": 0.300328677074774,
+      "acc_norm_stderr": 0.00929339473482123
+    },
+    "marc_ja-1.1-0.2": {
+      "acc": 0.745136186770428,
+      "acc_stderr": 0.005796054001130057,
+      "acc_norm": 0.745136186770428,
+      "acc_norm_stderr": 0.005796054001130057
+    },
+    "xwinograd_ja": {
+      "acc": 0.6611053180396246,
+      "acc_stderr": 0.015292727421996942
+    },
+    "jsquad-1.1-0.2": {
+      "exact_match": 56.55110310670869,
+      "f1": 69.46989310703984
+    },
+    "jaqket_v2-0.2-0.2": {
+      "exact_match": 52.06185567010309,
+      "f1": 60.433303332787865
+    },
+    "xlsum_ja": {
+      "rouge2": 8.408787633129647
+    },
+    "mgsm": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    }
+  },
+  "versions": {
+    "jcommonsenseqa-1.1-0.2": 1.1,
+    "jnli-1.1-0.2": 1.1,
+    "marc_ja-1.1-0.2": 1.1,
+    "jsquad-1.1-0.2": 1.1,
+    "jaqket_v2-0.2-0.2": 0.2,
+    "xlsum_ja": 1.0,
+    "xwinograd_ja": 1.0,
+    "mgsm": 1.0
+  },
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=line-corporation/japanese-large-lm-1.7b,use_fast=False,device_map=auto,torch_dtype=auto",
+    "num_fewshot": [
+      3,
+      3,
+      3,
+      2,
+      1,
+      1,
+      0,
+      5
+    ],
+    "batch_size": null,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

scripts/yans/eval/lm-evaluation-harness/models/line-corporation/line-corporation-japanese-large-lm-3.6b-instruction-sft/harness.sh

@@ -0,0 +1,3 @@
+MODEL_ARGS="pretrained=line-corporation/japanese-large-lm-3.6b-instruction-sft,use_fast=False,device_map=auto,torch_dtype=auto"
+TASK="jcommonsenseqa-1.1-0.5,jnli-1.1-0.5,marc_ja-1.1-0.5,jsquad-1.1-0.5,jaqket_v2-0.2-0.5,xlsum_ja-1.0-0.5,xwinograd_ja,mgsm-1.0-0.5"
+python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "3,3,3,2,1,1,0,5" --device "cuda" --output_path "models/line-corporation/line-corporation-japanese-large-lm-3.6b-instruction-sft/result.json"

scripts/yans/eval/lm-evaluation-harness/models/line-corporation/line-corporation-japanese-large-lm-3.6b-instruction-sft/result.json

@@ -0,0 +1,71 @@
+{
+  "results": {
+    "jcommonsenseqa-1.1-0.5": {
+      "acc": 0.3360142984807864,
+      "acc_stderr": 0.014126590011265207,
+      "acc_norm": 0.26720285969615726,
+      "acc_norm_stderr": 0.013234012242081952
+    },
+    "jnli-1.1-0.5": {
+      "acc": 0.4256368118323747,
+      "acc_stderr": 0.010024017935515625,
+      "acc_norm": 0.3019720624486442,
+      "acc_norm_stderr": 0.009307836171755053
+    },
+    "marc_ja-1.1-0.5": {
+      "acc": 0.5509373894587902,
+      "acc_stderr": 0.006615536639080702,
+      "acc_norm": 0.5509373894587902,
+      "acc_norm_stderr": 0.006615536639080702
+    },
+    "xwinograd_ja": {
+      "acc": 0.6465067778936392,
+      "acc_stderr": 0.015445228301221386
+    },
+    "jsquad-1.1-0.5": {
+      "exact_match": 44.371904547501124,
+      "f1": 59.516773934435584
+    },
+    "jaqket_v2-0.2-0.5": {
+      "exact_match": 39.86254295532646,
+      "f1": 51.98299576521227
+    },
+    "xlsum_ja-1.0-0.5": {
+      "rouge2": 6.577976426409143
+    },
+    "mgsm-1.0-0.5": {
+      "acc": 0.024,
+      "acc_stderr": 0.009699087026964249
+    }
+  },
+  "versions": {
+    "jcommonsenseqa-1.1-0.5": 1.1,
+    "jnli-1.1-0.5": 1.1,
+    "marc_ja-1.1-0.5": 1.1,
+    "jsquad-1.1-0.5": 1.1,
+    "jaqket_v2-0.2-0.5": 0.2,
+    "xlsum_ja-1.0-0.5": 1.0,
+    "xwinograd_ja": 1.0,
+    "mgsm-1.0-0.5": 1.0
+  },
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=line-corporation/japanese-large-lm-3.6b-instruction-sft,use_fast=False,device_map=auto,torch_dtype=auto",
+    "num_fewshot": [
+      3,
+      3,
+      3,
+      2,
+      1,
+      1,
+      0,
+      5
+    ],
+    "batch_size": null,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

scripts/yans/eval/lm-evaluation-harness/models/line-corporation/line-corporation-japanese-large-lm-3.6b/harness.sh

@@ -0,0 +1,3 @@
+MODEL_ARGS="pretrained=line-corporation/japanese-large-lm-3.6b,use_fast=False,device_map=auto,torch_dtype=auto"
+TASK="jcommonsenseqa-1.1-0.2,jnli-1.1-0.2,marc_ja-1.1-0.2,jsquad-1.1-0.2,jaqket_v2-0.2-0.2,xlsum_ja,xwinograd_ja,mgsm"
+python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "3,3,3,2,1,1,0,5" --device "cuda" --output_path "models/line-corporation/line-corporation-japanese-large-lm-3.6b/result.json"

scripts/yans/eval/lm-evaluation-harness/models/line-corporation/line-corporation-japanese-large-lm-3.6b/result.json

@@ -0,0 +1,71 @@
+{
+  "results": {
+    "jcommonsenseqa-1.1-0.2": {
+      "acc": 0.24039320822162646,
+      "acc_stderr": 0.01278011066769292,
+      "acc_norm": 0.2421805183199285,
+      "acc_norm_stderr": 0.0128124322893179
+    },
+    "jnli-1.1-0.2": {
+      "acc": 0.29950698438783896,
+      "acc_stderr": 0.009286120768078254,
+      "acc_norm": 0.30156121610517667,
+      "acc_norm_stderr": 0.009304239098715018
+    },
+    "marc_ja-1.1-0.2": {
+      "acc": 0.7939511850017686,
+      "acc_stderr": 0.005379506895071017,
+      "acc_norm": 0.7939511850017686,
+      "acc_norm_stderr": 0.005379506895071017
+    },
+    "xwinograd_ja": {
+      "acc": 0.7028154327424401,
+      "acc_stderr": 0.014765597190000436
+    },
+    "jsquad-1.1-0.2": {
+      "exact_match": 62.26924808644755,
+      "f1": 74.52057820837234
+    },
+    "jaqket_v2-0.2-0.2": {
+      "exact_match": 67.18213058419244,
+      "f1": 74.29659878113482
+    },
+    "xlsum_ja": {
+      "rouge2": 8.610239752200977
+    },
+    "mgsm": {
+      "acc": 0.028,
+      "acc_stderr": 0.010454721651927288
+    }
+  },
+  "versions": {
+    "jcommonsenseqa-1.1-0.2": 1.1,
+    "jnli-1.1-0.2": 1.1,
+    "marc_ja-1.1-0.2": 1.1,
+    "jsquad-1.1-0.2": 1.1,
+    "jaqket_v2-0.2-0.2": 0.2,
+    "xlsum_ja": 1.0,
+    "xwinograd_ja": 1.0,
+    "mgsm": 1.0
+  },
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=line-corporation/japanese-large-lm-3.6b,use_fast=False,device_map=auto,torch_dtype=auto",
+    "num_fewshot": [
+      3,
+      3,
+      3,
+      2,
+      1,
+      1,
+      0,
+      5
+    ],
+    "batch_size": null,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

scripts/yans/eval/lm-evaluation-harness/models/rinna/harness.conf

@@ -0,0 +1,4 @@
+[DEFAULT]
+# Recent Rinna models use the 0.4 prompt, though note that older ones used
+# other prompts.
+prompt = 0.4

scripts/yans/eval/lm-evaluation-harness/models/rinna/rinna-bilingual-gpt-neox-4b-instruction-sft/result.json

@@ -0,0 +1,71 @@
+{
+  "results": {
+    "jcommonsenseqa-1.1-0.5": {
+      "acc": 0.49508489722966936,
+      "acc_stderr": 0.014952992585674197,
+      "acc_norm": 0.4941912421805183,
+      "acc_norm_stderr": 0.014952705953248754
+    },
+    "jnli-1.1-0.5": {
+      "acc": 0.47082990961380444,
+      "acc_stderr": 0.010119489683056362,
+      "acc_norm": 0.45028759244042726,
+      "acc_norm_stderr": 0.010086528162038566
+    },
+    "marc_ja-1.1-0.5": {
+      "acc": 0.9527767951892465,
+      "acc_stderr": 0.0028211996518060353,
+      "acc_norm": 0.9527767951892465,
+      "acc_norm_stderr": 0.0028211996518060353
+    },
+    "xwinograd_ja": {
+      "acc": 0.6465067778936392,
+      "acc_stderr": 0.015445228301221378
+    },
+    "jsquad-1.1-0.5": {
+      "exact_match": 55.9882935614588,
+      "f1": 70.04345164121641
+    },
+    "jaqket_v2-0.1-0.5": {
+      "exact_match": 61.16838487972509,
+      "f1": 65.03049022378916
+    },
+    "xlsum_ja-1.0-0.5": {
+      "rouge2": 5.506882882949979
+    },
+    "mgsm-1.0-0.5": {
+      "acc": 0.028,
+      "acc_stderr": 0.010454721651927302
+    }
+  },
+  "versions": {
+    "jcommonsenseqa-1.1-0.5": 1.1,
+    "jnli-1.1-0.5": 1.1,
+    "marc_ja-1.1-0.5": 1.1,
+    "jsquad-1.1-0.5": 1.1,
+    "jaqket_v2-0.1-0.5": 0.1,
+    "xlsum_ja-1.0-0.5": 1.0,
+    "xwinograd_ja": 1.0,
+    "mgsm-1.0-0.5": 1.0
+  },
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=rinna/bilingual-gpt-neox-4b-instruction-sft,use_fast=False,device_map=auto,torch_dtype=auto",
+    "num_fewshot": [
+      3,
+      3,
+      3,
+      2,
+      1,
+      1,
+      0,
+      5
+    ],
+    "batch_size": null,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

scripts/yans/eval/lm-evaluation-harness/models/rinna/rinna-bilingual-gpt-neox-4b/harness.jsquad-1.2.sh

@@ -0,0 +1,3 @@
+MODEL_ARGS="pretrained=rinna/bilingual-gpt-neox-4b,use_fast=False,device_map=auto,torch_dtype=auto"
+TASK="jsquad-1.2-0.2"
+python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "2" --device "cuda" --output_path "models/rinna/rinna-bilingual-gpt-neox-4b/result.jsquad-1.2.json"

scripts/yans/eval/lm-evaluation-harness/models/rinna/rinna-bilingual-gpt-neox-4b/harness.sh

@@ -0,0 +1,3 @@
+MODEL_ARGS="pretrained=rinna/bilingual-gpt-neox-4b,use_fast=False,device_map=auto,torch_dtype=auto"
+TASK="jcommonsenseqa-1.1-0.2,jnli-1.1-0.2,marc_ja-1.1-0.2,jsquad-1.1-0.2,jaqket_v2-0.1-0.2,xlsum_ja,xwinograd_ja,mgsm"
+python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "3,3,3,2,1,1,0,5" --device "cuda" --output_path "models/rinna/rinna-bilingual-gpt-neox-4b/result.json"

scripts/yans/eval/lm-evaluation-harness/models/rinna/rinna-bilingual-gpt-neox-4b/result.json

@@ -0,0 +1,71 @@
+{
+  "results": {
+    "jcommonsenseqa-1.1-0.2": {
+      "acc": 0.20822162645218945,
+      "acc_stderr": 0.01214349876971715,
+      "acc_norm": 0.22788203753351208,
+      "acc_norm_stderr": 0.012545153313075156
+    },
+    "jnli-1.1-0.2": {
+      "acc": 0.5521774856203779,
+      "acc_stderr": 0.010081409479626453,
+      "acc_norm": 0.5304026294165982,
+      "acc_norm_stderr": 0.01011799843670741
+    },
+    "marc_ja-1.1-0.2": {
+      "acc": 0.5955076052352317,
+      "acc_stderr": 0.0065276873249124285,
+      "acc_norm": 0.5955076052352317,
+      "acc_norm_stderr": 0.0065276873249124285
+    },
+    "xwinograd_ja": {
+      "acc": 0.6642335766423357,
+      "acc_stderr": 0.01525795361580425
+    },
+    "jsquad-1.1-0.2": {
+      "exact_match": 50.78793336334984,
+      "f1": 61.684710792645284
+    },
+    "jaqket_v2-0.1-0.2": {
+      "exact_match": 59.450171821305844,
+      "f1": 65.22394415435645
+    },
+    "xlsum_ja": {
+      "rouge2": 5.54788534415756
+    },
+    "mgsm": {
+      "acc": 0.024,
+      "acc_stderr": 0.009699087026964261
+    }
+  },
+  "versions": {
+    "jcommonsenseqa-1.1-0.2": 1.1,
+    "jnli-1.1-0.2": 1.1,
+    "marc_ja-1.1-0.2": 1.1,
+    "jsquad-1.1-0.2": 1.1,
+    "jaqket_v2-0.1-0.2": 0.1,
+    "xlsum_ja": 1.0,
+    "xwinograd_ja": 1.0,
+    "mgsm": 1.0
+  },
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=rinna/bilingual-gpt-neox-4b,use_fast=False,device_map=auto,torch_dtype=auto",
+    "num_fewshot": [
+      3,
+      3,
+      3,
+      2,
+      1,
+      1,
+      0,
+      5
+    ],
+    "batch_size": null,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

scripts/yans/eval/lm-evaluation-harness/models/rinna/rinna-bilingual-gpt-neox-4b/result.jsquad-1.2.json

@@ -0,0 +1,22 @@
+{
+  "results": {
+    "jsquad-1.2-0.2": {
+      "exact_match": 51.32823052678973,
+      "f1": 61.9390389728309
+    }
+  },
+  "versions": {
+    "jsquad-1.2-0.2": 1.2
+  },
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=rinna/bilingual-gpt-neox-4b,use_fast=False,device_map=auto,torch_dtype=auto",
+    "num_fewshot": 2,
+    "batch_size": null,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

scripts/yans/eval/lm-evaluation-harness/models/rinna/rinna-japanese-gpt-1b/harness.jsquad-1.2.sh

@@ -0,0 +1,3 @@
+MODEL_ARGS="pretrained=rinna/japanese-gpt-1b,use_fast=False"
+TASK="jsquad-1.2-0.2"
+python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "2" --device "cuda" --output_path "models/rinna/rinna-japanese-gpt-1b/result.jsquad-1.2.json"

scripts/yans/eval/lm-evaluation-harness/models/rinna/rinna-japanese-gpt-1b/harness.sh

@@ -0,0 +1,3 @@
+MODEL_ARGS="pretrained=rinna/japanese-gpt-1b,use_fast=False"
+TASK="jcommonsenseqa-1.1-0.2,jnli-1.1-0.2,marc_ja-1.1-0.2,jsquad-1.1-0.2,jaqket_v2-0.1-0.2,xlsum_ja,xwinograd_ja,mgsm"
+python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "3,3,3,2,1,1,0,5" --device "cuda" --output_path "models/rinna/rinna-japanese-gpt-1b/result.json"

scripts/yans/eval/lm-evaluation-harness/models/rinna/rinna-japanese-gpt-1b/result.json

@@ -0,0 +1,71 @@
+{
+  "results": {
+    "jcommonsenseqa-1.1-0.2": {
+      "acc": 0.34763181411974975,
+      "acc_stderr": 0.014242467674129443,
+      "acc_norm": 0.257372654155496,
+      "acc_norm_stderr": 0.013075122531072186
+    },
+    "jnli-1.1-0.2": {
+      "acc": 0.37674609695973704,
+      "acc_stderr": 0.009823942907406482,
+      "acc_norm": 0.3011503697617091,
+      "acc_norm_stderr": 0.009300633175085522
+    },
+    "marc_ja-1.1-0.2": {
+      "acc": 0.8786187652598535,
+      "acc_stderr": 0.0043130554527802374,
+      "acc_norm": 0.8786187652598535,
+      "acc_norm_stderr": 0.0043130554527802374
+    },
+    "xwinograd_ja": {
+      "acc": 0.6454640250260688,
+      "acc_stderr": 0.015455512877686553
+    },
+    "jsquad-1.1-0.2": {
+      "exact_match": 26.181900045024765,
+      "f1": 44.67532835280053
+    },
+    "jaqket_v2-0.1-0.2": {
+      "exact_match": 37.02749140893471,
+      "f1": 57.99059569678122
+    },
+    "xlsum_ja": {
+      "rouge2": 5.335027032779865
+    },
+    "mgsm": {
+      "acc": 0.02,
+      "acc_stderr": 0.008872139507342681
+    }
+  },
+  "versions": {
+    "jcommonsenseqa-1.1-0.2": 1.1,
+    "jnli-1.1-0.2": 1.1,
+    "marc_ja-1.1-0.2": 1.1,
+    "jsquad-1.1-0.2": 1.1,
+    "jaqket_v2-0.1-0.2": 0.1,
+    "xlsum_ja": 1.0,
+    "xwinograd_ja": 1.0,
+    "mgsm": 1.0
+  },
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=rinna/japanese-gpt-1b,use_fast=False",
+    "num_fewshot": [
+      3,
+      3,
+      3,
+      2,
+      1,
+      1,
+      0,
+      5
+    ],
+    "batch_size": null,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

scripts/yans/eval/lm-evaluation-harness/models/rinna/rinna-japanese-gpt-1b/result.jsquad-1.2.json

@@ -0,0 +1,22 @@
+{
+  "results": {
+    "jsquad-1.2-0.2": {
+      "exact_match": 30.189104007203962,
+      "f1": 47.12467642283419
+    }
+  },
+  "versions": {
+    "jsquad-1.2-0.2": 1.2
+  },
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=rinna/japanese-gpt-1b,use_fast=False",
+    "num_fewshot": 2,
+    "batch_size": null,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

scripts/yans/eval/lm-evaluation-harness/models/rinna/rinna-japanese-gpt-neox-3.6b-instruction-ppo/harness.jsquad-1.2.sh

@@ -0,0 +1,3 @@
+MODEL_ARGS="pretrained=rinna/japanese-gpt-neox-3.6b-instruction-ppo,use_fast=False,device_map=auto,torch_dtype=auto"
+TASK="jsquad-1.2-0.4"
+python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "2" --device "cuda" --output_path "models/rinna/rinna-japanese-gpt-neox-3.6b-instruction-ppo/result.jsquad-1.2.json"

scripts/yans/eval/lm-evaluation-harness/models/rinna/rinna-japanese-gpt-neox-3.6b-instruction-ppo/harness.sh

@@ -0,0 +1,3 @@
+MODEL_ARGS="pretrained=rinna/japanese-gpt-neox-3.6b-instruction-ppo,use_fast=False,device_map=auto,torch_dtype=auto"
+TASK="jcommonsenseqa-1.1-0.4,jnli-1.1-0.4,marc_ja-1.1-0.4,jsquad-1.1-0.4,jaqket_v2-0.1-0.4,xlsum_ja-1.0-0.4,xwinograd_ja,mgsm-1.0-0.4"
+python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "3,3,3,2,1,1,0,5" --device "cuda" --output_path "models/rinna/rinna-japanese-gpt-neox-3.6b-instruction-ppo/result.json"

scripts/yans/eval/lm-evaluation-harness/models/rinna/rinna-japanese-gpt-neox-3.6b-instruction-ppo/result.json

@@ -0,0 +1,71 @@
+{
+  "results": {
+    "jcommonsenseqa-1.1-0.4": {
+      "acc": 0.44057193923145666,
+      "acc_stderr": 0.014847715520097282,
+      "acc_norm": 0.4226988382484361,
+      "acc_norm_stderr": 0.014773923335599326
+    },
+    "jnli-1.1-0.4": {
+      "acc": 0.5419063270336894,
+      "acc_stderr": 0.01010108912658305,
+      "acc_norm": 0.5312243221035333,
+      "acc_norm_stderr": 0.01011696986287914
+    },
+    "marc_ja-1.1-0.4": {
+      "acc": 0.8960585978374608,
+      "acc_stderr": 0.004030616889059545,
+      "acc_norm": 0.8960585978374608,
+      "acc_norm_stderr": 0.004030616889059545
+    },
+    "xwinograd_ja": {
+      "acc": 0.6913451511991658,
+      "acc_stderr": 0.014924550437257583
+    },
+    "jsquad-1.1-0.4": {
+      "exact_match": 51.62089149031968,
+      "f1": 63.676339985467465
+    },
+    "jaqket_v2-0.1-0.4": {
+      "exact_match": 50.945017182130584,
+      "f1": 55.79263424624247
+    },
+    "xlsum_ja-1.0-0.4": {
+      "rouge2": 6.633741717885442
+    },
+    "mgsm-1.0-0.4": {
+      "acc": 0.044,
+      "acc_stderr": 0.012997373846574957
+    }
+  },
+  "versions": {
+    "jcommonsenseqa-1.1-0.4": 1.1,
+    "jnli-1.1-0.4": 1.1,
+    "marc_ja-1.1-0.4": 1.1,
+    "jsquad-1.1-0.4": 1.1,
+    "jaqket_v2-0.1-0.4": 0.1,
+    "xlsum_ja-1.0-0.4": 1.0,
+    "xwinograd_ja": 1.0,
+    "mgsm-1.0-0.4": 1.0
+  },
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=rinna/japanese-gpt-neox-3.6b-instruction-ppo,use_fast=False,device_map=auto,torch_dtype=auto",
+    "num_fewshot": [
+      3,
+      3,
+      3,
+      2,
+      1,
+      1,
+      0,
+      5
+    ],
+    "batch_size": null,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

scripts/yans/eval/lm-evaluation-harness/models/rinna/rinna-japanese-gpt-neox-3.6b-instruction-ppo/result.jsquad-1.2.json

@@ -0,0 +1,22 @@
+{
+  "results": {
+    "jsquad-1.2-0.4": {
+      "exact_match": 52.633948671769474,
+      "f1": 64.387511749343
+    }
+  },
+  "versions": {
+    "jsquad-1.2-0.4": 1.2
+  },
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=rinna/japanese-gpt-neox-3.6b-instruction-ppo,use_fast=False,device_map=auto,torch_dtype=auto",
+    "num_fewshot": 2,
+    "batch_size": null,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

scripts/yans/eval/lm-evaluation-harness/models/rinna/rinna-japanese-gpt-neox-3.6b/harness.jsquad-1.2.sh

@@ -0,0 +1,3 @@
+MODEL_ARGS="pretrained=rinna/japanese-gpt-neox-3.6b,use_fast=False,device_map=auto,torch_dtype=auto"
+TASK="jsquad-1.2-0.2"
+python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "2" --device "cuda" --output_path "models/rinna/rinna-japanese-gpt-neox-3.6b/result.jsquad-1.2.json"

scripts/yans/eval/lm-evaluation-harness/models/rinna/rinna-japanese-gpt-neox-3.6b/harness.sh

@@ -0,0 +1,3 @@
+MODEL_ARGS="pretrained=rinna/japanese-gpt-neox-3.6b,use_fast=False"
+TASK="jcommonsenseqa-1.1-0.2,jnli-1.1-0.2,marc_ja-1.1-0.2,jsquad-1.1-0.2,jaqket_v2-0.1-0.2,xlsum_ja,xwinograd_ja,mgsm"
+python main.py --model hf-causal --model_args $MODEL_ARGS --tasks $TASK --num_fewshot "3,3,3,2,1,1,0,5" --device "cuda" --output_path "models/rinna/rinna-japanese-gpt-neox-3.6b/result.json"

scripts/yans/eval/lm-evaluation-harness/models/rinna/rinna-japanese-gpt-neox-3.6b/result.json

@@ -0,0 +1,71 @@
+{
+  "results": {
+    "jcommonsenseqa-1.1-0.2": {
+      "acc": 0.3163538873994638,
+      "acc_stderr": 0.013908534121227658,
+      "acc_norm": 0.2725647899910634,
+      "acc_norm_stderr": 0.01331714516405031
+    },
+    "jnli-1.1-0.2": {
+      "acc": 0.3442892358258012,
+      "acc_stderr": 0.009632673153167076,
+      "acc_norm": 0.3311421528348398,
+      "acc_norm_stderr": 0.009541202050062205
+    },
+    "marc_ja-1.1-0.2": {
+      "acc": 0.7481688175793513,
+      "acc_stderr": 0.005732757658862212,
+      "acc_norm": 0.7481688175793513,
+      "acc_norm_stderr": 0.005732757658862212
+    },
+    "xwinograd_ja": {
+      "acc": 0.708029197080292,
+      "acc_stderr": 0.014689686963716971
+    },
+    "jsquad-1.1-0.2": {
+      "exact_match": 47.90634849167042,
+      "f1": 58.804568288439675
+    },
+    "jaqket_v2-0.1-0.2": {
+      "exact_match": 68.38487972508591,
+      "f1": 72.4344388906244
+    },
+    "xlsum_ja": {
+      "rouge2": 5.157849646982534
+    },
+    "mgsm": {
+      "acc": 0.012,
+      "acc_stderr": 0.006900323023694271
+    }
+  },
+  "versions": {
+    "jcommonsenseqa-1.1-0.2": 1.1,
+    "jnli-1.1-0.2": 1.1,
+    "marc_ja-1.1-0.2": 1.1,
+    "jsquad-1.1-0.2": 1.1,
+    "jaqket_v2-0.1-0.2": 0.1,
+    "xlsum_ja": 1.0,
+    "xwinograd_ja": 1.0,
+    "mgsm": 1.0
+  },
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=rinna/japanese-gpt-neox-3.6b,use_fast=False",
+    "num_fewshot": [
+      3,
+      3,
+      3,
+      2,
+      1,
+      1,
+      0,
+      5
+    ],
+    "batch_size": null,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

scripts/yans/eval/lm-evaluation-harness/models/rinna/rinna-japanese-gpt-neox-3.6b/result.jsquad-1.2.json

@@ -0,0 +1,22 @@
+{
+  "results": {
+    "jsquad-1.2-0.2": {
+      "exact_match": 49.0094552003602,
+      "f1": 59.80363888369063
+    }
+  },
+  "versions": {
+    "jsquad-1.2-0.2": 1.2
+  },
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=rinna/japanese-gpt-neox-3.6b,use_fast=False,device_map=auto,torch_dtype=auto",
+    "num_fewshot": 2,
+    "batch_size": null,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

scripts/yans/eval/lm-evaluation-harness/models/stabilityai/experiments/stablelm-jp-1b-compact-v1/harness.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -eu
+PROJECT_DIR="/fsx/proj-jp-stablegpt"
+MODEL_ARGS="pretrained=${PROJECT_DIR}/hf_model/1b-compact-v1,tokenizer=${PROJECT_DIR}/tokenizers/compact-hf/,use_fast=False"
+TASK="jcommonsenseqa-1.1-0.2,jnli-1.1-0.2,marc_ja-1.1-0.2,jsquad-1.1-0.2,jaqket_v2-0.1-0.2,xlsum_ja,xwinograd_ja,mgsm"
+NUM_FEW_SHOTS="3,3,3,2,1,1,0,5"
+python main.py \
+    --model hf-causal \
+    --model_args $MODEL_ARGS \
+    --tasks $TASK \
+    --num_fewshot $NUM_FEW_SHOTS \
+    --device "cuda" \
+    --output_path "models/stablelm/stablelm-jp-1b-compact-v1/result.json"

scripts/yans/eval/lm-evaluation-harness/models/stabilityai/experiments/stablelm-jp-1b-compact-v1/result.json

@@ -0,0 +1,71 @@
+{
+  "results": {
+    "jcommonsenseqa-1.1-0.2": {
+      "acc": 0.4709562109025916,
+      "acc_stderr": 0.014928465632785326,
+      "acc_norm": 0.3485254691689008,
+      "acc_norm_stderr": 0.014250991444953297
+    },
+    "jnli-1.1-0.2": {
+      "acc": 0.4449465899753492,
+      "acc_stderr": 0.010075121089036965,
+      "acc_norm": 0.4026294165981923,
+      "acc_norm_stderr": 0.009942683448992417
+    },
+    "marc_ja-1.1-0.2": {
+      "acc": 0.757063132193931,
+      "acc_stderr": 0.005663981049607239,
+      "acc_norm": 0.757063132193931,
+      "acc_norm_stderr": 0.005663981049607239
+    },
+    "xwinograd_ja": {
+      "acc": 0.6339937434827946,
+      "acc_stderr": 0.015563382319228687
+    },
+    "jsquad-1.1-0.2": {
+      "exact_match": 29.536244934714095,
+      "f1": 39.00936796569676
+    },
+    "jaqket_v2-0.1-0.2": {
+      "exact_match": 33.24742268041237,
+      "f1": 38.13348879070528
+    },
+    "xlsum_ja": {
+      "rouge2": 4.3964148234614
+    },
+    "mgsm": {
+      "acc": 0.012,
+      "acc_stderr": 0.0069003230236942764
+    }
+  },
+  "versions": {
+    "jcommonsenseqa-1.1-0.2": 1.1,
+    "jnli-1.1-0.2": 1.1,
+    "marc_ja-1.1-0.2": 1.1,
+    "jsquad-1.1-0.2": 1.1,
+    "jaqket_v2-0.1-0.2": 0.1,
+    "xlsum_ja": 1.0,
+    "xwinograd_ja": 1.0,
+    "mgsm": 1.0
+  },
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/fsx/proj-jp-stablegpt/hf_model/1b-compact-v1,tokenizer=/fsx/proj-jp-stablegpt/tokenizers/compact-hf/,use_fast=False",
+    "num_fewshot": [
+      3,
+      3,
+      3,
+      2,
+      1,
+      1,
+      0,
+      5
+    ],
+    "batch_size": null,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

scripts/yans/eval/lm-evaluation-harness/models/stabilityai/experiments/stablelm-jp-1b-jav1-sl2k-300b/harness.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -eu
+PROJECT_DIR=""
+MODEL_ARGS="pretrained=${PROJECT_DIR}/hf_model/1b-jav1-sl2k-300b,tokenizer=${PROJECT_DIR}/tokenizers/nai-hf-tokenizer/,use_fast=False"
+TASK="jsquad-1.1-0.2,jcommonsenseqa-1.1-0.2,jnli-1.1-0.2,marc_ja-1.1-0.2"
+NUM_FEW_SHOTS="2,3,3,3"
+python main.py \
+    --model hf-causal \
+    --model_args $MODEL_ARGS \
+    --tasks $TASK \
+    --num_fewshot $NUM_FEW_SHOTS \
+    --device "cuda" \
+    --output_path "models/stablelm-jp-1b-jav1-sl2k-300b/result.json"

scripts/yans/eval/lm-evaluation-harness/models/stabilityai/experiments/stablelm-jp-1b-jav1-sl2k-300b/result.json

@@ -0,0 +1,48 @@
+{
+  "results": {
+    "jcommonsenseqa-1.1-0.2": {
+      "acc": 0.2555853440571939,
+      "acc_stderr": 0.013045313758426092,
+      "acc_norm": 0.23056300268096513,
+      "acc_norm_stderr": 0.012596805983976347
+    },
+    "jnli-1.1-0.2": {
+      "acc": 0.37880032867707475,
+      "acc_stderr": 0.009834442099385492,
+      "acc_norm": 0.3648315529991783,
+      "acc_norm_stderr": 0.009759320919777338
+    },
+    "marc_ja-1.1-0.2": {
+      "acc": 0.4899186416696144,
+      "acc_stderr": 0.006648783869548029,
+      "acc_norm": 0.4899186416696144,
+      "acc_norm_stderr": 0.006648783869548029
+    },
+    "jsquad-1.1-0.2": {
+      "exact_match": 34.5114813147231,
+      "f1": 44.58786913290027
+    }
+  },
+  "versions": {
+    "jcommonsenseqa-1.1-0.2": 1.1,
+    "jnli-1.1-0.2": 1.1,
+    "jsquad-1.1-0.2": 1.1,
+    "marc_ja-1.1-0.2": 1.1
+  },
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=${PROJECT_DIR}/hf_model/1b-jav1-sl2k-300b,tokenizer=${PROJECT_DIR}/tokenizers/nai-hf-tokenizer/,use_fast=False",
+    "num_fewshot": [
+      2,
+      3,
+      3,
+      3
+    ],
+    "batch_size": null,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

scripts/yans/eval/lm-evaluation-harness/models/stabilityai/experiments/stablelm-jp-1b-jav1-sl2k-slw-300b/harness.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -eu
+PROJECT_DIR=""
+MODEL_ARGS="pretrained=${PROJECT_DIR}/hf_model/1b-jav1-sl2k-slw-300b,tokenizer=${PROJECT_DIR}/tokenizers/nai-hf-tokenizer/,use_fast=False"
+TASK="jcommonsenseqa-1.1-0.2,jnli-1.1-0.2,marc_ja-1.1-0.2,jsquad-1.1-0.2,jaqket_v2-0.1-0.2,xlsum_ja,xwinograd_ja,mgsm"
+NUM_FEW_SHOTS="3,3,3,2,1,1,0,5"
+python main.py \
+    --model hf-causal \
+    --model_args $MODEL_ARGS \
+    --tasks $TASK \
+    --num_fewshot $NUM_FEW_SHOTS \
+    --device "cuda" \
+    --output_path "models/stablelm/stablelm-jp-1b-jav1-sl2k-slw-300b/result.json"

scripts/yans/eval/lm-evaluation-harness/models/stabilityai/experiments/stablelm-jp-1b-jav1-sl2k-slw-300b/result.json

@@ -0,0 +1,71 @@
+{
+  "results": {
+    "jcommonsenseqa-1.1-0.2": {
+      "acc": 0.28596961572832885,
+      "acc_stderr": 0.013514419338665247,
+      "acc_norm": 0.2421805183199285,
+      "acc_norm_stderr": 0.012812432289317909
+    },
+    "jnli-1.1-0.2": {
+      "acc": 0.3751027115858669,
+      "acc_stderr": 0.009815408241248628,
+      "acc_norm": 0.34880854560394414,
+      "acc_norm_stderr": 0.009662218404461801
+    },
+    "marc_ja-1.1-0.2": {
+      "acc": 0.7518311824206487,
+      "acc_stderr": 0.00570483124396955,
+      "acc_norm": 0.7518311824206487,
+      "acc_norm_stderr": 0.00570483124396955
+    },
+    "xwinograd_ja": {
+      "acc": 0.6527632950990615,
+      "acc_stderr": 0.015381826969142634
+    },
+    "jsquad-1.1-0.2": {
+      "exact_match": 42.390814948221525,
+      "f1": 52.94897262881226
+    },
+    "jaqket_v2-0.1-0.2": {
+      "exact_match": 42.439862542955325,
+      "f1": 48.18551246386296
+    },
+    "xlsum_ja": {
+      "rouge2": 8.371640364702019
+    },
+    "mgsm": {
+      "acc": 0.008,
+      "acc_stderr": 0.00564548367669017
+    }
+  },
+  "versions": {
+    "jcommonsenseqa-1.1-0.2": 1.1,
+    "jnli-1.1-0.2": 1.1,
+    "marc_ja-1.1-0.2": 1.1,
+    "jsquad-1.1-0.2": 1.1,
+    "jaqket_v2-0.1-0.2": 0.1,
+    "xlsum_ja": 1.0,
+    "xwinograd_ja": 1.0,
+    "mgsm": 1.0
+  },
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=${PROJECT_DIR}/hf_model/1b-jav1-sl2k-slw-300b,tokenizer=${PROJECT_DIR}/tokenizers/nai-hf-tokenizer/,use_fast=False",
+    "num_fewshot": [
+      3,
+      3,
+      3,
+      2,
+      1,
+      1,
+      0,
+      5
+    ],
+    "batch_size": null,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

scripts/yans/eval/lm-evaluation-harness/models/stabilityai/experiments/stablelm-jp-1b-jav1_rp-sl2k-slw-300b/harness.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -eu
+
+if [ -z ${JP_LLM_PATH+x} ]; then
+    echo "Error: The JP_LLM_PATH environment variable is not set"
+    exit 1
+fi
+
+MODEL_ARGS="pretrained=$JP_LLM_PATH/hf_model/1b-jav1_rp-sl2k-slw,tokenizer=$JP_LLM_PATH/tokenizers/nai-hf-tokenizer/,use_fast=False"
+TASK="jcommonsenseqa-1.1-0.2,jnli-1.1-0.2,marc_ja-1.1-0.2,jsquad-1.1-0.2,jaqket_v2-0.1-0.2,xlsum_ja,xwinograd_ja,mgsm"
+NUM_FEW_SHOTS="3,3,3,2,1,1,0,5"
+python main.py \
+    --model hf-causal \
+    --model_args $MODEL_ARGS \
+    --tasks $TASK \
+    --num_fewshot $NUM_FEW_SHOTS \
+    --device "cuda" \
+    --output_path "models/stablelm/stablelm-jp-1b-jav1_rp-sl2k-slw-300b/result.json"

scripts/yans/eval/lm-evaluation-harness/models/stabilityai/experiments/stablelm-jp-1b-jav1_rp-sl2k-slw-300b/result.json

@@ -0,0 +1,71 @@
+{
+  "results": {
+    "jcommonsenseqa-1.1-0.2": {
+      "acc": 0.34137622877569257,
+      "acc_stderr": 0.014181247513525478,
+      "acc_norm": 0.2645218945487042,
+      "acc_norm_stderr": 0.013191518316844342
+    },
+    "jnli-1.1-0.2": {
+      "acc": 0.3373048479868529,
+      "acc_stderr": 0.00958511072017679,
+      "acc_norm": 0.3360723089564503,
+      "acc_norm_stderr": 0.009576475494957559
+    },
+    "marc_ja-1.1-0.2": {
+      "acc": 0.6860830136030694,
+      "acc_stderr": 0.006129213801621414,
+      "acc_norm": 0.6860830136030694,
+      "acc_norm_stderr": 0.006129213801621414
+    },
+    "xwinograd_ja": {
+      "acc": 0.6016684045881127,
+      "acc_stderr": 0.015816785549652837
+    },
+    "jsquad-1.1-0.2": {
+      "exact_match": 24.53849617289509,
+      "f1": 33.53058791900235
+    },
+    "jaqket_v2-0.1-0.2": {
+      "exact_match": 23.969072164948454,
+      "f1": 27.900030000545463
+    },
+    "xlsum_ja": {
+      "rouge2": 7.9292934294551545
+    },
+    "mgsm": {
+      "acc": 0.016,
+      "acc_stderr": 0.007951661188874313
+    }
+  },
+  "versions": {
+    "jcommonsenseqa-1.1-0.2": 1.1,
+    "jnli-1.1-0.2": 1.1,
+    "marc_ja-1.1-0.2": 1.1,
+    "jsquad-1.1-0.2": 1.1,
+    "jaqket_v2-0.1-0.2": 0.1,
+    "xlsum_ja": 1.0,
+    "xwinograd_ja": 1.0,
+    "mgsm": 1.0
+  },
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/fsx/proj-jp-stablegpt/hf_model/1b-jav1_rp-sl2k-slw,tokenizer=/fsx/proj-jp-stablegpt/tokenizers/nai-hf-tokenizer/,use_fast=False",
+    "num_fewshot": [
+      3,
+      3,
+      3,
+      2,
+      1,
+      1,
+      0,
+      5
+    ],
+    "batch_size": null,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

scripts/yans/eval/lm-evaluation-harness/models/stabilityai/experiments/stablelm-jp-1b-rp_then_jav1-294b/harness.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -eu
+
+if [ -z ${JP_LLM_PATH+x} ]; then
+    echo "Error: The JP_LLM_PATH environment variable is not set"
+    exit 1
+fi
+
+MODEL_ARGS="pretrained=$JP_LLM_PATH/hf_model/1b-rp_then_jav1-294b,tokenizer=$JP_LLM_PATH/tokenizers/nai-hf-tokenizer/,use_fast=False"
+TASK="jcommonsenseqa-1.1-0.2,jnli-1.1-0.2,marc_ja-1.1-0.2,jsquad-1.1-0.2,jaqket_v2-0.1-0.2,xlsum_ja,xwinograd_ja,mgsm"
+NUM_FEW_SHOTS="3,3,3,2,1,1,0,5"
+python main.py \
+    --model hf-causal \
+    --model_args $MODEL_ARGS \
+    --tasks $TASK \
+    --num_fewshot $NUM_FEW_SHOTS \
+    --device "cuda" \
+    --output_path "models/stablelm/stablelm-jp-1b-rp_then_jav1-294b/result.json"

scripts/yans/eval/lm-evaluation-harness/models/stabilityai/experiments/stablelm-jp-1b-rp_then_jav1-294b/result.json

@@ -0,0 +1,71 @@
+{
+  "results": {
+    "jcommonsenseqa-1.1-0.2": {
+      "acc": 0.2680965147453083,
+      "acc_stderr": 0.013248038756079302,
+      "acc_norm": 0.24039320822162646,
+      "acc_norm_stderr": 0.012780110667692907
+    },
+    "jnli-1.1-0.2": {
+      "acc": 0.3278553820870994,
+      "acc_stderr": 0.009517030628219573,
+      "acc_norm": 0.31183237469186526,
+      "acc_norm_stderr": 0.009391536814742456
+    },
+    "marc_ja-1.1-0.2": {
+      "acc": 0.7771189396581792,
+      "acc_stderr": 0.005496539565709208,
+      "acc_norm": 0.7771189396581792,
+      "acc_norm_stderr": 0.005496539565709208
+    },
+    "xwinograd_ja": {
+      "acc": 0.6861313868613139,
+      "acc_stderr": 0.01499321721472398
+    },
+    "jsquad-1.1-0.2": {
+      "exact_match": 54.02971634398919,
+      "f1": 64.2854711987419
+    },
+    "jaqket_v2-0.1-0.2": {
+      "exact_match": 59.450171821305844,
+      "f1": 65.37892424490362
+    },
+    "xlsum_ja": {
+      "rouge2": 9.662662093427816
+    },
+    "mgsm": {
+      "acc": 0.008,
+      "acc_stderr": 0.0056454836766901585
+    }
+  },
+  "versions": {
+    "jcommonsenseqa-1.1-0.2": 1.1,
+    "jnli-1.1-0.2": 1.1,
+    "marc_ja-1.1-0.2": 1.1,
+    "jsquad-1.1-0.2": 1.1,
+    "jaqket_v2-0.1-0.2": 0.1,
+    "xlsum_ja": 1.0,
+    "xwinograd_ja": 1.0,
+    "mgsm": 1.0
+  },
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=${PROJECT_DIR}/hf_model/1b-rp_then_jav1-294b,tokenizer=${PROJECT_DIR}/tokenizers/nai-hf-tokenizer/,use_fast=False",
+    "num_fewshot": [
+      3,
+      3,
+      3,
+      2,
+      1,
+      1,
+      0,
+      5
+    ],
+    "batch_size": null,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

scripts/yans/eval/lm-evaluation-harness/models/stabilityai/experiments/stablelm-jp-3b-ja50_rp50-700b/harness_template-0.1.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+PROJECT_DIR=""
+MODEL_ARGS="pretrained=${PROJECT_DIR}/hf_model/3b-ja50_rp50-700b,tokenizer=${PROJECT_DIR}/tokenizers/nai-hf-tokenizer/,use_fast=False"
+TASK="jcommonsenseqa-1.1-0.1,jnli,marc_ja,jsquad-1.1-0.1,jaqket_v2-0.1-0.1,xlsum_ja,xwinograd_ja,mgsm"
+NUM_FEW_SHOTS="3,3,3,2,1,1,0,5"
+python main.py \
+    --model hf-causal \
+    --model_args $MODEL_ARGS \
+    --tasks $TASK \
+    --num_fewshot $NUM_FEW_SHOTS \
+    --device "cuda" \
+    --output_path "models/stablelm/stablelm-jp-3b-ja50_rp50-700b/result_template-0.1.json"

scripts/yans/eval/lm-evaluation-harness/models/stabilityai/experiments/stablelm-jp-3b-ja50_rp50-700b/harness_template-0.2.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+PROJECT_DIR=""
+MODEL_ARGS="pretrained=${PROJECT_DIR}/hf_model/3b-ja50_rp50-700b,tokenizer=${PROJECT_DIR}/tokenizers/nai-hf-tokenizer/,use_fast=False"
+TASK="jcommonsenseqa-1.1-0.2,jnli-1.1-0.2,marc_ja-1.1-0.2,jsquad-1.1-0.2,jaqket_v2-0.1-0.2,xlsum_ja,xwinograd_ja,mgsm"
+NUM_FEW_SHOTS="3,3,3,2,1,1,0,5"
+python main.py \
+    --model hf-causal \
+    --model_args $MODEL_ARGS \
+    --tasks $TASK \
+    --num_fewshot $NUM_FEW_SHOTS \
+    --device "cuda" \
+    --output_path "models/stablelm/stablelm-jp-3b-ja50_rp50-700b/result_template-0.2.json"

scripts/yans/eval/lm-evaluation-harness/models/stabilityai/experiments/stablelm-jp-3b-ja50_rp50-700b/result_template-0.1.json

@@ -0,0 +1,71 @@
+{
+  "results": {
+    "jcommonsenseqa-1.1-0.1": {
+      "acc": 0.4280607685433423,
+      "acc_stderr": 0.014798127177394432,
+      "acc_norm": 0.40214477211796246,
+      "acc_norm_stderr": 0.014664536048234705
+    },
+    "jnli": {
+      "acc": 0.36442070665571075,
+      "acc_stderr": 0.009756978284439256,
+      "acc_norm": 0.3245686113393591,
+      "acc_norm_stderr": 0.00949232990976085
+    },
+    "marc_ja": {
+      "acc": 0.7539239623299616,
+      "acc_stderr": 0.005688627090173545,
+      "acc_norm": 0.7539239623299616,
+      "acc_norm_stderr": 0.005688627090173545
+    },
+    "xwinograd_ja": {
+      "acc": 0.6819603753910324,
+      "acc_stderr": 0.015046567305192259
+    },
+    "jsquad-1.1-0.1": {
+      "exact_match": 57.29401170643854,
+      "f1": 66.44109170808048
+    },
+    "jaqket_v2-0.1-0.1": {
+      "exact_match": 52.40549828178694,
+      "f1": 58.039235010884475
+    },
+    "xlsum_ja": {
+      "rouge2": 8.644546504860047
+    },
+    "mgsm": {
+      "acc": 0.016,
+      "acc_stderr": 0.00795166118887434
+    }
+  },
+  "versions": {
+    "jcommonsenseqa-1.1-0.1": 1.1,
+    "jnli": 1.1,
+    "marc_ja": 1.1,
+    "jsquad-1.1-0.1": 1.1,
+    "jaqket_v2-0.1-0.1": 0.1,
+    "xlsum_ja": 1.0,
+    "xwinograd_ja": 1.0,
+    "mgsm": 1.0
+  },
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/PROJECT_DIR/hf_model/3b-ja50_rp50-700b,tokenizer=/PROJECT_DIR/tokenizers/nai-hf-tokenizer/,use_fast=False",
+    "num_fewshot": [
+      3,
+      3,
+      3,
+      2,
+      1,
+      1,
+      0,
+      5
+    ],
+    "batch_size": null,
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}