Add files

.gitattributes

@@ -30,3 +30,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+*.jsonl filter=lfs diff=lfs merge=lfs -text

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/config.json

@@ -0,0 +1,31 @@
+{
+  "apply_residual_connection_post_layernorm": false,
+  "architectures": [
+    "BloomModel"
+  ],
+  "attention_dropout": 0.0,
+  "attention_softmax_in_fp32": true,
+  "bias_dropout_fusion": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_dropout": 0.0,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "masked_softmax_fusion": true,
+  "model_type": "bloom",
+  "n_embed": 4096,
+  "n_inner": null,
+  "n_layer": 30,
+  "num_attention_heads": 32,
+  "offset_alibi": 100,
+  "pad_token_id": 3,
+  "pretraining_tp": 4,
+  "seq_length": 2048,
+  "skip_bias_add": true,
+  "skip_bias_add_qkv": false,
+  "slow_but_exact": false,
+  "transformers_version": "4.21.0.dev0",
+  "unk_token_id": 0,
+  "use_cache": true,
+  "vocab_size": 250880
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/amazon_reviews_multi/en/prompt_body_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "en",
+  "template_name": "prompt_body_title_to_star",
+  "evaluation": {
+    "accuracy": 0.571
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/amazon_reviews_multi/en/prompt_review_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "en",
+  "template_name": "prompt_review_to_star",
+  "evaluation": {
+    "accuracy": 0.53
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/amazon_reviews_multi/en/prompt_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "en",
+  "template_name": "prompt_title_to_star",
+  "evaluation": {
+    "accuracy": 0.4174
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/amazon_reviews_multi/es/prompt_body_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "es",
+  "template_name": "prompt_body_title_to_star",
+  "evaluation": {
+    "accuracy": 0.5164
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/amazon_reviews_multi/es/prompt_review_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "es",
+  "template_name": "prompt_review_to_star",
+  "evaluation": {
+    "accuracy": 0.4972
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/amazon_reviews_multi/es/prompt_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "es",
+  "template_name": "prompt_title_to_star",
+  "evaluation": {
+    "accuracy": 0.3392
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/amazon_reviews_multi/fr/prompt_body_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "fr",
+  "template_name": "prompt_body_title_to_star",
+  "evaluation": {
+    "accuracy": 0.5248
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/amazon_reviews_multi/fr/prompt_review_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "fr",
+  "template_name": "prompt_review_to_star",
+  "evaluation": {
+    "accuracy": 0.4976
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/amazon_reviews_multi/fr/prompt_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "fr",
+  "template_name": "prompt_title_to_star",
+  "evaluation": {
+    "accuracy": 0.3476
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/amazon_reviews_multi/zh/prompt_body_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "zh",
+  "template_name": "prompt_body_title_to_star",
+  "evaluation": {
+    "accuracy": 0.4834
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/amazon_reviews_multi/zh/prompt_review_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "zh",
+  "template_name": "prompt_review_to_star",
+  "evaluation": {
+    "accuracy": 0.4602
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/amazon_reviews_multi/zh/prompt_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "zh",
+  "template_name": "prompt_title_to_star",
+  "evaluation": {
+    "accuracy": 0.3522
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/aqua_rat/raw/Answer_questions_from_options/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "aqua_rat",
+  "dataset_config_name": "raw",
+  "template_name": "Answer questions from options",
+  "evaluation": {
+    "accuracy": 0.24803149606299213
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='raw', dataset_name='aqua_rat', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Answer questions from options', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/aqua_rat/raw/answer_quiz/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "aqua_rat",
+  "dataset_config_name": "raw",
+  "template_name": "answer_quiz",
+  "evaluation": {
+    "accuracy": 0.23622047244094488
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='raw', dataset_name='aqua_rat', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='answer_quiz', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/aqua_rat/raw/select_the_best_option/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "aqua_rat",
+  "dataset_config_name": "raw",
+  "template_name": "select_the_best_option",
+  "evaluation": {
+    "accuracy": 0.21653543307086615
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='raw', dataset_name='aqua_rat', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='select_the_best_option', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/art/choose_hypothesis/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "art",
+  "dataset_config_name": null,
+  "template_name": "choose_hypothesis",
+  "evaluation": {
+    "accuracy": 0.6135770234986945
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/art/choose_hypothesis_believable/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "art",
+  "dataset_config_name": null,
+  "template_name": "choose_hypothesis_believable",
+  "evaluation": {
+    "accuracy": 0.6168407310704961
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_believable', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/art/choose_hypothesis_desc/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "art",
+  "dataset_config_name": null,
+  "template_name": "choose_hypothesis_desc",
+  "evaluation": {
+    "accuracy": 0.554177545691906
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_desc', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/art/choose_hypothesis_likely/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "art",
+  "dataset_config_name": null,
+  "template_name": "choose_hypothesis_likely",
+  "evaluation": {
+    "accuracy": 0.5496083550913838
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_likely', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/art/choose_hypothesis_options/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "art",
+  "dataset_config_name": null,
+  "template_name": "choose_hypothesis_options",
+  "evaluation": {
+    "accuracy": 0.6142297650130548
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_options', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/banking77/direct_to_which_department/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "banking77",
+  "dataset_config_name": null,
+  "template_name": "direct_to_which_department",
+  "evaluation": {
+    "accuracy": 0.19025974025974027
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='banking77', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='direct_to_which_department', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/banking77/help_page_topic/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "banking77",
+  "dataset_config_name": null,
+  "template_name": "help_page_topic",
+  "evaluation": {
+    "accuracy": 0.2327922077922078
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='banking77', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='help_page_topic', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/banking77/rephrase_as_banking_term/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "banking77",
+  "dataset_config_name": null,
+  "template_name": "rephrase_as_banking_term",
+  "evaluation": {
+    "accuracy": 0.23474025974025975
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='banking77', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='rephrase_as_banking_term', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/blbooksgenre/title_genre_classifiction/classify/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blbooksgenre",
+  "dataset_config_name": "title_genre_classifiction",
+  "template_name": "classify",
+  "evaluation": {
+    "accuracy": 0.2730414746543779
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='title_genre_classifiction', dataset_name='blbooksgenre', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='classify', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/blbooksgenre/title_genre_classifiction/multi-choice/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blbooksgenre",
+  "dataset_config_name": "title_genre_classifiction",
+  "template_name": "multi-choice",
+  "evaluation": {
+    "accuracy": 0.7453917050691244
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='title_genre_classifiction', dataset_name='blbooksgenre', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='multi-choice', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/blbooksgenre/title_genre_classifiction/premise_context_first/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blbooksgenre",
+  "dataset_config_name": "title_genre_classifiction",
+  "template_name": "premise_context_first",
+  "evaluation": {
+    "accuracy": 0.7511520737327189
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='title_genre_classifiction', dataset_name='blbooksgenre', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='premise_context_first', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/blimp/adjunct_island/grammatical_between_1_2/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blimp",
+  "dataset_config_name": "adjunct_island",
+  "template_name": "grammatical_between_1_2",
+  "evaluation": {
+    "accuracy": 0.542
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='grammatical_between_1_2', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/blimp/adjunct_island/grammatical_between_A_B/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blimp",
+  "dataset_config_name": "adjunct_island",
+  "template_name": "grammatical_between_A_B",
+  "evaluation": {
+    "accuracy": 0.401
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='grammatical_between_A_B', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/blimp/adjunct_island/grammatical_which_one_1_2/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blimp",
+  "dataset_config_name": "adjunct_island",
+  "template_name": "grammatical_which_one_1_2",
+  "evaluation": {
+    "accuracy": 0.557
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='grammatical_which_one_1_2', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/blimp/adjunct_island/single_sentence_bad_yes_no/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blimp",
+  "dataset_config_name": "adjunct_island",
+  "template_name": "single_sentence_bad_yes_no",
+  "evaluation": {
+    "accuracy": 0.514
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='single_sentence_bad_yes_no', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/blimp/adjunct_island/single_sentence_good_yes_no/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blimp",
+  "dataset_config_name": "adjunct_island",
+  "template_name": "single_sentence_good_yes_no",
+  "evaluation": {
+    "accuracy": 0.506
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='single_sentence_good_yes_no', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/climate_fever/claim_and_all_supporting_evidences/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "climate_fever",
+  "dataset_config_name": null,
+  "template_name": "claim_and_all_supporting_evidences",
+  "evaluation": {
+    "accuracy": 0.3674267100977199
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='claim_and_all_supporting_evidences', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/climate_fever/fifth_evidence_and_claim_itemization/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "climate_fever",
+  "dataset_config_name": null,
+  "template_name": "fifth_evidence_and_claim_itemization",
+  "evaluation": {
+    "accuracy": 0.5700325732899023
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='fifth_evidence_and_claim_itemization', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/climate_fever/first_evidence_and_claim_itemization/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "climate_fever",
+  "dataset_config_name": null,
+  "template_name": "first_evidence_and_claim_itemization",
+  "evaluation": {
+    "accuracy": 0.4938110749185668
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='first_evidence_and_claim_itemization', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/climate_fever/second_evidence_and_claim_itemization/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "climate_fever",
+  "dataset_config_name": null,
+  "template_name": "second_evidence_and_claim_itemization",
+  "evaluation": {
+    "accuracy": 0.5615635179153095
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='second_evidence_and_claim_itemization', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/climate_fever/third_evidence_claim_pair/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "climate_fever",
+  "dataset_config_name": null,
+  "template_name": "third_evidence_claim_pair",
+  "evaluation": {
+    "accuracy": 0.44625407166123776
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='third_evidence_claim_pair', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/codah/codah/affirmative_instruction_after_sentence_and_choices/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "codah",
+  "dataset_config_name": "codah",
+  "template_name": "affirmative_instruction_after_sentence_and_choices",
+  "evaluation": {
+    "accuracy": 0.7600864553314121
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='codah', dataset_name='codah', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='affirmative_instruction_after_sentence_and_choices', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/codah/codah/affirmative_instruction_before_sentence_and_choices/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "codah",
+  "dataset_config_name": "codah",
+  "template_name": "affirmative_instruction_before_sentence_and_choices",
+  "evaluation": {
+    "accuracy": 0.7514409221902018
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='codah', dataset_name='codah', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='affirmative_instruction_before_sentence_and_choices', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/codah/codah/interrogative_instruction_after_sentence_and_choices/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "codah",
+  "dataset_config_name": "codah",
+  "template_name": "interrogative_instruction_after_sentence_and_choices",
+  "evaluation": {
+    "accuracy": 0.7654899135446686
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='codah', dataset_name='codah', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='interrogative_instruction_after_sentence_and_choices', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/commonsense_qa/answer_given_question_without_options/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "commonsense_qa",
+  "dataset_config_name": null,
+  "template_name": "answer_given_question_without_options",
+  "evaluation": {
+    "accuracy": 0.6347256347256347
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='commonsense_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='answer_given_question_without_options', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/commonsense_qa/most_suitable_answer/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "commonsense_qa",
+  "dataset_config_name": null,
+  "template_name": "most_suitable_answer",
+  "evaluation": {
+    "accuracy": 0.9492219492219492
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='commonsense_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='most_suitable_answer', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/commonsense_qa/question_answering/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "commonsense_qa",
+  "dataset_config_name": null,
+  "template_name": "question_answering",
+  "evaluation": {
+    "accuracy": 0.9508599508599509
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='commonsense_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='question_answering', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/conv_ai_3/ambiguous/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "conv_ai_3",
+  "dataset_config_name": null,
+  "template_name": "ambiguous",
+  "evaluation": {
+    "accuracy": 0.39040207522697795
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='ambiguous', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/conv_ai_3/clarification_needed/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "conv_ai_3",
+  "dataset_config_name": null,
+  "template_name": "clarification_needed",
+  "evaluation": {
+    "accuracy": 0.39040207522697795
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='clarification_needed', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/conv_ai_3/directly_answer/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "conv_ai_3",
+  "dataset_config_name": null,
+  "template_name": "directly_answer",
+  "evaluation": {
+    "accuracy": 0.6095979247730221
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='directly_answer', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/conv_ai_3/score_give_number/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "conv_ai_3",
+  "dataset_config_name": null,
+  "template_name": "score_give_number",
+  "evaluation": {
+    "accuracy": 0.05058365758754864
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='score_give_number', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/conv_ai_3/score_how_much/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "conv_ai_3",
+  "dataset_config_name": null,
+  "template_name": "score_how_much",
+  "evaluation": {
+    "accuracy": 0.01599654128837008
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='score_how_much', tokenizer_name=None, use_slow_tokenizer=False)"
+}

tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation/craigslist_bargains/best_deal/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "craigslist_bargains",
+  "dataset_config_name": null,
+  "template_name": "best deal",
+  "evaluation": {
+    "accuracy": 0.5175879396984925
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='craigslist_bargains', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/6b3t0/tr13f-6b3-ml-t0-lmtoks341b-t0toks10b5-xp3capmixnewcodelonglossseq/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='best deal', tokenizer_name=None, use_slow_tokenizer=False)"
+}