Add files

.gitattributes

@@ -30,3 +30,16 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+xp3capmixlossseq_global_step1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixlossseq_global_step1250/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixlossseq_global_step2500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixlossseq_global_step3000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixlossseq_global_step500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixlossseq_global_step750/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixlossseq_global_step1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixlossseq_global_step1750/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixlossseq_global_step2000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixlossseq_global_step2250/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixlossseq_global_step250/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixlossseq_global_step2750/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

xp3capmixlossseq_global_step1000/config.json

@@ -0,0 +1,30 @@
+{
+  "apply_residual_connection_post_layernorm": false,
+  "attention_dropout": 0.0,
+  "attention_softmax_in_fp32": true,
+  "bias_dropout_fusion": true,
+  "architectures": [
+    "BloomModel"
+  ],
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 3,
+  "unk_token_id": 0,
+  "hidden_dropout": 0.0,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "masked_softmax_fusion": true,
+  "model_type": "bloom",
+  "n_embed": 2560,
+  "n_inner": null,
+  "n_layer": 30,
+  "num_attention_heads": 32,
+  "offset_alibi": 100,
+  "pretraining_tp": 4,
+  "seq_length": 2048,
+  "skip_bias_add": true,
+  "skip_bias_add_qkv": false,
+  "transformers_version": "4.20.0",
+  "use_cache": true,
+  "vocab_size": 250880
+}

xp3capmixlossseq_global_step1000/evaluation/amazon_reviews_multi/en/prompt_body_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "en",
+  "template_name": "prompt_body_title_to_star",
+  "evaluation": {
+    "accuracy": 0.5152
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/amazon_reviews_multi/en/prompt_review_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "en",
+  "template_name": "prompt_review_to_star",
+  "evaluation": {
+    "accuracy": 0.4722
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/amazon_reviews_multi/en/prompt_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "en",
+  "template_name": "prompt_title_to_star",
+  "evaluation": {
+    "accuracy": 0.328
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/amazon_reviews_multi/es/prompt_body_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "es",
+  "template_name": "prompt_body_title_to_star",
+  "evaluation": {
+    "accuracy": 0.4506
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/amazon_reviews_multi/es/prompt_review_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "es",
+  "template_name": "prompt_review_to_star",
+  "evaluation": {
+    "accuracy": 0.4368
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/amazon_reviews_multi/es/prompt_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "es",
+  "template_name": "prompt_title_to_star",
+  "evaluation": {
+    "accuracy": 0.3098
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/amazon_reviews_multi/fr/prompt_body_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "fr",
+  "template_name": "prompt_body_title_to_star",
+  "evaluation": {
+    "accuracy": 0.4492
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/amazon_reviews_multi/fr/prompt_review_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "fr",
+  "template_name": "prompt_review_to_star",
+  "evaluation": {
+    "accuracy": 0.4226
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/amazon_reviews_multi/fr/prompt_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "fr",
+  "template_name": "prompt_title_to_star",
+  "evaluation": {
+    "accuracy": 0.2842
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/amazon_reviews_multi/zh/prompt_body_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "zh",
+  "template_name": "prompt_body_title_to_star",
+  "evaluation": {
+    "accuracy": 0.3686
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/amazon_reviews_multi/zh/prompt_review_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "zh",
+  "template_name": "prompt_review_to_star",
+  "evaluation": {
+    "accuracy": 0.3538
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/amazon_reviews_multi/zh/prompt_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "zh",
+  "template_name": "prompt_title_to_star",
+  "evaluation": {
+    "accuracy": 0.2502
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/aqua_rat/raw/Answer_questions_from_options/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "aqua_rat",
+  "dataset_config_name": "raw",
+  "template_name": "Answer questions from options",
+  "evaluation": {
+    "accuracy": 0.1889763779527559
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='raw', dataset_name='aqua_rat', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Answer questions from options', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/aqua_rat/raw/answer_quiz/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "aqua_rat",
+  "dataset_config_name": "raw",
+  "template_name": "answer_quiz",
+  "evaluation": {
+    "accuracy": 0.20866141732283464
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='raw', dataset_name='aqua_rat', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='answer_quiz', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/aqua_rat/raw/select_the_best_option/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "aqua_rat",
+  "dataset_config_name": "raw",
+  "template_name": "select_the_best_option",
+  "evaluation": {
+    "accuracy": 0.20078740157480315
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='raw', dataset_name='aqua_rat', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='select_the_best_option', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/art/choose_hypothesis/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "art",
+  "dataset_config_name": null,
+  "template_name": "choose_hypothesis",
+  "evaluation": {
+    "accuracy": 0.566579634464752
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/art/choose_hypothesis_believable/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "art",
+  "dataset_config_name": null,
+  "template_name": "choose_hypothesis_believable",
+  "evaluation": {
+    "accuracy": 0.5469973890339426
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_believable', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/art/choose_hypothesis_desc/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "art",
+  "dataset_config_name": null,
+  "template_name": "choose_hypothesis_desc",
+  "evaluation": {
+    "accuracy": 0.5254569190600522
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_desc', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/art/choose_hypothesis_likely/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "art",
+  "dataset_config_name": null,
+  "template_name": "choose_hypothesis_likely",
+  "evaluation": {
+    "accuracy": 0.49673629242819844
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_likely', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/art/choose_hypothesis_options/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "art",
+  "dataset_config_name": null,
+  "template_name": "choose_hypothesis_options",
+  "evaluation": {
+    "accuracy": 0.5378590078328982
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_options', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/banking77/direct_to_which_department/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "banking77",
+  "dataset_config_name": null,
+  "template_name": "direct_to_which_department",
+  "evaluation": {
+    "accuracy": 0.1444805194805195
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='banking77', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='direct_to_which_department', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/banking77/help_page_topic/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "banking77",
+  "dataset_config_name": null,
+  "template_name": "help_page_topic",
+  "evaluation": {
+    "accuracy": 0.25
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='banking77', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='help_page_topic', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/banking77/rephrase_as_banking_term/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "banking77",
+  "dataset_config_name": null,
+  "template_name": "rephrase_as_banking_term",
+  "evaluation": {
+    "accuracy": 0.2922077922077922
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='banking77', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='rephrase_as_banking_term', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/blbooksgenre/title_genre_classifiction/classify/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blbooksgenre",
+  "dataset_config_name": "title_genre_classifiction",
+  "template_name": "classify",
+  "evaluation": {
+    "accuracy": 0.34274193548387094
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='title_genre_classifiction', dataset_name='blbooksgenre', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='classify', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/blbooksgenre/title_genre_classifiction/multi-choice/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blbooksgenre",
+  "dataset_config_name": "title_genre_classifiction",
+  "template_name": "multi-choice",
+  "evaluation": {
+    "accuracy": 0.47523041474654376
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='title_genre_classifiction', dataset_name='blbooksgenre', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='multi-choice', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/blbooksgenre/title_genre_classifiction/premise_context_first/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blbooksgenre",
+  "dataset_config_name": "title_genre_classifiction",
+  "template_name": "premise_context_first",
+  "evaluation": {
+    "accuracy": 0.7482718894009217
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='title_genre_classifiction', dataset_name='blbooksgenre', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='premise_context_first', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/blimp/adjunct_island/grammatical_between_1_2/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blimp",
+  "dataset_config_name": "adjunct_island",
+  "template_name": "grammatical_between_1_2",
+  "evaluation": {
+    "accuracy": 0.515
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='grammatical_between_1_2', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/blimp/adjunct_island/grammatical_between_A_B/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blimp",
+  "dataset_config_name": "adjunct_island",
+  "template_name": "grammatical_between_A_B",
+  "evaluation": {
+    "accuracy": 0.491
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='grammatical_between_A_B', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/blimp/adjunct_island/grammatical_which_one_1_2/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blimp",
+  "dataset_config_name": "adjunct_island",
+  "template_name": "grammatical_which_one_1_2",
+  "evaluation": {
+    "accuracy": 0.509
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='grammatical_which_one_1_2', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/blimp/adjunct_island/single_sentence_bad_yes_no/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blimp",
+  "dataset_config_name": "adjunct_island",
+  "template_name": "single_sentence_bad_yes_no",
+  "evaluation": {
+    "accuracy": 0.493
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='single_sentence_bad_yes_no', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/blimp/adjunct_island/single_sentence_good_yes_no/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blimp",
+  "dataset_config_name": "adjunct_island",
+  "template_name": "single_sentence_good_yes_no",
+  "evaluation": {
+    "accuracy": 0.48
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='single_sentence_good_yes_no', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/climate_fever/claim_and_all_supporting_evidences/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "climate_fever",
+  "dataset_config_name": null,
+  "template_name": "claim_and_all_supporting_evidences",
+  "evaluation": {
+    "accuracy": 0.30749185667752443
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='claim_and_all_supporting_evidences', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/climate_fever/fifth_evidence_and_claim_itemization/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "climate_fever",
+  "dataset_config_name": null,
+  "template_name": "fifth_evidence_and_claim_itemization",
+  "evaluation": {
+    "accuracy": 0.10618892508143322
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='fifth_evidence_and_claim_itemization', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/climate_fever/first_evidence_and_claim_itemization/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "climate_fever",
+  "dataset_config_name": null,
+  "template_name": "first_evidence_and_claim_itemization",
+  "evaluation": {
+    "accuracy": 0.12638436482084692
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='first_evidence_and_claim_itemization', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/climate_fever/second_evidence_and_claim_itemization/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "climate_fever",
+  "dataset_config_name": null,
+  "template_name": "second_evidence_and_claim_itemization",
+  "evaluation": {
+    "accuracy": 0.10684039087947883
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='second_evidence_and_claim_itemization', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/climate_fever/third_evidence_claim_pair/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "climate_fever",
+  "dataset_config_name": null,
+  "template_name": "third_evidence_claim_pair",
+  "evaluation": {
+    "accuracy": 0.09902280130293159
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='third_evidence_claim_pair', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/codah/codah/affirmative_instruction_after_sentence_and_choices/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "codah",
+  "dataset_config_name": "codah",
+  "template_name": "affirmative_instruction_after_sentence_and_choices",
+  "evaluation": {
+    "accuracy": 0.5623198847262247
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='codah', dataset_name='codah', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='affirmative_instruction_after_sentence_and_choices', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/codah/codah/affirmative_instruction_before_sentence_and_choices/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "codah",
+  "dataset_config_name": "codah",
+  "template_name": "affirmative_instruction_before_sentence_and_choices",
+  "evaluation": {
+    "accuracy": 0.5947406340057637
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='codah', dataset_name='codah', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='affirmative_instruction_before_sentence_and_choices', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/codah/codah/interrogative_instruction_after_sentence_and_choices/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "codah",
+  "dataset_config_name": "codah",
+  "template_name": "interrogative_instruction_after_sentence_and_choices",
+  "evaluation": {
+    "accuracy": 0.5673631123919308
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='codah', dataset_name='codah', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='interrogative_instruction_after_sentence_and_choices', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/commonsense_qa/answer_given_question_without_options/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "commonsense_qa",
+  "dataset_config_name": null,
+  "template_name": "answer_given_question_without_options",
+  "evaluation": {
+    "accuracy": 0.5585585585585585
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='commonsense_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='answer_given_question_without_options', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/commonsense_qa/most_suitable_answer/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "commonsense_qa",
+  "dataset_config_name": null,
+  "template_name": "most_suitable_answer",
+  "evaluation": {
+    "accuracy": 0.7526617526617526
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='commonsense_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='most_suitable_answer', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/commonsense_qa/question_answering/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "commonsense_qa",
+  "dataset_config_name": null,
+  "template_name": "question_answering",
+  "evaluation": {
+    "accuracy": 0.7444717444717445
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='commonsense_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='question_answering', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/conv_ai_3/ambiguous/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "conv_ai_3",
+  "dataset_config_name": null,
+  "template_name": "ambiguous",
+  "evaluation": {
+    "accuracy": 0.39040207522697795
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='ambiguous', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/conv_ai_3/clarification_needed/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "conv_ai_3",
+  "dataset_config_name": null,
+  "template_name": "clarification_needed",
+  "evaluation": {
+    "accuracy": 0.39040207522697795
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='clarification_needed', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/conv_ai_3/directly_answer/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "conv_ai_3",
+  "dataset_config_name": null,
+  "template_name": "directly_answer",
+  "evaluation": {
+    "accuracy": 0.6095979247730221
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='directly_answer', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/conv_ai_3/score_give_number/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "conv_ai_3",
+  "dataset_config_name": null,
+  "template_name": "score_give_number",
+  "evaluation": {
+    "accuracy": 0.27626459143968873
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='score_give_number', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/conv_ai_3/score_how_much/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "conv_ai_3",
+  "dataset_config_name": null,
+  "template_name": "score_how_much",
+  "evaluation": {
+    "accuracy": 0.3186338089061824
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='score_how_much', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/craigslist_bargains/best_deal/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "craigslist_bargains",
+  "dataset_config_name": null,
+  "template_name": "best deal",
+  "evaluation": {
+    "accuracy": 0.20603015075376885
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='craigslist_bargains', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/2b5t0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='best deal', tokenizer_name=None, use_slow_tokenizer=False)"
+}