Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,23 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+llmtf_eval_k0/daru_treewayextractive.jsonl filter=lfs diff=lfs merge=lfs -text
+llmtf_eval_k0/darumeru_ruMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
+llmtf_eval_k0/darumeru_ruTiE.jsonl filter=lfs diff=lfs merge=lfs -text
+llmtf_eval_k0/nlpcoreteam_enMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
+llmtf_eval_k0/nlpcoreteam_ruMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
+llmtf_eval_k1/daru_treewayextractive.jsonl filter=lfs diff=lfs merge=lfs -text
+llmtf_eval_k1/darumeru_ruMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
+llmtf_eval_k1/darumeru_ruTiE.jsonl filter=lfs diff=lfs merge=lfs -text
+llmtf_eval_k1/nlpcoreteam_enMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
+llmtf_eval_k1/nlpcoreteam_ruMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
+llmtf_eval_k5/daru_treewayabstractive.jsonl filter=lfs diff=lfs merge=lfs -text
+llmtf_eval_k5/daru_treewayextractive.jsonl filter=lfs diff=lfs merge=lfs -text
+llmtf_eval_k5/darumeru_MultiQ.jsonl filter=lfs diff=lfs merge=lfs -text
+llmtf_eval_k5/darumeru_USE.jsonl filter=lfs diff=lfs merge=lfs -text
+llmtf_eval_k5/darumeru_ruMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
+llmtf_eval_k5/darumeru_ruOpenBookQA.jsonl filter=lfs diff=lfs merge=lfs -text
+llmtf_eval_k5/darumeru_ruTiE.jsonl filter=lfs diff=lfs merge=lfs -text
+llmtf_eval_k5/nlpcoreteam_enMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
+llmtf_eval_k5/nlpcoreteam_ruMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
+llmtf_eval_k5/russiannlp_rucola_custom.jsonl filter=lfs diff=lfs merge=lfs -text

llmtf_eval_k0/daru_treewayabstractive_params.jsonl

@@ -0,0 +1,59 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "IlyaGusev/saiga_llama3_8b",
+        "generation_config": {
+            "_from_model_config": true,
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128009,
+                198,
+                271
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 512,
+            "pad_token_id": 128000,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2"
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "",
+            "user_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template_incomplete": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "<|begin_of_text|>",
+            "suffix": "<|start_header_id|>assistant<|end_header_id|>\n\n",
+            "add_special_tokens": false,
+            "eos_token": "<|eot_id|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 1,
+        "max_sample_per_dataset": 500,
+        "method": "generate"
+    }
+}

llmtf_eval_k0/daru_treewayabstractive_total.jsonl

@@ -0,0 +1,8 @@
+{
+    "task_name": "daru/treewayabstractive",
+    "results": {
+        "rouge1": 0.35122062259344017,
+        "rouge2": 0.12350994560599565
+    },
+    "leaderboard_result": 0.2373652840997179
+}

llmtf_eval_k0/daru_treewayextractive.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac64b29939ae73e49949386a6154d256cac7947baa0286b10115ecbf8104fefd
+size 1984751752

llmtf_eval_k0/daru_treewayextractive_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "IlyaGusev/saiga_llama3_8b",
+        "generation_config": {
+            "_from_model_config": true,
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128009
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 1,
+            "pad_token_id": 128000,
+            "stop_strings": [],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2"
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "",
+            "user_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template_incomplete": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "<|begin_of_text|>",
+            "suffix": "<|start_header_id|>assistant<|end_header_id|>\n\n",
+            "add_special_tokens": false,
+            "eos_token": "<|eot_id|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 1,
+        "max_sample_per_dataset": 500,
+        "method": "calculate_logsoftmax"
+    }
+}

llmtf_eval_k0/daru_treewayextractive_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "daru/treewayextractive",
+    "results": {
+        "r-prec": 0.4058313131313131
+    },
+    "leaderboard_result": 0.4058313131313131
+}

llmtf_eval_k0/darumeru_MultiQ_params.jsonl

@@ -0,0 +1,59 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "IlyaGusev/saiga_llama3_8b",
+        "generation_config": {
+            "_from_model_config": true,
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128009,
+                198,
+                271
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 64,
+            "pad_token_id": 128000,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2"
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "",
+            "user_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template_incomplete": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "<|begin_of_text|>",
+            "suffix": "<|start_header_id|>assistant<|end_header_id|>\n\n",
+            "add_special_tokens": false,
+            "eos_token": "<|eot_id|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 1,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "generate"
+    }
+}

llmtf_eval_k0/darumeru_MultiQ_total.jsonl

@@ -0,0 +1,8 @@
+{
+    "task_name": "darumeru/MultiQ",
+    "results": {
+        "f1": 0.25688951963582124,
+        "em": 0.13766730401529637
+    },
+    "leaderboard_result": 0.1972784118255588
+}

llmtf_eval_k0/darumeru_PARus_params.jsonl

@@ -0,0 +1,59 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "IlyaGusev/saiga_llama3_8b",
+        "generation_config": {
+            "_from_model_config": true,
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128009,
+                198,
+                271
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 64,
+            "pad_token_id": 128000,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2"
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "",
+            "user_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template_incomplete": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "<|begin_of_text|>",
+            "suffix": "<|start_header_id|>assistant<|end_header_id|>\n\n",
+            "add_special_tokens": false,
+            "eos_token": "<|eot_id|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 1,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval_k0/darumeru_PARus_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "darumeru/PARus",
+    "results": {
+        "acc": 0.72
+    },
+    "leaderboard_result": 0.72
+}

llmtf_eval_k0/darumeru_RCB_params.jsonl

@@ -0,0 +1,59 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "IlyaGusev/saiga_llama3_8b",
+        "generation_config": {
+            "_from_model_config": true,
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128009,
+                198,
+                271
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 64,
+            "pad_token_id": 128000,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2"
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "",
+            "user_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template_incomplete": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "<|begin_of_text|>",
+            "suffix": "<|start_header_id|>assistant<|end_header_id|>\n\n",
+            "add_special_tokens": false,
+            "eos_token": "<|eot_id|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 1,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval_k0/darumeru_RCB_total.jsonl

@@ -0,0 +1,8 @@
+{
+    "task_name": "darumeru/RCB",
+    "results": {
+        "acc": 0.5,
+        "f1_macro": 0.40846427823172005
+    },
+    "leaderboard_result": 0.45423213911586
+}

llmtf_eval_k0/darumeru_RWSD_params.jsonl

@@ -0,0 +1,59 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "IlyaGusev/saiga_llama3_8b",
+        "generation_config": {
+            "_from_model_config": true,
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128009,
+                198,
+                271
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 64,
+            "pad_token_id": 128000,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2"
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "",
+            "user_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template_incomplete": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "<|begin_of_text|>",
+            "suffix": "<|start_header_id|>assistant<|end_header_id|>\n\n",
+            "add_special_tokens": false,
+            "eos_token": "<|eot_id|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 1,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval_k0/darumeru_RWSD_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "darumeru/RWSD",
+    "results": {
+        "acc": 0.5637254901960784
+    },
+    "leaderboard_result": 0.5637254901960784
+}

llmtf_eval_k0/darumeru_USE_params.jsonl

@@ -0,0 +1,59 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "IlyaGusev/saiga_llama3_8b",
+        "generation_config": {
+            "_from_model_config": true,
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128009,
+                198,
+                271
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 64,
+            "pad_token_id": 128000,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2"
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "",
+            "user_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template_incomplete": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "<|begin_of_text|>",
+            "suffix": "<|start_header_id|>assistant<|end_header_id|>\n\n",
+            "add_special_tokens": false,
+            "eos_token": "<|eot_id|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 1,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "generate"
+    }
+}

llmtf_eval_k0/darumeru_USE_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "darumeru/USE",
+    "results": {
+        "grade_norm": 0.13333333333333333
+    },
+    "leaderboard_result": 0.13333333333333333
+}

llmtf_eval_k0/darumeru_cp_para_en_params.jsonl

@@ -0,0 +1,59 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "IlyaGusev/saiga_llama3_8b",
+        "generation_config": {
+            "_from_model_config": true,
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128009,
+                198,
+                271
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 1024,
+            "pad_token_id": 128000,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2"
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "",
+            "user_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template_incomplete": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "<|begin_of_text|>",
+            "suffix": "<|start_header_id|>assistant<|end_header_id|>\n\n",
+            "add_special_tokens": false,
+            "eos_token": "<|eot_id|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 1,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "generate"
+    }
+}

llmtf_eval_k0/darumeru_cp_para_en_total.jsonl

@@ -0,0 +1,9 @@
+{
+    "task_name": "darumeru/cp_para_en",
+    "results": {
+        "symbol_per_token": 4.480576106482915,
+        "len": 0.992331176940417,
+        "lcs": 0.9043190574683162
+    },
+    "leaderboard_result": 0.9043190574683162
+}

llmtf_eval_k0/darumeru_cp_para_ru_params.jsonl

@@ -0,0 +1,59 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "IlyaGusev/saiga_llama3_8b",
+        "generation_config": {
+            "_from_model_config": true,
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128009,
+                198,
+                271
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 1024,
+            "pad_token_id": 128000,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2"
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "",
+            "user_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template_incomplete": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "<|begin_of_text|>",
+            "suffix": "<|start_header_id|>assistant<|end_header_id|>\n\n",
+            "add_special_tokens": false,
+            "eos_token": "<|eot_id|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 1,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "generate"
+    }
+}

llmtf_eval_k0/darumeru_cp_para_ru_total.jsonl

@@ -0,0 +1,9 @@
+{
+    "task_name": "darumeru/cp_para_ru",
+    "results": {
+        "symbol_per_token": 2.9699926532822025,
+        "len": 0.9860381898327103,
+        "lcs": 0.8231040113557345
+    },
+    "leaderboard_result": 0.8231040113557345
+}

llmtf_eval_k0/darumeru_cp_sent_en_params.jsonl

@@ -0,0 +1,59 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "IlyaGusev/saiga_llama3_8b",
+        "generation_config": {
+            "_from_model_config": true,
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128009,
+                198,
+                271
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 128,
+            "pad_token_id": 128000,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2"
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "",
+            "user_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template_incomplete": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "<|begin_of_text|>",
+            "suffix": "<|start_header_id|>assistant<|end_header_id|>\n\n",
+            "add_special_tokens": false,
+            "eos_token": "<|eot_id|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 1,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "generate"
+    }
+}

llmtf_eval_k0/darumeru_cp_sent_en_total.jsonl

@@ -0,0 +1,9 @@
+{
+    "task_name": "darumeru/cp_sent_en",
+    "results": {
+        "symbol_per_token": 4.424251537340727,
+        "len": 0.999397410430255,
+        "lcs": 0.9916454162768136
+    },
+    "leaderboard_result": 0.999397410430255
+}

llmtf_eval_k0/darumeru_cp_sent_ru_params.jsonl

@@ -0,0 +1,59 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "IlyaGusev/saiga_llama3_8b",
+        "generation_config": {
+            "_from_model_config": true,
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128009,
+                198,
+                271
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 128,
+            "pad_token_id": 128000,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2"
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "",
+            "user_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template_incomplete": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "<|begin_of_text|>",
+            "suffix": "<|start_header_id|>assistant<|end_header_id|>\n\n",
+            "add_special_tokens": false,
+            "eos_token": "<|eot_id|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 1,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "generate"
+    }
+}

llmtf_eval_k0/darumeru_cp_sent_ru_total.jsonl

@@ -0,0 +1,9 @@
+{
+    "task_name": "darumeru/cp_sent_ru",
+    "results": {
+        "symbol_per_token": 2.837319872762208,
+        "len": 0.9845781877722732,
+        "lcs": 0.9208112603069822
+    },
+    "leaderboard_result": 0.9845781877722732
+}

llmtf_eval_k0/darumeru_ruMMLU.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c704838c7b2aca6f0cf66576239b7b2ee353cffe9214e0ef90494844c1fccc12
+size 32887585

llmtf_eval_k0/darumeru_ruMMLU_params.jsonl

@@ -0,0 +1,59 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "IlyaGusev/saiga_llama3_8b",
+        "generation_config": {
+            "_from_model_config": true,
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128009,
+                198,
+                271
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 64,
+            "pad_token_id": 128000,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2"
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "",
+            "user_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template_incomplete": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "<|begin_of_text|>",
+            "suffix": "<|start_header_id|>assistant<|end_header_id|>\n\n",
+            "add_special_tokens": false,
+            "eos_token": "<|eot_id|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 1,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval_k0/darumeru_ruMMLU_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "darumeru/ruMMLU",
+    "results": {
+        "acc": 0.5117230370148658
+    },
+    "leaderboard_result": 0.5117230370148658
+}

llmtf_eval_k0/darumeru_ruOpenBookQA_params.jsonl

@@ -0,0 +1,59 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "IlyaGusev/saiga_llama3_8b",
+        "generation_config": {
+            "_from_model_config": true,
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128009,
+                198,
+                271
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 64,
+            "pad_token_id": 128000,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2"
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "",
+            "user_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template_incomplete": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "<|begin_of_text|>",
+            "suffix": "<|start_header_id|>assistant<|end_header_id|>\n\n",
+            "add_special_tokens": false,
+            "eos_token": "<|eot_id|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 1,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval_k0/darumeru_ruOpenBookQA_total.jsonl

@@ -0,0 +1,8 @@
+{
+    "task_name": "darumeru/ruOpenBookQA",
+    "results": {
+        "acc": 0.711340206185567,
+        "f1_macro": 0.710987663128528
+    },
+    "leaderboard_result": 0.7111639346570475
+}

llmtf_eval_k0/darumeru_ruTiE.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0902664ac8acfc5e5dca2660d1d7bec14cd9e46be4c289b7fab71870eab75d1
+size 12832224

llmtf_eval_k0/darumeru_ruTiE_params.jsonl

@@ -0,0 +1,59 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "IlyaGusev/saiga_llama3_8b",
+        "generation_config": {
+            "_from_model_config": true,
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128009,
+                198,
+                271
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 64,
+            "pad_token_id": 128000,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2"
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "",
+            "user_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template_incomplete": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "<|begin_of_text|>",
+            "suffix": "<|start_header_id|>assistant<|end_header_id|>\n\n",
+            "add_special_tokens": false,
+            "eos_token": "<|eot_id|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 1,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval_k0/darumeru_ruTiE_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "darumeru/ruTiE",
+    "results": {
+        "acc": 0.4441860465116279
+    },
+    "leaderboard_result": 0.4441860465116279
+}

llmtf_eval_k0/darumeru_ruWorldTree_params.jsonl

@@ -0,0 +1,59 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "IlyaGusev/saiga_llama3_8b",
+        "generation_config": {
+            "_from_model_config": true,
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128009,
+                198,
+                271
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 64,
+            "pad_token_id": 128000,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2"
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "",
+            "user_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template_incomplete": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "<|begin_of_text|>",
+            "suffix": "<|start_header_id|>assistant<|end_header_id|>\n\n",
+            "add_special_tokens": false,
+            "eos_token": "<|eot_id|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 1,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval_k0/darumeru_ruWorldTree_total.jsonl

@@ -0,0 +1,8 @@
+{
+    "task_name": "darumeru/ruWorldTree",
+    "results": {
+        "acc": 0.8476190476190476,
+        "f1_macro": 0.8452567487675969
+    },
+    "leaderboard_result": 0.8464378981933223
+}

llmtf_eval_k0/evaluation_log.txt

@@ -0,0 +1,273 @@
+INFO: 2024-07-12 16:39:04,242: llmtf.base.evaluator: Starting eval on ['darumeru/multiq', 'darumeru/parus', 'darumeru/rcb', 'darumeru/ruopenbookqa', 'darumeru/rutie', 'darumeru/ruworldtree', 'darumeru/rwsd', 'darumeru/use', 'russiannlp/rucola_custom']
+INFO: 2024-07-12 16:39:04,244: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128009, 198, 271]
+INFO: 2024-07-12 16:39:04,244: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-12 16:39:05,043: llmtf.base.evaluator: Starting eval on ['darumeru/rummlu']
+INFO: 2024-07-12 16:39:05,044: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128009, 198, 271]
+INFO: 2024-07-12 16:39:05,044: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-12 16:39:07,157: llmtf.base.evaluator: Starting eval on ['nlpcoreteam/rummlu']
+INFO: 2024-07-12 16:39:07,158: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128009]
+INFO: 2024-07-12 16:39:07,158: llmtf.base.hfmodel: Updated generation_config.stop_strings: []
+INFO: 2024-07-12 16:39:08,324: llmtf.base.darumeru/MultiQ: Loading Dataset: 4.08s
+INFO: 2024-07-12 16:39:08,773: llmtf.base.evaluator: Starting eval on ['nlpcoreteam/enmmlu']
+INFO: 2024-07-12 16:39:08,774: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128009]
+INFO: 2024-07-12 16:39:08,774: llmtf.base.hfmodel: Updated generation_config.stop_strings: []
+INFO: 2024-07-12 16:39:11,352: llmtf.base.evaluator: Starting eval on ['daru/treewayabstractive']
+INFO: 2024-07-12 16:39:11,353: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128009, 198, 271]
+INFO: 2024-07-12 16:39:11,353: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-12 16:39:13,160: llmtf.base.evaluator: Starting eval on ['daru/treewayextractive']
+INFO: 2024-07-12 16:39:13,161: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128009]
+INFO: 2024-07-12 16:39:13,161: llmtf.base.hfmodel: Updated generation_config.stop_strings: []
+INFO: 2024-07-12 16:39:13,414: llmtf.base.darumeru/ruMMLU: Loading Dataset: 8.37s
+INFO: 2024-07-12 16:39:14,664: llmtf.base.evaluator: Starting eval on ['darumeru/cp_sent_ru', 'darumeru/cp_sent_en', 'darumeru/cp_para_ru', 'darumeru/cp_para_en']
+INFO: 2024-07-12 16:39:14,664: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128009, 198, 271]
+INFO: 2024-07-12 16:39:14,664: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-12 16:39:15,607: llmtf.base.daru/treewayabstractive: Loading Dataset: 4.25s
+INFO: 2024-07-12 16:39:17,669: llmtf.base.darumeru/cp_sent_ru: Loading Dataset: 3.00s
+INFO: 2024-07-12 16:39:24,707: llmtf.base.daru/treewayextractive: Loading Dataset: 11.55s
+INFO: 2024-07-12 16:41:19,393: llmtf.base.nlpcoreteam/enMMLU: Loading Dataset: 130.62s
+INFO: 2024-07-12 16:41:22,595: llmtf.base.nlpcoreteam/ruMMLU: Loading Dataset: 135.44s
+INFO: 2024-07-12 16:45:26,204: llmtf.base.darumeru/ruMMLU: Processing Dataset: 372.79s
+INFO: 2024-07-12 16:45:26,207: llmtf.base.darumeru/ruMMLU: Results for darumeru/ruMMLU:
+INFO: 2024-07-12 16:45:26,228: llmtf.base.darumeru/ruMMLU: {'acc': 0.5117230370148658}
+INFO: 2024-07-12 16:45:26,263: llmtf.base.evaluator: Ended eval
+INFO: 2024-07-12 16:45:26,268: llmtf.base.evaluator: 
+mean	darumeru/ruMMLU
+0.512	0.512
+INFO: 2024-07-12 16:48:53,262: llmtf.base.darumeru/cp_sent_ru: Processing Dataset: 575.59s
+INFO: 2024-07-12 16:48:53,265: llmtf.base.darumeru/cp_sent_ru: Results for darumeru/cp_sent_ru:
+INFO: 2024-07-12 16:48:53,269: llmtf.base.darumeru/cp_sent_ru: {'symbol_per_token': 2.837319872762208, 'len': 0.9845781877722732, 'lcs': 0.9208112603069822}
+INFO: 2024-07-12 16:48:53,271: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128009, 198, 271]
+INFO: 2024-07-12 16:48:53,271: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-12 16:48:55,985: llmtf.base.darumeru/cp_sent_en: Loading Dataset: 2.71s
+INFO: 2024-07-12 16:49:13,988: llmtf.base.nlpcoreteam/enMMLU: Processing Dataset: 474.56s
+INFO: 2024-07-12 16:49:13,989: llmtf.base.nlpcoreteam/enMMLU: Results for nlpcoreteam/enMMLU:
+INFO: 2024-07-12 16:49:14,028: llmtf.base.nlpcoreteam/enMMLU:                                        metric
+subject                                      
+abstract_algebra                     0.280000
+anatomy                              0.703704
+astronomy                            0.743421
+business_ethics                      0.680000
+clinical_knowledge                   0.724528
+college_biology                      0.791667
+college_chemistry                    0.510000
+college_computer_science             0.570000
+college_mathematics                  0.340000
+college_medicine                     0.653179
+college_physics                      0.382353
+computer_security                    0.750000
+conceptual_physics                   0.561702
+econometrics                         0.473684
+electrical_engineering               0.620690
+elementary_mathematics               0.476190
+formal_logic                         0.563492
+global_facts                         0.380000
+high_school_biology                  0.812903
+high_school_chemistry                0.541872
+high_school_computer_science         0.690000
+high_school_european_history         0.745455
+high_school_geography                0.828283
+high_school_government_and_politics  0.854922
+high_school_macroeconomics           0.646154
+high_school_mathematics              0.381481
+high_school_microeconomics           0.764706
+high_school_physics                  0.357616
+high_school_psychology               0.855046
+high_school_statistics               0.583333
+high_school_us_history               0.828431
+high_school_world_history            0.835443
+human_aging                          0.699552
+human_sexuality                      0.740458
+international_law                    0.768595
+jurisprudence                        0.787037
+logical_fallacies                    0.748466
+machine_learning                     0.482143
+management                           0.834951
+marketing                            0.876068
+medical_genetics                     0.780000
+miscellaneous                        0.848020
+moral_disputes                       0.687861
+moral_scenarios                      0.322905
+nutrition                            0.761438
+philosophy                           0.678457
+prehistory                           0.700617
+professional_accounting              0.503546
+professional_law                     0.463494
+professional_medicine                0.750000
+professional_psychology              0.668301
+public_relations                     0.681818
+security_studies                     0.702041
+sociology                            0.840796
+us_foreign_policy                    0.860000
+virology                             0.487952
+world_religions                      0.836257
+INFO: 2024-07-12 16:49:14,036: llmtf.base.nlpcoreteam/enMMLU:                                    metric
+subject                                  
+STEM                             0.548632
+humanities                       0.689732
+other (business, health, misc.)  0.691638
+social sciences                  0.743017
+INFO: 2024-07-12 16:49:14,043: llmtf.base.nlpcoreteam/enMMLU: {'acc': 0.6682548109047595}
+INFO: 2024-07-12 16:49:14,074: llmtf.base.evaluator: Ended eval
+INFO: 2024-07-12 16:49:14,079: llmtf.base.evaluator: 
+mean	darumeru/cp_sent_ru	darumeru/ruMMLU	nlpcoreteam/enMMLU
+0.722	0.985	0.512	0.668
+INFO: 2024-07-12 16:49:59,627: llmtf.base.nlpcoreteam/ruMMLU: Processing Dataset: 517.03s
+INFO: 2024-07-12 16:49:59,648: llmtf.base.nlpcoreteam/ruMMLU: Results for nlpcoreteam/ruMMLU:
+INFO: 2024-07-12 16:49:59,687: llmtf.base.nlpcoreteam/ruMMLU:                                        metric
+subject                                      
+abstract_algebra                     0.310000
+anatomy                              0.525926
+astronomy                            0.644737
+business_ethics                      0.640000
+clinical_knowledge                   0.562264
+college_biology                      0.569444
+college_chemistry                    0.370000
+college_computer_science             0.410000
+college_mathematics                  0.330000
+college_medicine                     0.508671
+college_physics                      0.372549
+computer_security                    0.590000
+conceptual_physics                   0.497872
+econometrics                         0.377193
+electrical_engineering               0.537931
+elementary_mathematics               0.407407
+formal_logic                         0.373016
+global_facts                         0.380000
+high_school_biology                  0.674194
+high_school_chemistry                0.438424
+high_school_computer_science         0.680000
+high_school_european_history         0.733333
+high_school_geography                0.676768
+high_school_government_and_politics  0.595855
+high_school_macroeconomics           0.517949
+high_school_mathematics              0.366667
+high_school_microeconomics           0.512605
+high_school_physics                  0.370861
+high_school_psychology               0.675229
+high_school_statistics               0.486111
+high_school_us_history               0.671569
+high_school_world_history            0.721519
+human_aging                          0.524664
+human_sexuality                      0.625954
+international_law                    0.702479
+jurisprudence                        0.611111
+logical_fallacies                    0.503067
+machine_learning                     0.303571
+management                           0.679612
+marketing                            0.709402
+medical_genetics                     0.600000
+miscellaneous                        0.667944
+moral_disputes                       0.549133
+moral_scenarios                      0.251397
+nutrition                            0.630719
+philosophy                           0.585209
+prehistory                           0.555556
+professional_accounting              0.390071
+professional_law                     0.358540
+professional_medicine                0.496324
+professional_psychology              0.465686
+public_relations                     0.563636
+security_studies                     0.628571
+sociology                            0.686567
+us_foreign_policy                    0.750000
+virology                             0.421687
+world_religions                      0.690058
+INFO: 2024-07-12 16:49:59,695: llmtf.base.nlpcoreteam/ruMMLU:                                    metric
+subject                                  
+STEM                             0.464432
+humanities                       0.561999
+other (business, health, misc.)  0.552663
+social sciences                  0.589668
+INFO: 2024-07-12 16:49:59,703: llmtf.base.nlpcoreteam/ruMMLU: {'acc': 0.5421903513895152}
+INFO: 2024-07-12 16:49:59,735: llmtf.base.evaluator: Ended eval
+INFO: 2024-07-12 16:49:59,741: llmtf.base.evaluator: 
+mean	darumeru/cp_sent_ru	darumeru/ruMMLU	nlpcoreteam/enMMLU	nlpcoreteam/ruMMLU
+0.677	0.985	0.512	0.668	0.542
+INFO: 2024-07-12 16:52:26,008: llmtf.base.darumeru/MultiQ: Processing Dataset: 797.68s
+INFO: 2024-07-12 16:52:26,011: llmtf.base.darumeru/MultiQ: Results for darumeru/MultiQ:
+INFO: 2024-07-12 16:52:26,029: llmtf.base.darumeru/MultiQ: {'f1': 0.25688951963582124, 'em': 0.13766730401529637}
+INFO: 2024-07-12 16:52:26,034: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128009, 198, 271]
+INFO: 2024-07-12 16:52:26,034: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-12 16:52:28,374: llmtf.base.darumeru/PARus: Loading Dataset: 2.34s
+INFO: 2024-07-12 16:52:34,776: llmtf.base.darumeru/PARus: Processing Dataset: 6.39s
+INFO: 2024-07-12 16:52:34,777: llmtf.base.darumeru/PARus: Results for darumeru/PARus:
+INFO: 2024-07-12 16:52:34,792: llmtf.base.darumeru/PARus: {'acc': 0.72}
+INFO: 2024-07-12 16:52:34,794: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128009, 198, 271]
+INFO: 2024-07-12 16:52:34,794: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-12 16:52:36,848: llmtf.base.darumeru/RCB: Loading Dataset: 2.05s
+INFO: 2024-07-12 16:52:44,579: llmtf.base.darumeru/RCB: Processing Dataset: 7.73s
+INFO: 2024-07-12 16:52:44,581: llmtf.base.darumeru/RCB: Results for darumeru/RCB:
+INFO: 2024-07-12 16:52:44,588: llmtf.base.darumeru/RCB: {'acc': 0.5, 'f1_macro': 0.40846427823172005}
+INFO: 2024-07-12 16:52:44,589: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128009, 198, 271]
+INFO: 2024-07-12 16:52:44,589: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-12 16:52:47,425: llmtf.base.darumeru/ruOpenBookQA: Loading Dataset: 2.83s
+INFO: 2024-07-12 16:54:02,874: llmtf.base.darumeru/ruOpenBookQA: Processing Dataset: 75.45s
+INFO: 2024-07-12 16:54:02,875: llmtf.base.darumeru/ruOpenBookQA: Results for darumeru/ruOpenBookQA:
+INFO: 2024-07-12 16:54:02,904: llmtf.base.darumeru/ruOpenBookQA: {'acc': 0.711340206185567, 'f1_macro': 0.710987663128528}
+INFO: 2024-07-12 16:54:02,911: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128009, 198, 271]
+INFO: 2024-07-12 16:54:02,911: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-12 16:54:10,220: llmtf.base.darumeru/ruTiE: Loading Dataset: 7.31s
+INFO: 2024-07-12 16:56:47,398: llmtf.base.darumeru/cp_sent_en: Processing Dataset: 471.41s
+INFO: 2024-07-12 16:56:47,429: llmtf.base.darumeru/cp_sent_en: Results for darumeru/cp_sent_en:
+INFO: 2024-07-12 16:56:47,434: llmtf.base.darumeru/cp_sent_en: {'symbol_per_token': 4.424251537340727, 'len': 0.999397410430255, 'lcs': 0.9916454162768136}
+INFO: 2024-07-12 16:56:47,435: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128009, 198, 271]
+INFO: 2024-07-12 16:56:47,435: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-12 16:56:49,599: llmtf.base.darumeru/cp_para_ru: Loading Dataset: 2.16s
+INFO: 2024-07-12 16:58:42,790: llmtf.base.darumeru/ruTiE: Processing Dataset: 272.57s
+INFO: 2024-07-12 16:58:42,791: llmtf.base.darumeru/ruTiE: Results for darumeru/ruTiE:
+INFO: 2024-07-12 16:58:42,847: llmtf.base.darumeru/ruTiE: {'acc': 0.4441860465116279}
+INFO: 2024-07-12 16:58:42,850: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128009, 198, 271]
+INFO: 2024-07-12 16:58:42,850: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-12 16:58:45,326: llmtf.base.darumeru/ruWorldTree: Loading Dataset: 2.48s
+INFO: 2024-07-12 16:58:48,756: llmtf.base.darumeru/ruWorldTree: Processing Dataset: 3.43s
+INFO: 2024-07-12 16:58:48,758: llmtf.base.darumeru/ruWorldTree: Results for darumeru/ruWorldTree:
+INFO: 2024-07-12 16:58:48,777: llmtf.base.darumeru/ruWorldTree: {'acc': 0.8476190476190476, 'f1_macro': 0.8452567487675969}
+INFO: 2024-07-12 16:58:48,778: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128009, 198, 271]
+INFO: 2024-07-12 16:58:48,778: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-12 16:58:50,784: llmtf.base.darumeru/RWSD: Loading Dataset: 2.01s
+INFO: 2024-07-12 16:58:57,907: llmtf.base.darumeru/RWSD: Processing Dataset: 7.12s
+INFO: 2024-07-12 16:58:57,909: llmtf.base.darumeru/RWSD: Results for darumeru/RWSD:
+INFO: 2024-07-12 16:58:57,912: llmtf.base.darumeru/RWSD: {'acc': 0.5637254901960784}
+INFO: 2024-07-12 16:58:57,914: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128009, 198, 271]
+INFO: 2024-07-12 16:58:57,914: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-12 16:59:00,847: llmtf.base.darumeru/USE: Loading Dataset: 2.93s
+INFO: 2024-07-12 16:59:57,883: llmtf.base.daru/treewayextractive: Processing Dataset: 1233.17s
+INFO: 2024-07-12 16:59:57,884: llmtf.base.daru/treewayextractive: Results for daru/treewayextractive:
+INFO: 2024-07-12 16:59:58,126: llmtf.base.daru/treewayextractive: {'r-prec': 0.4058313131313131}
+INFO: 2024-07-12 16:59:58,534: llmtf.base.evaluator: Ended eval
+INFO: 2024-07-12 16:59:58,544: llmtf.base.evaluator: 
+mean	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruMMLU	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree	nlpcoreteam/enMMLU	nlpcoreteam/ruMMLU
+0.619	0.406	0.197	0.720	0.454	0.564	0.999	0.985	0.512	0.711	0.444	0.846	0.668	0.542
+INFO: 2024-07-12 17:05:59,675: llmtf.base.darumeru/USE: Processing Dataset: 418.83s
+INFO: 2024-07-12 17:05:59,676: llmtf.base.darumeru/USE: Results for darumeru/USE:
+INFO: 2024-07-12 17:05:59,682: llmtf.base.darumeru/USE: {'grade_norm': 0.13333333333333333}
+INFO: 2024-07-12 17:05:59,685: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128009]
+INFO: 2024-07-12 17:05:59,685: llmtf.base.hfmodel: Updated generation_config.stop_strings: []
+INFO: 2024-07-12 17:06:03,784: llmtf.base.russiannlp/rucola_custom: Loading Dataset: 4.10s
+INFO: 2024-07-12 17:07:33,004: llmtf.base.russiannlp/rucola_custom: Processing Dataset: 89.22s
+INFO: 2024-07-12 17:07:33,009: llmtf.base.russiannlp/rucola_custom: Results for russiannlp/rucola_custom:
+INFO: 2024-07-12 17:07:33,037: llmtf.base.russiannlp/rucola_custom: {'acc': 0.7194115536419089, 'mcc': 0.29389648992217976}
+INFO: 2024-07-12 17:07:33,040: llmtf.base.evaluator: Ended eval
+INFO: 2024-07-12 17:07:33,052: llmtf.base.evaluator: 
+mean	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruMMLU	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree	nlpcoreteam/enMMLU	nlpcoreteam/ruMMLU	russiannlp/rucola_custom
+0.579	0.406	0.197	0.720	0.454	0.564	0.133	0.999	0.985	0.512	0.711	0.444	0.846	0.668	0.542	0.507
+INFO: 2024-07-12 17:09:25,911: llmtf.base.darumeru/cp_para_ru: Processing Dataset: 756.31s
+INFO: 2024-07-12 17:09:25,929: llmtf.base.darumeru/cp_para_ru: Results for darumeru/cp_para_ru:
+INFO: 2024-07-12 17:09:25,933: llmtf.base.darumeru/cp_para_ru: {'symbol_per_token': 2.9699926532822025, 'len': 0.9860381898327103, 'lcs': 0.8231040113557345}
+INFO: 2024-07-12 17:09:25,934: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128009, 198, 271]
+INFO: 2024-07-12 17:09:25,935: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-12 17:09:28,044: llmtf.base.darumeru/cp_para_en: Loading Dataset: 2.11s
+INFO: 2024-07-12 17:14:01,272: llmtf.base.daru/treewayabstractive: Processing Dataset: 2085.66s
+INFO: 2024-07-12 17:14:01,291: llmtf.base.daru/treewayabstractive: Results for daru/treewayabstractive:
+INFO: 2024-07-12 17:14:01,294: llmtf.base.daru/treewayabstractive: {'rouge1': 0.35122062259344017, 'rouge2': 0.12350994560599565}
+INFO: 2024-07-12 17:14:01,297: llmtf.base.evaluator: Ended eval
+INFO: 2024-07-12 17:14:01,324: llmtf.base.evaluator: 
+mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_ru	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruMMLU	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree	nlpcoreteam/enMMLU	nlpcoreteam/ruMMLU	russiannlp/rucola_custom
+0.573	0.237	0.406	0.197	0.720	0.454	0.564	0.133	0.823	0.999	0.985	0.512	0.711	0.444	0.846	0.668	0.542	0.507
+INFO: 2024-07-12 17:19:34,717: llmtf.base.darumeru/cp_para_en: Processing Dataset: 606.67s
+INFO: 2024-07-12 17:19:34,735: llmtf.base.darumeru/cp_para_en: Results for darumeru/cp_para_en:
+INFO: 2024-07-12 17:19:34,740: llmtf.base.darumeru/cp_para_en: {'symbol_per_token': 4.480576106482915, 'len': 0.992331176940417, 'lcs': 0.9043190574683162}
+INFO: 2024-07-12 17:19:34,741: llmtf.base.evaluator: Ended eval
+INFO: 2024-07-12 17:19:34,787: llmtf.base.evaluator: 
+mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_en	darumeru/cp_para_ru	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruMMLU	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree	nlpcoreteam/enMMLU	nlpcoreteam/ruMMLU	russiannlp/rucola_custom
+0.592	0.237	0.406	0.197	0.720	0.454	0.564	0.133	0.904	0.823	0.999	0.985	0.512	0.711	0.444	0.846	0.668	0.542	0.507

llmtf_eval_k0/evaluation_results.txt

@@ -0,0 +1,2 @@
+mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_en	darumeru/cp_para_ru	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruMMLU	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree	nlpcoreteam/enMMLU	nlpcoreteam/ruMMLU	russiannlp/rucola_custom
+0.592	0.237	0.406	0.197	0.720	0.454	0.564	0.133	0.904	0.823	0.999	0.985	0.512	0.711	0.444	0.846	0.668	0.542	0.507

llmtf_eval_k0/nlpcoreteam_enMMLU.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b55bdb29816127b80e22272360e6715f888364a0a2c185835d41a24b5d0351c7
+size 38066792

llmtf_eval_k0/nlpcoreteam_enMMLU_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "IlyaGusev/saiga_llama3_8b",
+        "generation_config": {
+            "_from_model_config": true,
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128009
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 64,
+            "pad_token_id": 128000,
+            "stop_strings": [],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2"
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "",
+            "user_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}<|eot_id|>",
+            "bot_message_template_incomplete": "<|start_header_id|>{role}<|end_header_id|>\n\n{content}",
+            "user_role": "user",
+            "bot_role": "assistant",
+            "system_role": "system",
+            "global_prefix": "<|begin_of_text|>",
+            "suffix": "<|start_header_id|>assistant<|end_header_id|>\n\n",
+            "add_special_tokens": false,
+            "eos_token": "<|eot_id|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": false,
+        "space_token": null,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 0,
+        "batch_size": 1,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}