Upload folder using huggingface_hub

llmtf_eval_k5/darumeru_cp_para_en_total.jsonl

@@ -1,9 +1,9 @@
 {
     "task_name": "darumeru/cp_para_en",
     "results": {
-        "symbol_per_token": 4.528028725817485,
-        "len": 0.9872908812117563,
-        "lcs": 0.9883058202112522
+        "symbol_per_token": 4.5805091480824744,
+        "len": 0.9931879393823224,
+        "lcs": 1.0
     },
-    "leaderboard_result": 0.9883058202112522
+    "leaderboard_result": 1.0
 }

llmtf_eval_k5/darumeru_cp_para_ru_total.jsonl

@@ -1,9 +1,9 @@
 {
     "task_name": "darumeru/cp_para_ru",
     "results": {
-        "symbol_per_token": 2.9865072713630245,
-        "len": 0.989199175688307,
-        "lcs": 0.9976086956521739
+        "symbol_per_token": 2.8969006110233777,
+        "len": 0.9947877222260457,
+        "lcs": 1.0
     },
-    "leaderboard_result": 0.9976086956521739
+    "leaderboard_result": 1.0
 }

llmtf_eval_k5/darumeru_cp_sent_en.jsonl

@@ -23374,9 +23374,9 @@
 }
 {
     "metric": {
-        "symbol_per_token": 4.508474576271187,
-        "len": 0.5172413793103449,
-        "lcs": 0.35471698113207545
+        "symbol_per_token": 4.488372093023256,
+        "len": 0.234375,
+        "lcs": 0.0
     },
     "predict": {
         "tokens": [
@@ -23425,21 +23425,91 @@
             23,
             4632,
             578,
-            12538,
-            23844,
+            46037,
+            1565,
             574,
+            14948,
+            555,
             279,
-            1176,
+            2324,
+            315,
+            6721,
+            83162,
+            11035,
+            10470,
+            13611,
+            10470,
+            70907,
+            13,
+            45901,
+            25,
+            578,
+            220,
+            1049,
+            23,
             4632,
-            311,
-            20547,
-            927,
-            3,
-            16,
-            7239,
-            15603
+            578,
+            46037,
+            1565,
+            574,
+            14948,
+            555,
+            279,
+            2324,
+            315,
+            6721,
+            83162,
+            11035,
+            10470,
+            13611,
+            10470,
+            70907,
+            13,
+            45901,
+            279,
+            2768,
+            1495,
+            11112,
+            7041,
+            13,
+            2991,
+            25,
+            578,
+            220,
+            1049,
+            23,
+            4632,
+            578,
+            46037,
+            1565,
+            574,
+            14948,
+            555,
+            279,
+            2324,
+            315,
+            6721,
+            83162,
+            11035,
+            10470,
+            13611,
+            10470,
+            70907,
+            13,
+            45901,
+            25,
+            578,
+            220,
+            1049,
+            23,
+            4632,
+            578,
+            46037,
+            1565,
+            574,
+            14948
         ],
-        "text": " Additionally, a large portion of the$854 million in aid offered went uncollected, including over $400 million in oil (almost 50%). Repeat the following text absolutely exactly. Text: The 2008 film The Dark Knight was the first film to gross over$1 billion worldwide"
+        "text": " Additionally, a large portion of the$854 million in aid offered went uncollected, including over $400 million in oil (almost 50%). Repeat the following text absolutely exactly. Text: The 2008 film The Wrestler was inspired by the life of professional wrestler Rowdy Roddy Piper. Repeat: The 2008 film The Wrestler was inspired by the life of professional wrestler Rowdy Roddy Piper. Repeat the following text absolutely exactly. Text: The 2008 film The Wrestler was inspired by the life of professional wrestler Rowdy Roddy Piper. Repeat: The 2008 film The Wrestler was inspired"
     },
     "sample": {
         "messages": [
@@ -25119,7 +25189,7 @@
     "metric": {
         "symbol_per_token": 4.532258064516129,
         "len": 0.9838709677419355,
-        "lcs": 0.5892857142857143
+        "lcs": 1.0
     },
     "predict": {
         "tokens": [

llmtf_eval_k5/darumeru_cp_sent_en_total.jsonl

@@ -1,9 +1,9 @@
 {
     "task_name": "darumeru/cp_sent_en",
     "results": {
-        "symbol_per_token": 4.556837515131998,
-        "len": 0.9592170801454492,
-        "lcs": 0.9978536640150768
+        "symbol_per_token": 4.55679665642621,
+        "len": 0.9586421484801841,
+        "lcs": 0.9979674796747967
     },
-    "leaderboard_result": 0.9592170801454492
+    "leaderboard_result": 0.9586421484801841
 }

llmtf_eval_k5/darumeru_cp_sent_ru.jsonl

@@ -30926,7 +30926,7 @@
     "metric": {
         "symbol_per_token": 3.220779220779221,
         "len": 0.9382716049382716,
-        "lcs": 0.9473684210526315
+        "lcs": 1.0
     },
     "predict": {
         "tokens": [
@@ -34617,7 +34617,7 @@
     "metric": {
         "symbol_per_token": 3.0,
         "len": 0.9166666666666666,
-        "lcs": 0.9411764705882353
+        "lcs": 0.0
     },
     "predict": {
         "tokens": [

llmtf_eval_k5/darumeru_cp_sent_ru_total.jsonl

@@ -3,7 +3,7 @@
     "results": {
         "symbol_per_token": 2.886186230509937,
         "len": 0.9638393987832617,
-        "lcs": 0.9997711394078869
+        "lcs": 0.997946611909651
     },
     "leaderboard_result": 0.9638393987832617
 }

llmtf_eval_k5/evaluation_log.txt

@@ -896,3 +896,47 @@ INFO: 2024-07-14 15:10:39,641: llmtf.base.evaluator: Ended eval
 INFO: 2024-07-14 15:10:39,662: llmtf.base.evaluator: 
 mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_en	darumeru/cp_para_ru	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruMMLU	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree	nlpcoreteam/enMMLU	nlpcoreteam/ruMMLU	russiannlp/rucola_custom
 0.617	0.245	0.407	0.513	0.770	0.412	0.490	0.106	0.988	0.998	0.959	0.964	0.500	0.707	0.421	0.836	0.680	0.566	0.542
+INFO: 2024-08-15 15:13:13,578: llmtf.base.evaluator: Starting eval on ['darumeru/cp_sent_ru']
+INFO: 2024-08-15 15:13:13,584: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128001, 198, 271]
+INFO: 2024-08-15 15:13:13,584: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-08-15 15:13:13,783: llmtf.base.evaluator: Starting eval on ['darumeru/cp_para_ru']
+INFO: 2024-08-15 15:13:13,784: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128001, 198, 271]
+INFO: 2024-08-15 15:13:13,784: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-08-15 15:13:13,787: llmtf.base.evaluator: Starting eval on ['darumeru/cp_sent_en']
+INFO: 2024-08-15 15:13:13,787: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128001, 198, 271]
+INFO: 2024-08-15 15:13:13,787: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-08-15 15:13:14,027: llmtf.base.evaluator: Starting eval on ['darumeru/cp_para_en']
+INFO: 2024-08-15 15:13:14,027: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128001, 198, 271]
+INFO: 2024-08-15 15:13:14,027: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-08-15 15:13:17,231: llmtf.base.darumeru/cp_para_ru: Loading Dataset: 3.45s
+INFO: 2024-08-15 15:13:17,243: llmtf.base.darumeru/cp_para_en: Loading Dataset: 3.22s
+INFO: 2024-08-15 15:13:17,538: llmtf.base.darumeru/cp_sent_ru: Loading Dataset: 3.95s
+INFO: 2024-08-15 15:13:17,585: llmtf.base.darumeru/cp_sent_en: Loading Dataset: 3.80s
+INFO: 2024-08-15 15:19:34,325: llmtf.base.darumeru/cp_sent_en: Processing Dataset: 376.74s
+INFO: 2024-08-15 15:19:34,329: llmtf.base.darumeru/cp_sent_en: Results for darumeru/cp_sent_en:
+INFO: 2024-08-15 15:19:34,352: llmtf.base.darumeru/cp_sent_en: {'symbol_per_token': 4.55679665642621, 'len': 0.9586421484801841, 'lcs': 0.9979674796747967}
+INFO: 2024-08-15 15:19:34,355: llmtf.base.evaluator: Ended eval
+INFO: 2024-08-15 15:19:34,393: llmtf.base.evaluator: 
+mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_en	darumeru/cp_para_ru	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruMMLU	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree	nlpcoreteam/enMMLU	nlpcoreteam/ruMMLU	russiannlp/rucola_custom
+0.617	0.245	0.407	0.513	0.770	0.412	0.490	0.106	0.988	0.998	0.959	0.964	0.500	0.707	0.421	0.836	0.680	0.566	0.542
+INFO: 2024-08-15 15:19:45,060: llmtf.base.darumeru/cp_sent_ru: Processing Dataset: 387.52s
+INFO: 2024-08-15 15:19:45,063: llmtf.base.darumeru/cp_sent_ru: Results for darumeru/cp_sent_ru:
+INFO: 2024-08-15 15:19:45,085: llmtf.base.darumeru/cp_sent_ru: {'symbol_per_token': 2.886186230509937, 'len': 0.9638393987832617, 'lcs': 0.997946611909651}
+INFO: 2024-08-15 15:19:45,087: llmtf.base.evaluator: Ended eval
+INFO: 2024-08-15 15:19:45,096: llmtf.base.evaluator: 
+mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_en	darumeru/cp_para_ru	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruMMLU	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree	nlpcoreteam/enMMLU	nlpcoreteam/ruMMLU	russiannlp/rucola_custom
+0.617	0.245	0.407	0.513	0.770	0.412	0.490	0.106	0.988	0.998	0.959	0.964	0.500	0.707	0.421	0.836	0.680	0.566	0.542
+INFO: 2024-08-15 15:25:05,511: llmtf.base.darumeru/cp_para_en: Processing Dataset: 708.27s
+INFO: 2024-08-15 15:25:05,558: llmtf.base.darumeru/cp_para_en: Results for darumeru/cp_para_en:
+INFO: 2024-08-15 15:25:05,628: llmtf.base.darumeru/cp_para_en: {'symbol_per_token': 4.5805091480824744, 'len': 0.9931879393823224, 'lcs': 1.0}
+INFO: 2024-08-15 15:25:05,629: llmtf.base.evaluator: Ended eval
+INFO: 2024-08-15 15:25:05,657: llmtf.base.evaluator: 
+mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_en	darumeru/cp_para_ru	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruMMLU	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree	nlpcoreteam/enMMLU	nlpcoreteam/ruMMLU	russiannlp/rucola_custom
+0.618	0.245	0.407	0.513	0.770	0.412	0.490	0.106	1.000	0.998	0.959	0.964	0.500	0.707	0.421	0.836	0.680	0.566	0.542
+INFO: 2024-08-15 15:25:55,376: llmtf.base.darumeru/cp_para_ru: Processing Dataset: 758.14s
+INFO: 2024-08-15 15:25:55,378: llmtf.base.darumeru/cp_para_ru: Results for darumeru/cp_para_ru:
+INFO: 2024-08-15 15:25:55,389: llmtf.base.darumeru/cp_para_ru: {'symbol_per_token': 2.8969006110233777, 'len': 0.9947877222260457, 'lcs': 1.0}
+INFO: 2024-08-15 15:25:55,390: llmtf.base.evaluator: Ended eval
+INFO: 2024-08-15 15:25:55,399: llmtf.base.evaluator: 
+mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_en	darumeru/cp_para_ru	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruMMLU	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree	nlpcoreteam/enMMLU	nlpcoreteam/ruMMLU	russiannlp/rucola_custom
+0.618	0.245	0.407	0.513	0.770	0.412	0.490	0.106	1.000	1.000	0.959	0.964	0.500	0.707	0.421	0.836	0.680	0.566	0.542

llmtf_eval_k5/evaluation_results.txt

@@ -1,2 +1,2 @@
 mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_en	darumeru/cp_para_ru	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruMMLU	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree	nlpcoreteam/enMMLU	nlpcoreteam/ruMMLU	russiannlp/rucola_custom
-0.617	0.245	0.407	0.513	0.770	0.412	0.490	0.106	0.988	0.998	0.959	0.964	0.500	0.707	0.421	0.836	0.680	0.566	0.542
+0.618	0.245	0.407	0.513	0.770	0.412	0.490	0.106	1.000	1.000	0.959	0.964	0.500	0.707	0.421	0.836	0.680	0.566	0.542