Release: Quartz-R1 8B GGUF Quants Family with Full Evaluation Suite Results (part 2)

results.json

@@ -0,0 +1,566 @@
+{
+  "model_name": "quartz_r1_genesis_clean",
+  "date": "2026-08-18",
+  "results": {
+    "gsm8k": {
+      "name": "gsm8k",
+      "alias": "gsm8k",
+      "sample_len": 1319,
+      "exact_match,strict-match": 0.7384382107657316,
+      "exact_match_stderr,strict-match": 0.012105605733382459,
+      "exact_match,flexible-extract": 0.7422289613343442,
+      "exact_match_stderr,flexible-extract": 0.01204837021357661
+    },
+    "minerva_math500": {
+      "name": "minerva_math500",
+      "alias": "minerva_math500",
+      "sample_len": 500,
+      "exact_match,none": 0.394,
+      "exact_match_stderr,none": 0.021874299301689257,
+      "math_verify,none": 0.434,
+      "math_verify_stderr,none": 0.022187215803029008
+    },
+    "bbh_cot_fewshot": {
+      "alias": "bbh_cot_fewshot",
+      "name": "bbh_cot_fewshot",
+      "sample_len": 6511,
+      "exact_match,get-answer": 0.6848410382429734,
+      "exact_match_stderr,get-answer": 0.005162813446797612,
+      "sample_count": {
+        "exact_match,get-answer": 6511
+      }
+    },
+    "mmlu_pro": {
+      "alias": "mmlu_pro",
+      "name": "mmlu_pro",
+      "sample_len": 12032,
+      "exact_match,custom-extract": 0.4493849734042553,
+      "exact_match_stderr,custom-extract": 0.004369131624528862,
+      "sample_count": {
+        "exact_match,custom-extract": 12032
+      }
+    },
+    "ifeval": {
+      "name": "ifeval",
+      "alias": "ifeval",
+      "sample_len": 541,
+      "prompt_level_strict_acc,none": 0.38817005545286504,
+      "prompt_level_strict_acc_stderr,none": 0.020971500215794775,
+      "inst_level_strict_acc,none": 0.5071942446043165,
+      "inst_level_strict_acc_stderr,none": "N/A",
+      "prompt_level_loose_acc,none": 0.43622920517560076,
+      "prompt_level_loose_acc_stderr,none": 0.02134085308994027,
+      "inst_level_loose_acc,none": 0.539568345323741,
+      "inst_level_loose_acc_stderr,none": "N/A"
+    },
+    "drop": {
+      "name": "drop",
+      "alias": "drop",
+      "sample_len": 9536,
+      "em,none": 0.0,
+      "em_stderr,none": 0.0,
+      "f1,none": 0.04738255033557058,
+      "f1_stderr,none": 0.001040437534856409
+    },
+    "hle": {
+      "acc,none": 0.3284
+    },
+    "ru_mmlu": {
+      "acc,none": 0.2518
+    },
+    "ru_humaneval": {
+      "acc,none": 0.23170000000000002
+    },
+    "deep_swe": {
+      "acc,none": 0.012
+    },
+    "gpqa_diamond_cot_zeroshot": {
+      "name": "gpqa_diamond_cot_zeroshot",
+      "alias": "gpqa_diamond_cot_zeroshot",
+      "sample_len": 198,
+      "exact_match,strict-match": 0.0,
+      "exact_match_stderr,strict-match": 0.0,
+      "exact_match,flexible-extract": 0.13131313131313133,
+      "exact_match_stderr,flexible-extract": 0.024063156416822527
+    },
+    "gpqa_main_cot_zeroshot": {
+      "name": "gpqa_main_cot_zeroshot",
+      "alias": "gpqa_main_cot_zeroshot",
+      "sample_len": 448,
+      "exact_match,strict-match": 0.004464285714285714,
+      "exact_match_stderr,strict-match": 0.0031531937104656623,
+      "exact_match,flexible-extract": 0.19642857142857142,
+      "exact_match_stderr,flexible-extract": 0.018791472419524242
+    },
+    "arc_challenge": {
+      "acc,none": 0.8676999999999999
+    },
+    "hellaswag": {
+      "acc,none": 0.7190000000000001
+    },
+    "winogrande": {
+      "acc,none": 0.49090000000000006
+    },
+    "truthfulqa_mc2": {
+      "acc,none": 0.2827
+    },
+    "humaneval": {
+      "acc,none": 0.0
+    },
+    "mbpp": {
+      "acc,none": 0.0
+    }
+  },
+  "configs": {
+    "gsm8k": {
+      "task": "gsm8k",
+      "dataset_path": "openai/gsm8k",
+      "dataset_name": "main",
+      "training_split": "train",
+      "test_split": "test",
+      "fewshot_split": "train",
+      "doc_to_text": "Question: {{question}}\nAnswer:",
+      "doc_to_target": "{{answer}}",
+      "unsafe_code": false,
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": "train",
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "Question: {{question}}\nAnswer:",
+        "doc_to_choice": null,
+        "doc_to_target": "{{answer}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true,
+          "ignore_case": true,
+          "ignore_punctuation": false,
+          "regexes_to_ignore": [
+            ",",
+            "\\$",
+            "(?s).*#### ",
+            "\\.$"
+          ]
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Question:",
+          "</s>",
+          "<|im_end|>"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "filter_list": [
+        {
+          "name": "strict-match",
+          "filter": [
+            {
+              "function": "regex",
+              "regex_pattern": "#### (\\-?[0-9\\.\\,]+)"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        },
+        {
+          "name": "flexible-extract",
+          "filter": [
+            {
+              "function": "regex",
+              "group_select": -1,
+              "regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        }
+      ],
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 3.0,
+        "model": "quartz_r1_genesis_clean",
+        "base_url": "http://127.0.0.1:8080/v1/chat/completions",
+        "num_concurrent": 2,
+        "max_retries": 3,
+        "config_source": "/Users/vaultek/Documents/ai-fine-tuning/venv/lib/python3.11/site-packages/lm_eval/tasks/gsm8k/gsm8k.yaml"
+      }
+    },
+    "minerva_math500": {
+      "task": "minerva_math500",
+      "dataset_path": "HuggingFaceH4/MATH-500",
+      "dataset_name": "default",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc: dict) -> dict:\n        out_doc = {\n            \"problem\": doc[\"problem\"],\n            \"solution\": doc[\"solution\"],\n            \"answer\": normalize_final_answer(\n                remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n            ),\n        }\n        if getattr(doc, \"few_shot\", None) is not None:\n            out_doc[\"few_shot\"] = True\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "def doc_to_text(doc: dict) -> str:\n    return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
+      "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc: dict, results: list[str]) -> dict[str, int]:\n    candidates = results[0]\n\n    unnormalized_answer = get_unnormalized_answer(candidates)\n    answer = normalize_final_answer(unnormalized_answer)\n\n    if is_equiv(answer, doc[\"answer\"]):\n        retval = 1\n    else:\n        retval = 0\n\n    # math_verify\n    _mvres = verify(\n        gold=parse(doc[\"solution\"]),\n        target=parse(candidates),\n    )\n    mathval = 1 if _mvres else 0\n\n    res = {\n        \"exact_match\": retval,\n        \"math_verify\": mathval,\n    }\n    return res\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "split": null,
+        "process_docs": "<function process_docs at 0x7fb63b535760>",
+        "fewshot_indices": null,
+        "samples": "<function list_fewshot_samples at 0x7fb639b6a660>",
+        "doc_to_text": "<function doc_to_text at 0x7fb63a02e700>",
+        "doc_to_choice": null,
+        "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 4,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "math_verify",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Problem:"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 3.0,
+        "model": "quartz_r1_genesis_clean",
+        "base_url": "http://127.0.0.1:8080/v1/chat/completions",
+        "num_concurrent": 2,
+        "max_retries": 3,
+        "config_source": "/Users/vaultek/Documents/ai-fine-tuning/venv/lib/python3.11/site-packages/lm_eval/tasks/minerva_math/minerva_math500.yaml"
+      }
+    },
+    "bbh_cot_fewshot": {},
+    "mmlu_pro": {},
+    "ifeval": {
+      "task": "ifeval",
+      "dataset_path": "google/IFEval",
+      "test_split": "train",
+      "doc_to_text": "prompt",
+      "doc_to_target": 0,
+      "unsafe_code": false,
+      "process_results": "def process_results(doc, results):\n    inp = InputExample(\n        key=doc[\"key\"],\n        instruction_id_list=doc[\"instruction_id_list\"],\n        prompt=doc[\"prompt\"],\n        kwargs=doc[\"kwargs\"],\n    )\n    response = results[0]\n\n    out_strict = test_instruction_following_strict(inp, response)\n    out_loose = test_instruction_following_loose(inp, response)\n\n    return {\n        \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n        \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n        \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n        \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": null,
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "prompt",
+        "doc_to_choice": null,
+        "doc_to_target": 0,
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "prompt_level_strict_acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "inst_level_strict_acc",
+          "aggregation": "def agg_inst_level_acc(items):\n    flat_items = [item for sublist in items for item in sublist]\n    inst_level_acc = sum(flat_items) / len(flat_items)\n    return inst_level_acc\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "prompt_level_loose_acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "inst_level_loose_acc",
+          "aggregation": "def agg_inst_level_acc(items):\n    flat_items = [item for sublist in items for item in sublist]\n    inst_level_acc = sum(flat_items) / len(flat_items)\n    return inst_level_acc\n",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [],
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 1280
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 4.0,
+        "model": "quartz_r1_genesis_clean",
+        "base_url": "http://127.0.0.1:8080/v1/chat/completions",
+        "num_concurrent": 2,
+        "max_retries": 3,
+        "config_source": "/Users/vaultek/Documents/ai-fine-tuning/venv/lib/python3.11/site-packages/lm_eval/tasks/ifeval/ifeval.yaml"
+      }
+    },
+    "drop": {
+      "task": "drop",
+      "dataset_path": "EleutherAI/drop",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset):\n    def _process(doc):\n        return {\n            \"id\": doc[\"query_id\"],\n            \"passage\": doc[\"passage\"],\n            \"question\": doc[\"question\"],\n            \"answers\": get_answers(doc),\n        }\n\n    return dataset.map(_process)\n",
+      "doc_to_text": "{{passage}} {{question}}",
+      "doc_to_target": "{{ answer|join(',')}}",
+      "unsafe_code": false,
+      "process_results": "def process_results(doc, results):\n    preds, golds = results, doc[\"answers\"]\n    max_em = 0\n    max_f1 = 0\n    for gold_answer in golds:\n        exact_match, f1_score = get_metrics(preds, gold_answer)\n        if gold_answer[0].strip():\n            max_em = max(max_em, exact_match)\n            max_f1 = max(max_f1, f1_score)\n    return {\"em\": max_em, \"f1\": max_f1}\n",
+      "description": "",
+      "target_delimiter": "",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": "<function process_docs at 0x7fd1fa935580>",
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "{{passage}} {{question}}",
+        "doc_to_choice": null,
+        "doc_to_target": "{{ answer|join(',')}}",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": ""
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "em",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "f1",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "."
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "{{passage}} {{question}}",
+      "metadata": {
+        "version": 3.0,
+        "model": "quartz_r1_genesis_clean",
+        "base_url": "http://127.0.0.1:8080/v1/chat/completions",
+        "num_concurrent": 2,
+        "max_retries": 3,
+        "config_source": "/Users/vaultek/Documents/ai-fine-tuning/venv/lib/python3.11/site-packages/lm_eval/tasks/drop/default.yaml"
+      }
+    },
+    "hle": {},
+    "ru_mmlu": {},
+    "ru_humaneval": {},
+    "deep_swe": {},
+    "gpqa_diamond_cot_zeroshot": {
+      "task": "gpqa_diamond_cot_zeroshot",
+      "dataset_path": "Idavidrein/gpqa",
+      "dataset_name": "gpqa_diamond",
+      "training_split": "train",
+      "validation_split": "train",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        choices = [\n            preprocess(doc[\"Incorrect Answer 1\"]),\n            preprocess(doc[\"Incorrect Answer 2\"]),\n            preprocess(doc[\"Incorrect Answer 3\"]),\n            preprocess(doc[\"Correct Answer\"]),\n        ]\n\n        random.shuffle(choices)\n        correct_answer_index = choices.index(preprocess(doc[\"Correct Answer\"]))\n\n        out_doc = {\n            \"choice1\": choices[0],\n            \"choice2\": choices[1],\n            \"choice3\": choices[2],\n            \"choice4\": choices[3],\n            \"choices\": [choices[0], choices[1], choices[2], choices[3]],\n            \"answer\": f\"({chr(65 + correct_answer_index)})\",\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "What is the correct answer to this question:{{Question}}\nChoices:\n(A) {{choice1}}\n(B) {{choice2}}\n(C) {{choice3}}\n(D) {{choice4}}\nLet's think step by step: ",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": "<function process_docs at 0x7efb8f515620>",
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "What is the correct answer to this question:{{Question}}\nChoices:\n(A) {{choice1}}\n(B) {{choice2}}\n(C) {{choice3}}\n(D) {{choice4}}\nLet's think step by step: ",
+        "doc_to_choice": null,
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true,
+          "ignore_case": true,
+          "ignore_punctuation": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "</s>"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "filter_list": [
+        {
+          "name": "strict-match",
+          "filter": [
+            {
+              "function": "regex",
+              "regex_pattern": "(?<=The answer is )(.*)(?=.)"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        },
+        {
+          "name": "flexible-extract",
+          "filter": [
+            {
+              "function": "multi_choice_regex",
+              "group_select": -1,
+              "ignore_case": true,
+              "ignore_punctuation": true,
+              "regex_pattern": "(\\([A-Z]\\))"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        }
+      ],
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.2,
+        "model": "quartz_r1_genesis_clean",
+        "base_url": "http://127.0.0.1:8080/v1/chat/completions",
+        "num_concurrent": 2,
+        "max_retries": 3,
+        "config_source": "/Users/vaultek/Documents/ai-fine-tuning/venv/lib/python3.11/site-packages/lm_eval/tasks/gpqa/cot_zeroshot/gpqa_diamond_cot_zeroshot.yaml"
+      }
+    },
+    "gpqa_main_cot_zeroshot": {
+      "task": "gpqa_main_cot_zeroshot",
+      "dataset_path": "Idavidrein/gpqa",
+      "dataset_name": "gpqa_main",
+      "training_split": "train",
+      "validation_split": "train",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        choices = [\n            preprocess(doc[\"Incorrect Answer 1\"]),\n            preprocess(doc[\"Incorrect Answer 2\"]),\n            preprocess(doc[\"Incorrect Answer 3\"]),\n            preprocess(doc[\"Correct Answer\"]),\n        ]\n\n        random.shuffle(choices)\n        correct_answer_index = choices.index(preprocess(doc[\"Correct Answer\"]))\n\n        out_doc = {\n            \"choice1\": choices[0],\n            \"choice2\": choices[1],\n            \"choice3\": choices[2],\n            \"choice4\": choices[3],\n            \"choices\": [choices[0], choices[1], choices[2], choices[3]],\n            \"answer\": f\"({chr(65 + correct_answer_index)})\",\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "What is the correct answer to this question:{{Question}}\nChoices:\n(A) {{choice1}}\n(B) {{choice2}}\n(C) {{choice3}}\n(D) {{choice4}}\nLet's think step by step: ",
+      "doc_to_target": "answer",
+      "unsafe_code": false,
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "default",
+        "split": null,
+        "process_docs": "<function process_docs at 0x7fa5fb545620>",
+        "fewshot_indices": null,
+        "samples": null,
+        "doc_to_text": "What is the correct answer to this question:{{Question}}\nChoices:\n(A) {{choice1}}\n(B) {{choice2}}\n(C) {{choice3}}\n(D) {{choice4}}\nLet's think step by step: ",
+        "doc_to_choice": null,
+        "doc_to_target": "answer",
+        "gen_prefix": null,
+        "fewshot_delimiter": "\n\n",
+        "target_delimiter": " "
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true,
+          "ignore_case": true,
+          "ignore_punctuation": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "</s>"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "filter_list": [
+        {
+          "name": "strict-match",
+          "filter": [
+            {
+              "function": "regex",
+              "regex_pattern": "(?<=The answer is )(.*)(?=.)"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        },
+        {
+          "name": "flexible-extract",
+          "filter": [
+            {
+              "function": "multi_choice_regex",
+              "group_select": -1,
+              "ignore_case": true,
+              "ignore_punctuation": true,
+              "regex_pattern": "(\\([A-Z]\\))"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        }
+      ],
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.2,
+        "model": "quartz_r1_genesis_clean",
+        "base_url": "http://127.0.0.1:8080/v1/chat/completions",
+        "num_concurrent": 2,
+        "max_retries": 3,
+        "config_source": "/Users/vaultek/Documents/ai-fine-tuning/venv/lib/python3.11/site-packages/lm_eval/tasks/gpqa/cot_zeroshot/gpqa_main_cot_zeroshot.yaml"
+      }
+    },
+    "arc_challenge": {},
+    "hellaswag": {},
+    "winogrande": {},
+    "truthfulqa_mc2": {},
+    "humaneval": {},
+    "mbpp": {}
+  }
+}