data/developers/abacusai.json

{
  "developer": "abacusai",
  "models": [
    {
      "id": "abacusai/bigstral-12b-32k",
      "name": "bigstral-12b-32k",
      "developer": "abacusai",
      "evaluator_relationship": null,
      "benchmark_scores": {
        "hfopenllm_v2/IFEval": 0.4194,
        "hfopenllm_v2/BBH": 0.47,
        "hfopenllm_v2/MATH Level 5": 0.0151,
        "hfopenllm_v2/GPQA": 0.2928,
        "hfopenllm_v2/MUSR": 0.456,
        "hfopenllm_v2/MMLU-PRO": 0.2641
      }
    },
    {
      "id": "abacusai/bigyi-15b",
      "name": "bigyi-15b",
      "developer": "abacusai",
      "evaluator_relationship": null,
      "benchmark_scores": {
        "hfopenllm_v2/IFEval": 0.2094,
        "hfopenllm_v2/BBH": 0.4345,
        "hfopenllm_v2/MATH Level 5": 0.0295,
        "hfopenllm_v2/GPQA": 0.3096,
        "hfopenllm_v2/MUSR": 0.3538,
        "hfopenllm_v2/MMLU-PRO": 0.3003
      }
    },
    {
      "id": "abacusai/Dracarys-72B-Instruct",
      "name": "Dracarys-72B-Instruct",
      "developer": "abacusai",
      "evaluator_relationship": null,
      "benchmark_scores": {
        "hfopenllm_v2/IFEval": 0.7856,
        "hfopenllm_v2/BBH": 0.6944,
        "hfopenllm_v2/MATH Level 5": 0.3965,
        "hfopenllm_v2/GPQA": 0.3909,
        "hfopenllm_v2/MUSR": 0.4558,
        "hfopenllm_v2/MMLU-PRO": 0.5456
      }
    },
    {
      "id": "abacusai/Liberated-Qwen1.5-14B",
      "name": "Liberated-Qwen1.5-14B",
      "developer": "abacusai",
      "evaluator_relationship": null,
      "benchmark_scores": {
        "hfopenllm_v2/IFEval": 0.3631,
        "hfopenllm_v2/BBH": 0.4948,
        "hfopenllm_v2/MATH Level 5": 0.1601,
        "hfopenllm_v2/GPQA": 0.2836,
        "hfopenllm_v2/MUSR": 0.4175,
        "hfopenllm_v2/MMLU-PRO": 0.3512
      }
    },
    {
      "id": "abacusai/Llama-3-Smaug-8B",
      "name": "Llama-3-Smaug-8B",
      "developer": "abacusai",
      "evaluator_relationship": null,
      "benchmark_scores": {
        "hfopenllm_v2/IFEval": 0.4867,
        "hfopenllm_v2/BBH": 0.4931,
        "hfopenllm_v2/MATH Level 5": 0.0853,
        "hfopenllm_v2/GPQA": 0.2483,
        "hfopenllm_v2/MUSR": 0.3622,
        "hfopenllm_v2/MMLU-PRO": 0.3185
      }
    },
    {
      "id": "abacusai/Smaug-34B-v0.1",
      "name": "Smaug-34B-v0.1",
      "developer": "abacusai",
      "evaluator_relationship": null,
      "benchmark_scores": {
        "hfopenllm_v2/IFEval": 0.5016,
        "hfopenllm_v2/BBH": 0.5358,
        "hfopenllm_v2/MATH Level 5": 0.0718,
        "hfopenllm_v2/GPQA": 0.3297,
        "hfopenllm_v2/MUSR": 0.3979,
        "hfopenllm_v2/MMLU-PRO": 0.4543
      }
    },
    {
      "id": "abacusai/Smaug-72B-v0.1",
      "name": "Smaug-72B-v0.1",
      "developer": "abacusai",
      "evaluator_relationship": null,
      "benchmark_scores": {
        "hfopenllm_v2/IFEval": 0.5167,
        "hfopenllm_v2/BBH": 0.5996,
        "hfopenllm_v2/MATH Level 5": 0.1911,
        "hfopenllm_v2/GPQA": 0.3238,
        "hfopenllm_v2/MUSR": 0.4473,
        "hfopenllm_v2/MMLU-PRO": 0.4624
      }
    },
    {
      "id": "abacusai/Smaug-Llama-3-70B-Instruct-32K",
      "name": "Smaug-Llama-3-70B-Instruct-32K",
      "developer": "abacusai",
      "evaluator_relationship": null,
      "benchmark_scores": {
        "hfopenllm_v2/IFEval": 0.7761,
        "hfopenllm_v2/BBH": 0.6493,
        "hfopenllm_v2/MATH Level 5": 0.2749,
        "hfopenllm_v2/GPQA": 0.2961,
        "hfopenllm_v2/MUSR": 0.4208,
        "hfopenllm_v2/MMLU-PRO": 0.4765
      }
    },
    {
      "id": "abacusai/Smaug-Mixtral-v0.1",
      "name": "Smaug-Mixtral-v0.1",
      "developer": "abacusai",
      "evaluator_relationship": null,
      "benchmark_scores": {
        "hfopenllm_v2/IFEval": 0.5554,
        "hfopenllm_v2/BBH": 0.5162,
        "hfopenllm_v2/MATH Level 5": 0.0952,
        "hfopenllm_v2/GPQA": 0.3012,
        "hfopenllm_v2/MUSR": 0.4298,
        "hfopenllm_v2/MMLU-PRO": 0.3352
      }
    },
    {
      "id": "abacusai/Smaug-Qwen2-72B-Instruct",
      "name": "Smaug-Qwen2-72B-Instruct",
      "developer": "abacusai",
      "evaluator_relationship": null,
      "benchmark_scores": {
        "hfopenllm_v2/IFEval": 0.7825,
        "hfopenllm_v2/BBH": 0.691,
        "hfopenllm_v2/MATH Level 5": 0.4131,
        "hfopenllm_v2/GPQA": 0.3616,
        "hfopenllm_v2/MUSR": 0.4401,
        "hfopenllm_v2/MMLU-PRO": 0.519
      }
    }
  ]
}