evaluate_result/2_codegemma-7b-result.json

{
  "humaneval": {
    "pass@1": [
      [
        0,
        1.0
      ],
      [
        1,
        0.0
      ],
      [
        2,
        1.0
      ],
      [
        3,
        1.0
      ],
      [
        4,
        1.0
      ],
      [
        5,
        0.4999999999999999
      ],
      [
        6,
        0.050000000000000044
      ],
      [
        7,
        1.0
      ],
      [
        8,
        1.0
      ],
      [
        9,
        0.5499999999999998
      ],
      [
        10,
        0.0
      ],
      [
        11,
        1.0
      ],
      [
        12,
        0.4999999999999999
      ],
      [
        13,
        1.0
      ],
      [
        14,
        1.0
      ],
      [
        15,
        1.0
      ],
      [
        16,
        0.5499999999999998
      ],
      [
        17,
        0.30000000000000004
      ],
      [
        18,
        0.09999999999999998
      ],
      [
        19,
        0.0
      ],
      [
        20,
        0.09999999999999998
      ],
      [
        21,
        0.3500000000000001
      ],
      [
        22,
        1.0
      ],
      [
        23,
        1.0
      ],
      [
        24,
        0.65
      ],
      [
        25,
        0.4
      ],
      [
        26,
        0.0
      ],
      [
        27,
        1.0
      ],
      [
        28,
        0.85
      ],
      [
        29,
        1.0
      ],
      [
        30,
        0.95
      ],
      [
        31,
        0.95
      ],
      [
        32,
        0.0
      ],
      [
        33,
        0.0
      ],
      [
        34,
        1.0
      ],
      [
        35,
        1.0
      ],
      [
        36,
        0.15000000000000002
      ],
      [
        37,
        0.0
      ],
      [
        38,
        1.0
      ],
      [
        39,
        0.0
      ],
      [
        40,
        0.95
      ],
      [
        41,
        0.0
      ],
      [
        42,
        1.0
      ],
      [
        43,
        0.8999999999999999
      ],
      [
        44,
        0.75
      ],
      [
        45,
        1.0
      ],
      [
        46,
        0.7
      ],
      [
        47,
        0.75
      ],
      [
        48,
        0.8999999999999999
      ],
      [
        49,
        0.8999999999999999
      ],
      [
        50,
        1.0
      ],
      [
        51,
        0.5499999999999998
      ],
      [
        52,
        1.0
      ],
      [
        53,
        1.0
      ],
      [
        54,
        0.0
      ],
      [
        55,
        0.75
      ],
      [
        56,
        0.44999999999999984
      ],
      [
        57,
        0.5499999999999998
      ],
      [
        58,
        1.0
      ],
      [
        59,
        0.15000000000000002
      ],
      [
        60,
        1.0
      ],
      [
        61,
        0.6000000000000001
      ],
      [
        62,
        0.65
      ],
      [
        63,
        1.0
      ],
      [
        64,
        0.0
      ],
      [
        65,
        0.25
      ],
      [
        66,
        1.0
      ],
      [
        67,
        0.0
      ],
      [
        68,
        0.6000000000000001
      ],
      [
        69,
        0.09999999999999998
      ],
      [
        70,
        0.20000000000000007
      ],
      [
        71,
        0.7999999999999999
      ],
      [
        72,
        1.0
      ],
      [
        73,
        0.20000000000000007
      ],
      [
        74,
        0.44999999999999984
      ],
      [
        75,
        0.09999999999999998
      ],
      [
        76,
        0.7
      ],
      [
        77,
        0.050000000000000044
      ],
      [
        78,
        0.4999999999999999
      ],
      [
        79,
        0.5499999999999998
      ],
      [
        80,
        0.44999999999999984
      ],
      [
        81,
        0.0
      ],
      [
        82,
        0.050000000000000044
      ],
      [
        83,
        0.0
      ],
      [
        84,
        0.0
      ],
      [
        85,
        0.25
      ],
      [
        86,
        0.15000000000000002
      ],
      [
        87,
        0.3500000000000001
      ],
      [
        88,
        0.20000000000000007
      ],
      [
        89,
        0.25
      ],
      [
        90,
        0.30000000000000004
      ],
      [
        91,
        0.0
      ],
      [
        92,
        0.6000000000000001
      ],
      [
        93,
        0.050000000000000044
      ],
      [
        94,
        0.0
      ],
      [
        95,
        0.4
      ],
      [
        96,
        0.4
      ],
      [
        97,
        0.75
      ],
      [
        98,
        0.7
      ],
      [
        99,
        0.0
      ],
      [
        100,
        0.050000000000000044
      ],
      [
        101,
        0.30000000000000004
      ],
      [
        102,
        0.15000000000000002
      ],
      [
        103,
        0.65
      ],
      [
        104,
        0.20000000000000007
      ],
      [
        105,
        0.050000000000000044
      ],
      [
        106,
        0.0
      ],
      [
        107,
        0.6000000000000001
      ],
      [
        108,
        0.0
      ],
      [
        109,
        0.0
      ],
      [
        110,
        0.050000000000000044
      ],
      [
        111,
        0.15000000000000002
      ],
      [
        112,
        0.25
      ],
      [
        113,
        0.44999999999999984
      ],
      [
        114,
        0.4999999999999999
      ],
      [
        115,
        0.09999999999999998
      ],
      [
        116,
        1.0
      ],
      [
        117,
        0.25
      ],
      [
        118,
        0.0
      ],
      [
        119,
        0.0
      ],
      [
        120,
        0.15000000000000002
      ],
      [
        121,
        0.4
      ],
      [
        122,
        0.75
      ],
      [
        123,
        0.15000000000000002
      ],
      [
        124,
        0.20000000000000007
      ],
      [
        125,
        0.09999999999999998
      ],
      [
        126,
        0.0
      ],
      [
        127,
        0.050000000000000044
      ],
      [
        128,
        0.20000000000000007
      ],
      [
        129,
        0.0
      ],
      [
        130,
        0.0
      ],
      [
        131,
        0.0
      ],
      [
        132,
        0.0
      ],
      [
        133,
        0.0
      ],
      [
        134,
        0.0
      ],
      [
        135,
        0.0
      ],
      [
        136,
        0.75
      ],
      [
        137,
        0.0
      ],
      [
        138,
        0.09999999999999998
      ],
      [
        139,
        0.0
      ],
      [
        140,
        0.0
      ],
      [
        141,
        0.0
      ],
      [
        142,
        0.20000000000000007
      ],
      [
        143,
        0.15000000000000002
      ],
      [
        144,
        0.3500000000000001
      ],
      [
        145,
        0.0
      ],
      [
        146,
        0.0
      ],
      [
        147,
        0.3500000000000001
      ],
      [
        148,
        0.30000000000000004
      ],
      [
        149,
        0.25
      ],
      [
        150,
        0.050000000000000044
      ],
      [
        151,
        0.3500000000000001
      ],
      [
        152,
        0.75
      ],
      [
        153,
        0.15000000000000002
      ],
      [
        154,
        0.20000000000000007
      ],
      [
        155,
        0.0
      ],
      [
        156,
        0.30000000000000004
      ],
      [
        157,
        0.3500000000000001
      ],
      [
        158,
        0.25
      ],
      [
        159,
        0.09999999999999998
      ],
      [
        160,
        0.0
      ],
      [
        161,
        0.09999999999999998
      ],
      [
        162,
        0.5499999999999998
      ],
      [
        163,
        0.0
      ]
    ],
    "pass@10": [
      [
        0,
        1.0
      ],
      [
        1,
        0.0
      ],
      [
        2,
        1.0
      ],
      [
        3,
        1.0
      ],
      [
        4,
        1.0
      ],
      [
        5,
        0.9999945874558878
      ],
      [
        6,
        0.5
      ],
      [
        7,
        1.0
      ],
      [
        8,
        1.0
      ],
      [
        9,
        1.0
      ],
      [
        10,
        0.0
      ],
      [
        11,
        1.0
      ],
      [
        12,
        0.9999945874558878
      ],
      [
        13,
        1.0
      ],
      [
        14,
        1.0
      ],
      [
        15,
        1.0
      ],
      [
        16,
        1.0
      ],
      [
        17,
        0.9945820433436533
      ],
      [
        18,
        0.763157894736842
      ],
      [
        19,
        0.0
      ],
      [
        20,
        0.763157894736842
      ],
      [
        21,
        0.9984520123839009
      ],
      [
        22,
        1.0
      ],
      [
        23,
        1.0
      ],
      [
        24,
        1.0
      ],
      [
        25,
        0.9996427720885925
      ],
      [
        26,
        0.0
      ],
      [
        27,
        1.0
      ],
      [
        28,
        1.0
      ],
      [
        29,
        1.0
      ],
      [
        30,
        1.0
      ],
      [
        31,
        1.0
      ],
      [
        32,
        0.0
      ],
      [
        33,
        0.0
      ],
      [
        34,
        1.0
      ],
      [
        35,
        1.0
      ],
      [
        36,
        0.8947368421052632
      ],
      [
        37,
        0.0
      ],
      [
        38,
        1.0
      ],
      [
        39,
        0.0
      ],
      [
        40,
        1.0
      ],
      [
        41,
        0.0
      ],
      [
        42,
        1.0
      ],
      [
        43,
        1.0
      ],
      [
        44,
        1.0
      ],
      [
        45,
        1.0
      ],
      [
        46,
        1.0
      ],
      [
        47,
        1.0
      ],
      [
        48,
        1.0
      ],
      [
        49,
        1.0
      ],
      [
        50,
        1.0
      ],
      [
        51,
        1.0
      ],
      [
        52,
        1.0
      ],
      [
        53,
        1.0
      ],
      [
        54,
        0.0
      ],
      [
        55,
        1.0
      ],
      [
        56,
        0.9999404620147654
      ],
      [
        57,
        1.0
      ],
      [
        58,
        1.0
      ],
      [
        59,
        0.8947368421052632
      ],
      [
        60,
        1.0
      ],
      [
        61,
        1.0
      ],
      [
        62,
        1.0
      ],
      [
        63,
        1.0
      ],
      [
        64,
        0.0
      ],
      [
        65,
        0.9837461300309598
      ],
      [
        66,
        1.0
      ],
      [
        67,
        0.0
      ],
      [
        68,
        1.0
      ],
      [
        69,
        0.763157894736842
      ],
      [
        70,
        0.956656346749226
      ],
      [
        71,
        1.0
      ],
      [
        72,
        1.0
      ],
      [
        73,
        0.956656346749226
      ],
      [
        74,
        0.9999404620147654
      ],
      [
        75,
        0.763157894736842
      ],
      [
        76,
        1.0
      ],
      [
        77,
        0.5
      ],
      [
        78,
        0.9999945874558878
      ],
      [
        79,
        1.0
      ],
      [
        80,
        0.9999404620147654
      ],
      [
        81,
        0.0
      ],
      [
        82,
        0.5
      ],
      [
        83,
        0.0
      ],
      [
        84,
        0.0
      ],
      [
        85,
        0.9837461300309598
      ],
      [
        86,
        0.8947368421052632
      ],
      [
        87,
        0.9984520123839009
      ],
      [
        88,
        0.956656346749226
      ],
      [
        89,
        0.9837461300309598
      ],
      [
        90,
        0.9945820433436533
      ],
      [
        91,
        0.0
      ],
      [
        92,
        1.0
      ],
      [
        93,
        0.5
      ],
      [
        94,
        0.0
      ],
      [
        95,
        0.9996427720885925
      ],
      [
        96,
        0.9996427720885925
      ],
      [
        97,
        1.0
      ],
      [
        98,
        1.0
      ],
      [
        99,
        0.0
      ],
      [
        100,
        0.5
      ],
      [
        101,
        0.9945820433436533
      ],
      [
        102,
        0.8947368421052632
      ],
      [
        103,
        1.0
      ],
      [
        104,
        0.956656346749226
      ],
      [
        105,
        0.5
      ],
      [
        106,
        0.0
      ],
      [
        107,
        1.0
      ],
      [
        108,
        0.0
      ],
      [
        109,
        0.0
      ],
      [
        110,
        0.5
      ],
      [
        111,
        0.8947368421052632
      ],
      [
        112,
        0.9837461300309598
      ],
      [
        113,
        0.9999404620147654
      ],
      [
        114,
        0.9999945874558878
      ],
      [
        115,
        0.763157894736842
      ],
      [
        116,
        1.0
      ],
      [
        117,
        0.9837461300309598
      ],
      [
        118,
        0.0
      ],
      [
        119,
        0.0
      ],
      [
        120,
        0.8947368421052632
      ],
      [
        121,
        0.9996427720885925
      ],
      [
        122,
        1.0
      ],
      [
        123,
        0.8947368421052632
      ],
      [
        124,
        0.956656346749226
      ],
      [
        125,
        0.763157894736842
      ],
      [
        126,
        0.0
      ],
      [
        127,
        0.5
      ],
      [
        128,
        0.956656346749226
      ],
      [
        129,
        0.0
      ],
      [
        130,
        0.0
      ],
      [
        131,
        0.0
      ],
      [
        132,
        0.0
      ],
      [
        133,
        0.0
      ],
      [
        134,
        0.0
      ],
      [
        135,
        0.0
      ],
      [
        136,
        1.0
      ],
      [
        137,
        0.0
      ],
      [
        138,
        0.763157894736842
      ],
      [
        139,
        0.0
      ],
      [
        140,
        0.0
      ],
      [
        141,
        0.0
      ],
      [
        142,
        0.956656346749226
      ],
      [
        143,
        0.8947368421052632
      ],
      [
        144,
        0.9984520123839009
      ],
      [
        145,
        0.0
      ],
      [
        146,
        0.0
      ],
      [
        147,
        0.9984520123839009
      ],
      [
        148,
        0.9945820433436533
      ],
      [
        149,
        0.9837461300309598
      ],
      [
        150,
        0.5
      ],
      [
        151,
        0.9984520123839009
      ],
      [
        152,
        1.0
      ],
      [
        153,
        0.8947368421052632
      ],
      [
        154,
        0.956656346749226
      ],
      [
        155,
        0.0
      ],
      [
        156,
        0.9945820433436533
      ],
      [
        157,
        0.9984520123839009
      ],
      [
        158,
        0.9837461300309598
      ],
      [
        159,
        0.763157894736842
      ],
      [
        160,
        0.0
      ],
      [
        161,
        0.763157894736842
      ],
      [
        162,
        1.0
      ],
      [
        163,
        0.0
      ]
    ]
  },
  "config": {
    "prefix": "",
    "do_sample": true,
    "temperature": 0.2,
    "top_k": 0,
    "top_p": 0.95,
    "n_samples": 20,
    "eos": "<|endoftext|>",
    "seed": 0,
    "model": "codegemma-7b",
    "modeltype": "causal",
    "peft_model": null,
    "revision": null,
    "use_auth_token": false,
    "trust_remote_code": false,
    "tasks": "humaneval",
    "instruction_tokens": null,
    "batch_size": 1,
    "max_length_generation": 512,
    "precision": "fp32",
    "load_in_8bit": false,
    "load_in_4bit": false,
    "left_padding": false,
    "limit": null,
    "limit_start": 0,
    "save_every_k_tasks": -1,
    "postprocess": true,
    "allow_code_execution": true,
    "generation_only": false,
    "load_generations_path": "3_codegemma-7b_generations_humaneval_codegemma-7b_humaneval.json",
    "load_data_path": null,
    "metric_output_path": "3_codegemma-7b-result.json",
    "save_generations": false,
    "load_generations_intermediate_paths": null,
    "save_generations_path": "generations.json",
    "save_references": false,
    "save_references_path": "references.json",
    "prompt": "prompt",
    "max_memory_per_gpu": null,
    "check_references": false
  }
}