GEAK-agent/get_result.py

import json
import glob
import numpy as np


index = {
        'call_candidate': None,
        'exe_candidate': None,
        'perf_candidates': [],
        }

paths = [
    # '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_medium_v3_mem_0.json',
    # '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_medium_v3_mem_1.json',
    # '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_medium_v3_mem_2.json',
    # '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_medium_v3_mem_3.json',
    # '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_high_v3_mem_0.json',
    # '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_high_v3_mem_1.json',
    # '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_high_v3_mem_2.json',


    # '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1114_q8r_8b_mem_*.json',

    # '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1114_q8r_code_mem_*.json',

    # '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1114_q8r_30_mem_*.json',


    # '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1116_oss_20b_3_mem_*.json',

    # '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1116_oss_20b_m_mem_*.json',

    '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1114_oss_high_mem_*.json',
    ]

for tmp_path in paths:
    tmp_path = sorted(glob.glob(tmp_path))
    for path in tmp_path:
        result = {i:[] for i in index}
        data = json.load(open(path))
        for k, v in data.items():
            for i in index:
                result[i].append(v[i])
        
        print(f'[path]:')
        print(path)
        print('\n')
        for k, v in index.items():
            print(f'[{k}]:')
            print(np.array([str(i) != str(v) for i in result[k]]).mean())
            if k == 'perf_candidates':
                acc = []
                for i in result[k]:
                    if len(i) == 0:
                        acc.append(1)
                    else:
                        acc.append(max(1, i[-1][1]))
                print(np.array(acc).mean())

        print('\n')
        print('-'*100)