File size: 2,381 Bytes
02c783d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 | import json
import glob
import numpy as np
index = {
'call_candidate': None,
'exe_candidate': None,
'perf_candidates': [],
}
paths = [
# '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_medium_v3_mem_0.json',
# '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_medium_v3_mem_1.json',
# '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_medium_v3_mem_2.json',
# '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_medium_v3_mem_3.json',
# '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_high_v3_mem_0.json',
# '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_high_v3_mem_1.json',
# '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_high_v3_mem_2.json',
# '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1114_q8r_8b_mem_*.json',
# '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1114_q8r_code_mem_*.json',
# '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1114_q8r_30_mem_*.json',
# '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1116_oss_20b_3_mem_*.json',
# '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1116_oss_20b_m_mem_*.json',
'/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1114_oss_high_mem_*.json',
]
for tmp_path in paths:
tmp_path = sorted(glob.glob(tmp_path))
for path in tmp_path:
result = {i:[] for i in index}
data = json.load(open(path))
for k, v in data.items():
for i in index:
result[i].append(v[i])
print(f'[path]:')
print(path)
print('\n')
for k, v in index.items():
print(f'[{k}]:')
print(np.array([str(i) != str(v) for i in result[k]]).mean())
if k == 'perf_candidates':
acc = []
for i in result[k]:
if len(i) == 0:
acc.append(1)
else:
acc.append(max(1, i[-1][1]))
print(np.array(acc).mean())
print('\n')
print('-'*100)
|