File size: 4,030 Bytes
02c783d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 |
import json
import numpy as np
index = {
'call_candidate': None,
'exe_candidate': None,
'perf_candidates': [],
}
paths = [
'/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_medium_v3_mem_0.json',
'/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_medium_v3_mem_1.json',
'/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_medium_v3_mem_2.json',
'/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_medium_v3_mem_3.json',
'/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_high_v3_mem_0.json',
'/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_high_v3_mem_1.json',
'/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_high_v3_mem_2.json',
# '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8_mem_0.json',
# '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8_mem_1.json',
# '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8_mem_2.json',
# '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8_mem_3.json',
# '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8_mem_4.json',
# '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8_mem_5.json',
'/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_mem_0.json',
# '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_mem_1.json',
# '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_mem_2.json',
# '/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_mem_3.json',
'/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_code_mem_0.json',
'/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_code_mem_1.json',
'/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_code_mem_2.json',
'/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_code_mem_3.json',
'/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_code_mem_4.json',
'/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_high_v4_mem_0.json',
'/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_oss_120b_high_v4_mem_1.json',
'/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_30_mem_0.json',
'/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_30_mem_1.json',
'/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_30_mem_2.json',
'/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1107_q8r_30_mem_3.json',
'/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1114_q8r_code_mem_0.json',
'/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1114_q8r_8b_mem_0.json',
'/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1114_q8r_8b_mem_1.json',
'/wekafs/zihao/exp/agent/jianghui_triton/GEAK-agent/outputs/outputs/1114_q8r_8b_mem_2.json',
]
for path in paths:
result = {i:[] for i in index}
data = json.load(open(path))
for k, v in data.items():
for i in index:
result[i].append(v[i])
print(f'[path]:')
print(path)
print('\n')
for k, v in index.items():
print(f'[{k}]:')
print(np.array([str(i) != str(v) for i in result[k]]).mean())
if k == 'perf_candidates':
acc = []
for i in result[k]:
if len(i) == 0:
acc.append(1)
else:
acc.append(max(1, i[-1][1]))
print(np.array(acc).mean())
print('\n')
print('-'*100)
|