File size: 3,035 Bytes
9d5b280 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 |
import os
import json
from glob import glob
import pandas as pd
from collections import defaultdict
def process_json_files(base_path):
results = []
for folder_path in glob(os.path.join(base_path, "*/")):
model_name = os.path.basename(os.path.dirname(folder_path))
json_files = glob(os.path.join(folder_path, "results*.json"))
if json_files:
with open(json_files[0], 'r') as f:
data = json.load(f)
if 'results' in data:
for dataset, values in data['results'].items():
if 'acc,none' in values:
results.append({
'model': model_name,
'dataset': dataset,
'accuracy': round(values['acc,none']*100, 2)
})
return results
def process_results(results):
model_data = defaultdict(list)
for item in results:
if item['dataset'] != 'multimedqa':
model_data[item['model']].append(item)
new_results = []
for model, items in model_data.items():
total_acc = sum(item['accuracy'] for item in items)
avg_acc = round(total_acc / len(items), 2)
new_results.extend(items)
new_results.append({
'model': model,
'dataset': 'average',
'accuracy': avg_acc
})
return new_results
def format_results(results):
df = pd.DataFrame(results)
pivot_df = df.pivot(index='model', columns='dataset', values='accuracy')
pivot_df.reset_index(inplace=True)
pivot_df.rename(columns={'model': 'file_name'}, inplace=True)
# Reorder columns to have 'average' first after 'file_name'
columns = ['file_name', 'average'] + [col for col in pivot_df.columns if col not in ['file_name', 'average']]
pivot_df = pivot_df[columns]
return pivot_df
def create_summary_text(df):
# Create header
header = "file_name\taverage\tmedmcqa\tmedqa_4options\tmmlu_anatomy\tmmlu_clinical_knowledge\tmmlu_college_biology\tmmlu_college_medicine\tmmlu_medical_genetics\tmmlu_professional_medicine\tpubmedqa"
# Create data lines
data_lines = []
for _, row in df.iterrows():
model_name = row['file_name']
accuracies = [f"{row[col]:.2f}" if pd.notna(row[col]) else "" for col in df.columns if col != 'file_name']
data_line = f"{model_name}_\t" + "\t".join(accuracies)
data_lines.append(data_line)
# Combine header and data
return header + "\n" + "\n".join(data_lines)
# Main execution
results = process_json_files('.')
processed_results = process_results(results)
pivot_df = format_results(processed_results)
# Create summary text
summary_text = create_summary_text(pivot_df)
# Write summary to file
with open('results_summary.txt', 'w') as f:
f.write(summary_text)
print("Summary has been written to results_summary.txt")
# Optionally, still save the CSV if needed
pivot_df.to_csv('llama31_finalresult.csv', index=False) |