notebooks/print_results.py

import os
import sys
import json

from cliport import agents
from cliport import tasks
import argparse
import datetime
import matplotlib as mpl

mpl.use("Agg")
import argparse
import os
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib
import IPython
import numpy as np
font = {
    "size": 22,
}
matplotlib.rc("font", **font)
sns.set_context("paper", font_scale=2.0)


def mkdir_if_missing(dst_dir):
    if not os.path.exists(dst_dir):
        os.makedirs(dst_dir)


def save_figure(name, title=""):
    print(f"output/output_figures/{name}.png")
    if len(title) > 0:
        plt.title(title)
    plt.tight_layout()
    mkdir_if_missing(f"output/output_figures/{name}")
    plt.savefig(f"output/output_figures/{name}/output.png")
    plt.clf()


def print_and_write(file_handle, text):
    print(text)
    if file_handle is not None:
        file_handle.write(text + "\n")
    return text

parser = argparse.ArgumentParser()

# federated arguments (Notation for the arguments followed from paper)
parser.add_argument(
    "--results", "-r", type=str, default="exps/exps-singletask"
)
parser.add_argument(
    "--single", "-s", action="store_true", default=False
)
args = parser.parse_args()

root_folder = os.environ['GENSIM_ROOT']
exp_folder = os.path.join(root_folder, args.results) # replace 'cliport_quickstart' with your exps folder


mkdir_if_missing('output/output_figures')
mkdir_if_missing('output/cliport_output')
mkdir_if_missing('output/output_stat')



output_stat_file = os.path.join('output/', 'cliport_output/', 'cliport-training.txt')
mkdir_if_missing('output/cliport_output/')
file_handle = open(output_stat_file, 'a+')

tasks_list = list(tasks.names.keys())
agents_list = list(agents.names.keys())
demos_list = [1, 5, 10, 20, 30, 50, 100, 200, 1000] # 100,

results = {}
for t in tasks_list:
    for a in agents_list:
        for d in demos_list:
            task_folder = f'{t}-{a}-n{d}-train'
            task_folder_path = os.path.join(exp_folder, task_folder, 'checkpoints')

            if os.path.exists(task_folder_path):
                print(f"train {task_folder_path}")

                jsons = [f for f in os.listdir(task_folder_path) if '.json' in f]
                for j in jsons:
                    model_type = 'multi' if 'multi' in j else 'single'
                    eval_type = 'val' if 'val' in j else 'test'
                    
                    with open(os.path.join(task_folder_path, j)) as f:
                        res = json.load(f)
                    
                    results[f'{t}-{a}-n{d}-{model_type}-{eval_type}'] = res

dt_string = datetime.datetime.now().strftime("%d_%m_%Y_%H:%M:%S")
print_and_write(file_handle, f"==========================={dt_string}=========================\n")
print_and_write(file_handle, f'Experiments folder: {exp_folder}\n')

data = {'task': [], 'success': []}

for eval_type in ['val', 'test']:
    print_and_write(file_handle, f'----- {eval_type.upper()} -----\n')
    for t in tasks_list:
        for a in agents_list:
            for d in demos_list:
                for model_type in ['single', 'multi']:
                    eval_key = f'{t}-{a}-n{d}-{model_type}-{eval_type}'
                    
                    if eval_key in results:    
                        print_and_write(file_handle, f'{eval_key} {t} | Train Demos: {d}')
                        res = results[eval_key]
                        best_score, best_ckpt = max(zip([v['mean_reward'] for v in list(res.values())], res.keys()))
                        # TODO: test that this works for full results folder
                        
                        print_and_write(file_handle, f'\t{best_score*100:1.1f} : {a} | {model_type}\n')
                        data['task'].append(t)
                        data['success'].append(best_score)

data['task'].append("Average")
data['success'].append(np.mean(data["success"]))


# make figure as well for sinle expeirment results
dfs = []
suffix = ""
run_num = 0
df = pd.DataFrame.from_dict(data)
title = args.results + "_res"

# rewards
fig, ax = plt.subplots(figsize=(16, 8))
sns_plot = sns.barplot(
    data=df, x="task", y="success", errorbar=("sd", 1), palette="deep"
)

# label texts
for container in ax.containers:
    ax.bar_label(container, label_type="center", fontsize="x-large", fmt="%.2f")
# ax.set_xticklabels(ax.get_xticklabels(), rotation=90, ha="right")
ax.set_xticklabels(['\n'.join(str(xlabel.get_text()).split("-")) for xlabel in ax.get_xticklabels()])

# save plot
save_figure(f"{title}", title)