File size: 2,391 Bytes
987ed1b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 | if __name__ == "__main__":
import sys
import os
import pathlib
ROOT_DIR = str(pathlib.Path(__file__).parent.parent.parent)
sys.path.append(ROOT_DIR)
import os
import click
import collections
import numpy as np
from tqdm import tqdm
import json
@click.command()
@click.option(
'--reference', '-r', required=True,
help='Reference metrics_raw.json from demonstration dataset.'
)
@click.option(
'--input', '-i', required=True,
help='Data search path'
)
def main(reference, input):
# compute the min last metric for demo metrics
demo_metrics = json.load(open(reference, 'r'))
demo_min_metrics = collections.defaultdict(lambda:float('inf'))
for episode_idx, metrics in demo_metrics.items():
for key, value in metrics.items():
last_value = value[-1]
demo_min_metrics[key] = min(demo_min_metrics[key], last_value)
print(demo_min_metrics)
# find all metric
name = 'metrics_raw.json'
search_dir = pathlib.Path(input)
success_rate_map = dict()
for json_path in search_dir.glob('**/'+name):
rel_path = json_path.relative_to(search_dir)
rel_name = str(rel_path.parent)
this_metrics = json.load(json_path.open('r'))
metric_success_idxs = collections.defaultdict(list)
metric_failure_idxs = collections.defaultdict(list)
for episode_idx, metrics in this_metrics.items():
for key, value in metrics.items():
last_value = value[-1]
# print(episode_idx, key, last_value)
demo_min = demo_min_metrics[key]
if last_value >= demo_min:
# success
metric_success_idxs[key].append(episode_idx)
else:
metric_failure_idxs[key].append(episode_idx)
# in case of no success
_ = metric_success_idxs[key]
_ = metric_failure_idxs[key]
metric_success_rate = dict()
n_episodes = len(this_metrics)
for key, value in metric_success_idxs.items():
metric_success_rate[key] = len(value) / n_episodes
# metric_success_rate['failured_idxs'] = metric_failure_idxs
success_rate_map[rel_name] = metric_success_rate
text = json.dumps(success_rate_map, indent=2)
print(text)
if __name__ == '__main__':
main()
|