import argparse import glob import json import os def get_args(): parser = argparse.ArgumentParser() parser.add_argument("--shard_glob", type=str, required=True) parser.add_argument("--output_path", type=str, required=True) return parser.parse_args() def main(): args = get_args() shard_paths = sorted(glob.glob(args.shard_glob)) if not shard_paths: raise FileNotFoundError(f"No shard files matched: {args.shard_glob}") shards = [json.load(open(path, "r")) for path in shard_paths] model_name_red = shards[0]["model_name_red"] model_name_blue = shards[0]["model_name_blue"] file_name_red = shards[0]["file_name_red"] file_name_blue = shards[0]["file_name_blue"] evaluations = [] for shard in shards: evaluations.extend(shard["evaluations"]) evaluations.sort(key=lambda row: row.get("global_index", 10**9)) win_h = tie_h = lose_h = nd_h = 0 win_s = tie_s = lose_s = nd_s = 0 for row in evaluations: rh, bh = row["red_score_helpfulness"], row["blue_score_helpfulness"] rs, bs = row["red_score_harmlessness"], row["blue_score_harmlessness"] if rh < 0 or bh < 0: nd_h += 1 else: win_h += rh > bh tie_h += rh == bh lose_h += rh < bh if rs < 0 or bs < 0: nd_s += 1 else: win_s += rs > bs tie_s += rs == bs lose_s += rs < bs denom_h = win_h + tie_h + lose_h denom_s = win_s + tie_s + lose_s result = { "file_name_red": file_name_red, "file_name_blue": file_name_blue, "model_name_red": model_name_red, "model_name_blue": model_name_blue, "win_helpfulness": win_h, "tie_helpfulness": tie_h, "lose_helpfulness": lose_h, "not_determined_helpfulness": nd_h, "win_rate_helpfulness": win_h / denom_h if denom_h else None, "win_halfTie_helpfulness": (win_h + 0.5 * tie_h) / denom_h if denom_h else None, "win_harmlessness": win_s, "tie_harmlessness": tie_s, "lose_harmlessness": lose_s, "not_determined_harmlessness": nd_s, "win_rate_harmlessness": win_s / denom_s if denom_s else None, "win_halfTie_harmlessness": (win_s + 0.5 * tie_s) / denom_s if denom_s else None, "evaluations": evaluations, "source_shards": shard_paths, } os.makedirs(os.path.dirname(args.output_path), exist_ok=True) with open(args.output_path, "w", encoding="utf-8") as f: json.dump(result, f, ensure_ascii=False, indent=2) if __name__ == "__main__": main()