# TODO: Implement Chamfer for 3D tasks # Input modules import os import sys import subprocess import argparse import time import json from PIL import Image from utils import photometric_loss, img2img_clip_similarity, blender_step, clip_similarity from tqdm import tqdm task_instance_count_dict = { 'geometry': 45, 'material': 40, 'blendshape': 75, 'placement': 40, 'lighting': 40 } if __name__=='__main__': parser = argparse.ArgumentParser(description='Image-based program edits') parser.add_argument('--inference_metadata_saved_path', type=str, help="Path to the inference metadata in json format (paths of proposal edit scripts, winner information, etc.)" ) parser.add_argument('--eval_render_save_dir', type=str, default=None, help="The directory that all evaluation renders will be saved to.." ) parser.add_argument('--infinigen_installation_path', type=str, default=f"{os.path.abspath('infinigen/blender/blender')}", help="The installation path of blender executable file. It's `infinigen/blender/blender` by default." ) # parse, save, and validate the args args = parser.parse_args() inference_metadata_saved_path = args.inference_metadata_saved_path eval_render_save_dir = args.eval_render_save_dir infinigen_installation_path = args.infinigen_installation_path blender_render_script_path = "bench_data/all_render_script.py" if not os.path.isfile(inference_metadata_saved_path): raise ValueError(f'Invalid input for --inference_metadata_saved_path: {inference_metadata_saved_path}.') # Load in the data from pipeline inference with open(inference_metadata_saved_path, 'r') as file: inference_metadata = json.load(file) # Derive name for eval_render_save_dir if not eval_render_save_dir: eval_render_save_dir = f"eval_renders/{inference_metadata['output_dir_name']}" os.makedirs(eval_render_save_dir, exist_ok=True) tasks = inference_metadata.keys() # Create eval renders for all instances scores_across_tasks = {} intermediates = {} for task in tasks: if task not in task_instance_count_dict.keys(): continue # Iterate through each instance scores_across_instances = {'best_n_clip':[], 'selected_n_clip':[], 'best_pl':[], 'selected_pl':[]} for task_instance, instance_info in inference_metadata[task].items(): task_instance_dir = os.path.join(eval_render_save_dir, task_instance) os.makedirs(task_instance_dir, exist_ok=True) # Store local scores: score for each executable render task_instance_scores = {} try: # Iterate through each proposal_renders_path blender_file_path = instance_info['blender_file_path'] start_file_path = instance_info['start_script_path'] goal_file_path = instance_info['goal_script_path'] except: continue executable_proposal_names = [] for proposal_path in (instance_info['proposal_edits_paths'] + [start_file_path, goal_file_path]): # Render the images for that proposal_renders_path proposal_name = os.path.basename(proposal_path).split('.')[0] # Extract the name of py file, without suffix proposal_renders_dir = os.path.join(task_instance_dir, proposal_name) # Render images. "executable" checks whether the proposal is executable in Blender-Python API. if not os.path.exists(proposal_renders_dir) or not os.listdir(proposal_renders_dir): try: executable = blender_step(infinigen_installation_path, blender_file_path, blender_render_script_path, proposal_path, proposal_renders_dir, merge_all_renders=False, replace_if_overlap=True) except: continue if executable: executable_proposal_names.append((proposal_renders_dir,proposal_name)) else: executable_proposal_names.append((proposal_renders_dir,proposal_name)) # Loop through each executable proposal to compute their scores for proposal_renders_dir, proposal_name in tqdm(executable_proposal_names): if proposal_name == 'goal': continue task_instance_scores[proposal_name] = {} n_clip_views = [] pl_views = [] for render_name in os.listdir(proposal_renders_dir): task_instance_scores[proposal_name][render_name] = {} # Get path for render try: proposal_render = Image.open(os.path.join(proposal_renders_dir, render_name)) gt_render = Image.open(os.path.join(task_instance_dir, 'goal', render_name)) except: continue # Compute n_clip and pl n_clip = float(1 - clip_similarity(proposal_render, gt_render)) pl = float(photometric_loss(proposal_render, gt_render)) # Aggregate scores across all views for a proposal edit to compute average n_clip_views.append(n_clip) pl_views.append(pl) # Record scores for this render task_instance_scores[proposal_name][render_name]['n_clip'] = n_clip task_instance_scores[proposal_name][render_name]['pl'] = pl # Compute average n_clip for this proposal if n_clip_views: average_n_clip_views = sum(n_clip_views) / len(n_clip_views) # Compute average pl for this task instance if pl_views: average_pl_views = sum(pl_views) / len(pl_views) # Record average scores for a proposal task_instance_scores[proposal_name]['avg_n_clip'] = average_n_clip_views task_instance_scores[proposal_name]['avg_pl'] = average_pl_views # Save the local scores to the task_instance dir task_instance_scores_path = os.path.join(task_instance_dir, 'scores.json') with open(task_instance_scores_path, 'w') as file: json.dump(task_instance_scores, file, indent=4) # Extract best scores and record them best_n_clip_proposal_name = min(task_instance_scores, key=lambda proposal_name: task_instance_scores[proposal_name]['avg_n_clip']) best_pl_proposal_name = min(task_instance_scores, key=lambda proposal_name: task_instance_scores[proposal_name]['avg_pl']) best_n_clip = task_instance_scores[best_n_clip_proposal_name]['avg_n_clip'] best_pl = task_instance_scores[best_pl_proposal_name]['avg_pl'] task_instance_scores['best_n_clip'] = (best_n_clip_proposal_name, best_n_clip) task_instance_scores['best_pl'] = (best_pl_proposal_name, best_pl) # Register this instance to the scores across this task scores_across_instances['best_n_clip'].append(best_n_clip) scores_across_instances['best_pl'].append(best_pl) # Handle selected edit if applicable selected_proposal_path = instance_info['selected_edit_path'] if selected_proposal_path: selected_proposal_name = os.path.basename(selected_proposal_path).split('.')[0] try: selectd_n_clip = task_instance_scores[selected_proposal_name]['avg_n_clip'] selected_pl = task_instance_scores[selected_proposal_name]['avg_pl'] task_instance_scores['selected_scores'] = (selected_proposal_name, {'avg_n_clip':selectd_n_clip, 'avg_pl':selected_pl}) except: continue # Register this instance to the scores across this task scores_across_instances["selected_n_clip"].append(selectd_n_clip) scores_across_instances["selected_pl"].append(selected_pl) # Save the local scores to the task_instance dir task_instance_scores_path = os.path.join(task_instance_dir, 'scores.json') with open(task_instance_scores_path, 'w') as file: json.dump(task_instance_scores, file, indent=4) scores_across_instances_path = os.path.join(eval_render_save_dir, f'{task}_scores.json',) with open(scores_across_instances_path, 'w') as file: json.dump(scores_across_instances, file, indent=4) # If the model cannot provide any edit for more than 75% if len(scores_across_instances['best_n_clip']) < (len(inference_metadata[task]) * 0.25) : scores_across_tasks[task] = {} # If VLM system doesn't support selection elif not scores_across_instances["selected_n_clip"]: scores_across_tasks[task] = { 'best_n_clip': sum(scores_across_instances['best_n_clip']) / len(scores_across_instances['best_n_clip']), 'best_pl': sum(scores_across_instances['best_pl']) / len(scores_across_instances['best_pl']), } else: scores_across_tasks[task] = { 'best_n_clip': sum(scores_across_instances['best_n_clip']) / len(scores_across_instances['best_n_clip']), 'best_pl': sum(scores_across_instances['best_pl']) / len(scores_across_instances['best_pl']), 'selected_n_clip': sum(scores_across_instances['selected_n_clip']) / len(scores_across_instances['selected_n_clip']), 'selected_pl': sum(scores_across_instances['selected_pl']) / len(scores_across_instances['selected_pl']), } intermediates[task] = scores_across_instances scores_across_tasks_path = os.path.join(eval_render_save_dir, 'overall_scores.json',) with open(scores_across_tasks_path, 'w') as file: json.dump(scores_across_tasks, file, indent=4) scores_across_instances_path = os.path.join(eval_render_save_dir, 'intermediate_scores.json',) with open(scores_across_instances_path, 'w') as file: json.dump(intermediates, file, indent=4) # Compute Chamfer Distance for 3D-related tasks