{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "a24d02a2", "metadata": {}, "outputs": [], "source": [ "import json\n", "import pandas as pd\n", "import numpy as np\n", "from pathlib import Path\n", "from datetime import datetime\n", "import warnings\n", "warnings.filterwarnings('ignore')\n", "\n", "# ============================================================================\n", "# SECTION 1: Load and Parse Results from GPU Tuning Runs\n", "# ==========================-==================================================\n", "print(\"=\" * 80)\n", "print(\"LOADING TUNING RESULTS FROM GPU RUNS\")\n", "print(\"=\" * 80)\n", "\n", "results_dir = Path(\"RESULTS_TURNING/run_2\")\n", "all_experiments = []\n", "baseline_metrics = None\n", "\n", "# Collect results from all GPU runs\n", "for gpu_id in range(8):\n", " gpu_dir = results_dir / f\"gpu_{gpu_id}\"\n", " results_file = gpu_dir / \"tuning_results.json\"\n", " \n", " if results_file.exists():\n", " with open(results_file, 'r') as f:\n", " data = json.load(f)\n", " \n", " # Extract baseline (same across all GPUs)\n", " if baseline_metrics is None and \"baseline\" in data:\n", " baseline_metrics = data[\"baseline\"][\"metrics\"]\n", " print(f\"\\nšŸ“Š Baseline Metrics (cfg_scale=5.0):\")\n", " for metric, value in baseline_metrics.items():\n", " print(f\" {metric:15s}: {value:.6f}\")\n", " \n", " # Collect all experiments\n", " if \"experiments\" in data:\n", " all_experiments.extend(data[\"experiments\"])\n", " print(f\"āœ“ GPU {gpu_id}: {len(data['experiments'])} results loaded\")\n", "\n", "print(f\"\\nāœ“ Total experiments loaded: {len(all_experiments)}\")\n", "\n", "# ============================================================================\n", "# SECTION 2: Filter Top Configs with Improvements Across All Metrics\n", "# ============================================================================\n", "print(\"\\n\" + \"=\" * 80)\n", "print(\"FILTERING CONFIGURATIONS WITH IMPROVEMENTS IN ALL METRICS\")\n", "print(\"=\" * 80)\n", "\n", "# Define improvement metrics to track (using ImageReward instead of Reward)\n", "improvement_metrics = [\n", " \"aesthetic_improvement\", \n", " \"imagereward_improvement\", \n", " \"clip_improvement\", \n", " \"pickscore_improvement\", \n", " \"hpsv2_improvement\"\n", " ]\n", "\n", "# Filter experiments with improvements in ALL metrics\n", "top_configs = []\n", "\n", "for exp in all_experiments:\n", " if \"improvements\" not in exp or \"config\" not in exp or \"metrics\" not in exp:\n", " continue\n", " \n", " improvements = exp[\"improvements\"]\n", " config = exp[\"config\"]\n", " metrics = exp[\"metrics\"]\n", " \n", " # Check if ALL improvements are positive (>0)\n", " all_positive = all(improvements.get(metric, -1) > 0 for metric in improvement_metrics)\n", " \n", " if all_positive:\n", " # Calculate aggregate improvement score\n", " avg_improvement = np.mean([improvements.get(metric, 0) for metric in improvement_metrics])\n", " \n", " top_configs.append({\n", " \"config\": config,\n", " \"metrics\": metrics,\n", " \"improvements\": improvements,\n", " \"avg_improvement\": avg_improvement\n", " })\n", "\n", "print(f\"āœ“ Found {len(top_configs)} configurations with improvements in ALL metrics\")\n", "\n", "# Sort by average improvement\n", "top_configs.sort(key=lambda x: x[\"avg_improvement\"], reverse=True)\n", "\n", "# Get top 10\n", "top_10 = top_configs[:10]\n", "print(f\"āœ“ Extracted top 10 best performing configurations\")\n", "\n", "# ============================================================================\n", "# SECTION 3: Create Comprehensive Results Table\n", "# ============================================================================\n", "print(\"\\n\" + \"=\" * 80)\n", "print(\"CREATING COMPREHENSIVE RESULTS TABLE\")\n", "print(\"=\" * 80)\n", "\n", "# Build detailed table data\n", "table_data = []\n", "\n", "for rank, result in enumerate(top_10, 1):\n", " cfg = result[\"config\"]\n", " metrics = result[\"metrics\"]\n", " improvements = result[\"improvements\"]\n", " \n", " row = {\n", " \"Rank\": rank,\n", " \"CFG Scale\": cfg.get(\"cfg_scale\", \"N/A\"),\n", " \"Grad Config\": cfg.get(\"grad_config\", \"N/A\"),\n", " \"Steps\": cfg.get(\"num_grad_steps\", \"N/A\"),\n", " \"LR\": cfg.get(\"grad_step_size\", \"N/A\"),\n", " \"Momentum\": cfg.get(\"momentum\", \"N/A\"),\n", " \"ImageReward\": f\"{metrics.get('imagereward', 0):.6f}\",\n", " \"ImageReward ā†‘\": f\"{improvements.get('imagereward_improvement', 0):+.2f}%\",\n", " \"CLIP\": f\"{metrics.get('clip', 0):.4f}\",\n", " \"CLIP ā†‘\": f\"{improvements.get('clip_improvement', 0):+.2f}%\",\n", " \"Aesthetic\": f\"{metrics.get('aesthetic', 0):.4f}\",\n", " \"Aesthetic ā†‘\": f\"{improvements.get('aesthetic_improvement', 0):+.2f}%\",\n", " \"PickScore\": f\"{metrics.get('pickscore', 0):.4f}\",\n", " \"PickScore ā†‘\": f\"{improvements.get('pickscore_improvement', 0):+.2f}%\",\n", " \"HPSv2\": f\"{metrics.get('hpsv2', 0):.4f}\",\n", " \"HPSv2 ā†‘\": f\"{improvements.get('hpsv2_improvement', 0):+.2f}%\",\n", " \"Avg Improvement\": f\"{result['avg_improvement']:+.2f}%\",\n", " }\n", " \n", " table_data.append(row)\n", "\n", "df_top_10 = pd.DataFrame(table_data)\n", "\n", "print(\"\\nšŸ“‹ TOP 10 CONFIGURATIONS WITH IMPROVEMENTS IN ALL METRICS:\")\n", "print(\"=\" * 180)\n", "print(df_top_10.to_string(index=False))\n", "print(\"=\" * 180)\n", "\n", "# ============================================================================\n", "# SECTION 4: Visualize and Summary Statistics\n", "# ============================================================================\n", "print(\"\\n\" + \"=\" * 80)\n", "print(\"SUMMARY STATISTICS\")\n", "print(\"=\" * 80)\n", "\n", "# Extract numeric improvement values for analysis\n", "improvement_summary = []\n", "for result in top_10:\n", " improvements = result[\"improvements\"]\n", " for metric in [\"imagereward_improvement\", \"clip_improvement\", \"aesthetic_improvement\", \n", " \"pickscore_improvement\", \"hpsv2_improvement\"]:\n", " metric_name = metric.replace(\"_improvement\", \"\").upper()\n", " improvement_summary.append({\n", " \"Metric\": metric_name,\n", " \"Improvement %\": improvements.get(metric, 0)\n", " })\n", "\n", "df_summary = pd.DataFrame(improvement_summary)\n", "\n", "print(\"\\nšŸ“Š Average Improvements by Metric (Top 10):\")\n", "metric_stats = df_summary.groupby(\"Metric\")[\"Improvement %\"].agg([\"mean\", \"std\", \"min\", \"max\"])\n", "print(metric_stats.round(2))\n", "\n", "print(\"\\nšŸ“ˆ Best Configuration Details:\")\n", "best = top_10[0]\n", "best_cfg = best[\"config\"]\n", "best_metrics = best[\"metrics\"]\n", "best_improvements = best[\"improvements\"]\n", "\n", "print(f\"\\nāœ“ RANK #1 - Best Performing Configuration:\")\n", "print(f\" Configuration:\")\n", "print(f\" ā€¢ CFG Scale: {best_cfg.get('cfg_scale')}\")\n", "print(f\" ā€¢ Gradient Config: {best_cfg.get('grad_config')}\")\n", "print(f\" ā€¢ Gradient Steps: {best_cfg.get('num_grad_steps')}\")\n", "print(f\" ā€¢ Step Size: {best_cfg.get('grad_step_size')}\")\n", "print(f\" ā€¢ Momentum: {best_cfg.get('momentum')}\")\n", "print(f\"\\n Metrics:\")\n", "for metric in [\"imagereward\", \"clip\", \"aesthetic\", \"pickscore\", \"hpsv2\"]:\n", " baseline_val = baseline_metrics.get(metric, 0)\n", " current_val = best_metrics.get(metric, 0)\n", " improvement = best_improvements.get(f\"{metric}_improvement\", 0)\n", " print(f\" ā€¢ {metric:12s}: {current_val:8.6f} (baseline: {baseline_val:8.6f}) ā†‘ {improvement:+6.2f}%\")\n", "\n", "print(\"\\n\" + \"=\" * 80)\n", "print(\"āœ“ ANALYSIS COMPLETE - TOP 10 CONFIGURATIONS IDENTIFIED\")\n", "print(\"=\" * 80)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.18" } }, "nbformat": 4, "nbformat_minor": 5 }