Update winrate.py

winrate.py

@@ -136,7 +136,131 @@ def compare_win_rate(dir1, dir2):
         print("-" * 30)
     
     return win_rate_df
-
+def compare_category_scores(dir1, dir2):
+    """
+    比较两个目录中每个大类和子类的平均分数，并输出总结性比较
+    
+    参数:
+        dir1: 第一个目录路径
+        dir2: 第二个目录路径
+    
+    返回:
+        DataFrame: 包含所有比较结果的DataFrame
+    """
+    # 收集两个目录的分数数据
+    def collect_scores(directory):
+        scores = defaultdict(lambda: {'total': 0, 'count': 0})
+        sub_scores = defaultdict(lambda: {'total': 0, 'count': 0})
+        
+        for category in os.listdir(directory):
+            category_path = os.path.join(directory, category)
+            if not os.path.isdir(category_path):
+                continue
+                
+            for subcategory in os.listdir(category_path):
+                subcategory_path = os.path.join(category_path, subcategory)
+                if not os.path.isdir(subcategory_path):
+                    continue
+                    
+                report_path = os.path.join(subcategory_path, 'report', 'full_image_comet.csv')
+                if os.path.exists(report_path):
+                    try:
+                        df = pd.read_csv(report_path)
+                        avg_score = df['avg_comet_sentence'].mean()
+                        # 更新大类统计
+                        scores[category]['total'] += avg_score
+                        scores[category]['count'] += 1
+                        # 更新子类统计
+                        sub_scores[(category, subcategory)]['total'] = avg_score
+                        sub_scores[(category, subcategory)]['count'] = 1
+                    except Exception as e:
+                        print(f"Error processing {report_path}: {e}")
+        
+        # 计算平均值
+        for category in scores:
+            if scores[category]['count'] > 0:
+                scores[category]['avg'] = scores[category]['total'] / scores[category]['count']
+        
+        for key in sub_scores:
+            if sub_scores[key]['count'] > 0:
+                sub_scores[key]['avg'] = sub_scores[key]['total'] / sub_scores[key]['count']
+        
+        return scores, sub_scores
+    
+    # 收集两个目录的数据
+    scores1, sub_scores1 = collect_scores(dir1)
+    scores2, sub_scores2 = collect_scores(dir2)
+    
+    # 准备结果数据
+    results = []
+    
+    # 比较大类
+    all_categories = set(scores1.keys()).union(set(scores2.keys()))
+    category_comparison = []
+    
+    for category in all_categories:
+        avg1 = scores1.get(category, {}).get('avg', 0)
+        avg2 = scores2.get(category, {}).get('avg', 0)
+        comparison = "dir1 > dir2" if avg1 > avg2 else ("dir1 < dir2" if avg1 < avg2 else "dir1 == dir2")
+        
+        category_comparison.append({
+            'category': category,
+            'type': 'category',
+            'dir1_avg': round(avg1, 3),
+            'dir2_avg': round(avg2, 3),
+            'comparison': comparison
+        })
+    
+    # 比较子类
+    all_subcategories = set(sub_scores1.keys()).union(set(sub_scores2.keys()))
+    subcategory_comparison = []
+    
+    for (category, subcategory) in all_subcategories:
+        avg1 = sub_scores1.get((category, subcategory), {}).get('avg', 0)
+        avg2 = sub_scores2.get((category, subcategory), {}).get('avg', 0)
+        comparison = "dir1 > dir2" if avg1 > avg2 else ("dir1 < dir2" if avg1 < avg2 else "dir1 == dir2")
+        
+        subcategory_comparison.append({
+            'category': category,
+            'type': 'subcategory',
+            'subcategory': subcategory,
+            'dir1_avg': round(avg1, 3),
+            'dir2_avg': round(avg2, 3),
+            'comparison': comparison
+        })
+    
+    # 合并结果
+    results = category_comparison + subcategory_comparison
+    
+    # 创建DataFrame
+    comparison_df = pd.DataFrame(results)
+    
+    # 总结性比较
+    total_categories = len(all_categories)
+    dir1_win_categories = sum(1 for c in category_comparison if c['comparison'] == 'dir1 > dir2')
+    dir2_win_categories = sum(1 for c in category_comparison if c['comparison'] == 'dir1 < dir2')
+    
+    total_subcategories = len(all_subcategories)
+    dir1_win_subcategories = sum(1 for c in subcategory_comparison if c['comparison'] == 'dir1 > dir2')
+    dir2_win_subcategories = sum(1 for c in subcategory_comparison if c['comparison'] == 'dir1 < dir2')
+    
+    # 打印总结
+    print("\n=== 总结性比较 ===")
+    print(f"大类总数: {total_categories}")
+    print(f"dir1获胜的大类数: {dir1_win_categories} ({dir1_win_categories/total_categories:.1%})")
+    print(f"dir2获胜的大类数: {dir2_win_categories} ({dir2_win_categories/total_categories:.1%})")
+    print(f"平局的大类数: {total_categories - dir1_win_categories - dir2_win_categories}")
+    
+    print(f"\n子类总数: {total_subcategories}")
+    print(f"dir1获胜的子类数: {dir1_win_subcategories} ({dir1_win_subcategories/total_subcategories:.1%})")
+    print(f"dir2获胜的子类数: {dir2_win_subcategories} ({dir2_win_subcategories/total_subcategories:.1%})")
+    print(f"平局的子类数: {total_subcategories - dir1_win_subcategories - dir2_win_subcategories}")
+    
+    # 保存结果
+    comparison_df.to_csv('category_score_comparison.csv', index=False)
+    print("\n比较结果已保存到 category_score_comparison.csv")
+    
+    return comparison_df
 if __name__ == "__main__":
     print("Processing task 1...")
     avg_scores = calculate_avg_comet('organized_data_1')
@@ -144,4 +268,4 @@ if __name__ == "__main__":
     
     print("\nProcessing task 2...")
     win_rate_df = compare_win_rate('organized_data_1', 'organized_data_2')
-    print("\nWin rate comparison saved to win_rate_comparison.csv")
+    print("\nWin rate comparison saved to win_rate_comparison.csv")