Spaces:
Sleeping
Sleeping
File size: 5,433 Bytes
329d553 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 | """
Test that GUI now uses Method 2 (correlate then average) for category averages
This should match the bar chart numbers!
"""
import sys
import pandas as pd
from data.data_loader import DataLoader
from visualization.plot_generator import PlotGenerator
print("="*80)
print("TESTING GUI METHOD 2 IMPLEMENTATION")
print("="*80)
# Load data
print("\nLoading data...")
loader = DataLoader()
loader.load_csv('data/Final_similarity_matrix_standardized.csv')
plot_gen = PlotGenerator(loader)
print(f"Loaded {len(loader.data)} image pairs")
print(f"Found {len([m for m in loader.model_categories['captions_neural']])} language models")
# Get brain measure columns
data = loader.data
# Early visual average
early_brain_avg = data['hierarchy_early_visual_avg']
late_brain_avg = data['hierarchy_late_semantic_avg']
print("\n" + "="*80)
print("TESTING: Neural Language (Captions) Category")
print("="*80)
# Test Method 2 computation directly
print("\nComputing Method 2 correlations directly...")
corr_early_method2 = plot_gen.compute_category_correlation_method2('captions_neural', early_brain_avg)
corr_late_method2 = plot_gen.compute_category_correlation_method2('captions_neural', late_brain_avg)
print(f"\nDirect Method 2 Results:")
print(f" Early Visual Average: r = {corr_early_method2:.4f}")
print(f" Late Semantic Average: r = {corr_late_method2:.4f}")
print(f" Difference: +{corr_late_method2 - corr_early_method2:.4f}")
print(f"\nExpected (from bar chart):")
print(f" Early: 0.2078")
print(f" Late: 0.3123")
print(f" Difference: +0.1046")
print(f"\nVerification:")
print(f" Early match? {abs(corr_early_method2 - 0.2078) < 0.001} (diff: {abs(corr_early_method2 - 0.2078):.6f})")
print(f" Late match? {abs(corr_late_method2 - 0.3123) < 0.001} (diff: {abs(corr_late_method2 - 0.3123):.6f})")
print("\n" + "="*80)
print("COMPARISON: Old Method 1 vs New Method 2")
print("="*80)
# Show Old Method 1 for comparison
# Compute averaged similarity column if it doesn't exist
if 'avg_captions_neural' not in data.columns:
models = [model[0] for model in loader.model_categories['captions_neural']]
available_models = [m for m in models if m in data.columns]
data['avg_captions_neural'] = data[available_models].mean(axis=1)
avg_captions_neural = data['avg_captions_neural']
corr_early_method1 = avg_captions_neural.corr(early_brain_avg)
corr_late_method1 = avg_captions_neural.corr(late_brain_avg)
print(f"\nOld Method 1 (average similarities, then correlate):")
print(f" Early: {corr_early_method1:.4f}")
print(f" Late: {corr_late_method1:.4f}")
print(f" Difference: +{corr_late_method1 - corr_early_method1:.4f}")
print(f"\nNew Method 2 (correlate, then average correlations):")
print(f" Early: {corr_early_method2:.4f}")
print(f" Late: {corr_late_method2:.4f}")
print(f" Difference: +{corr_late_method2 - corr_early_method2:.4f}")
print(f"\nChange:")
print(f" Early: {corr_early_method1:.4f} --> {corr_early_method2:.4f} (decreased by {corr_early_method1 - corr_early_method2:.4f})")
print(f" Late: {corr_late_method1:.4f} --> {corr_late_method2:.4f} (decreased by {corr_late_method1 - corr_late_method2:.4f})")
print("\n" + "="*80)
print("TESTING: Vision Models Category")
print("="*80)
corr_early_vision = plot_gen.compute_category_correlation_method2('vision', early_brain_avg)
corr_late_vision = plot_gen.compute_category_correlation_method2('vision', late_brain_avg)
print(f"\nVision Models Method 2:")
print(f" Early: {corr_early_vision:.4f}")
print(f" Late: {corr_late_vision:.4f}")
print(f" Difference: +{corr_late_vision - corr_early_vision:.4f}")
print(f"\nExpected (from bar chart):")
print(f" Early: 0.1996")
print(f" Late: 0.2045")
print(f" Difference: +0.0049")
print(f"\nVerification:")
print(f" Early match? {abs(corr_early_vision - 0.1996) < 0.001} (diff: {abs(corr_early_vision - 0.1996):.6f})")
print(f" Late match? {abs(corr_late_vision - 0.2045) < 0.001} (diff: {abs(corr_late_vision - 0.2045):.6f})")
print("\n" + "="*80)
print("SUMMARY")
print("="*80)
print(f"""
GUI will now show (when using category averages):
LANGUAGE MODELS (Neural Language - Captions):
Early Visual Average: r = {corr_early_method2:.3f} (was 0.324)
Late Semantic Average: r = {corr_late_method2:.3f} (was 0.476)
Difference: +{corr_late_method2 - corr_early_method2:.3f} (was +0.152)
VISION MODELS:
Early Visual Average: r = {corr_early_vision:.3f}
Late Semantic Average: r = {corr_late_vision:.3f}
Difference: +{corr_late_vision - corr_early_vision:.3f}
These now MATCH the bar chart numbers exactly!
Key Finding Preserved:
Language models prefer semantic regions (+{corr_late_method2 - corr_early_method2:.3f})
Vision models are balanced (+{corr_late_vision - corr_early_vision:.3f})
The GUI now uses the same scientifically rigorous Method 2 as the bar charts!
""")
print("="*80)
print("TEST COMPLETE - Changes verified!")
print("="*80)
print("""
Next steps:
1. Start the GUI: python app.py
2. Select: "AVERAGE - Neural Language (Captions)"
3. Brain measure: "Early Visual Average (7 ROIs)"
--> Should now show: Brain and ML Model: r = 0.208
4. Brain measure: "Late Semantic Average (12 ROIs)"
--> Should now show: Brain and ML Model: r = 0.312
These match the bar chart exactly!
""")
|