more cross comparing

script/hyperparameter_tuning.py

@@ -84,8 +84,9 @@ def objective(trial, hyperparam_run_dir, data_path):
             'dataset_label': dataset_label,
             'trial_number': trial.number,
             'parameters': trial.params,
-            'value': val_accuracy,
-            'visualization_dir': vis_dir
+            'accuracy': val_accuracy,
+            'visualization_dir': vis_dir,
+            'trial_dir': trial_dir
         }
         
         with open(os.path.join(trial_dir, 'trial_info.json'), 'w') as f:

script/visualization/analyze_trials.py

@@ -0,0 +1,225 @@
+import os
+import json
+import pandas as pd
+from pathlib import Path
+from collections import defaultdict
+
+def parse_error_analysis(vis_dir):
+    """Parse the error_analysis.txt file to get accuracy and misclassification details"""
+    metrics = {}
+    class_accuracies = {}
+    misclassified_files = []
+    
+    with open(os.path.join(vis_dir, 'error_analysis.txt'), 'r') as f:
+        lines = f.readlines()
+        parsing_errors = False
+        header_found = False
+        
+        for line in lines:
+            # Get overall accuracy
+            if line.startswith("Overall Accuracy:"):
+                metrics['overall_accuracy'] = float(line.split(":")[1].strip().rstrip('%')) / 100
+            
+            # Parse per-class accuracy
+            if "samples)" in line and ":" in line:
+                class_name = line.split(":")[0].strip()
+                accuracy = float(line.split(":")[1].split("%")[0].strip()) / 100
+                samples = int(line.split("(")[1].split(" ")[0])
+                class_accuracies[class_name] = {
+                    'accuracy': accuracy,
+                    'samples': samples
+                }
+            
+            # Parse misclassified files
+            if "Misclassified Videos:" in line:
+                parsing_errors = True
+                continue
+            if "Filename" in line and "True Class" in line:
+                header_found = True
+                continue
+            if parsing_errors and header_found and line.strip() and not line.startswith("-"):
+                try:
+                    # Split the line while preserving filename with spaces
+                    parts = line.strip().split()
+                    # Find the confidence value (last element with %)
+                    confidence_idx = next(i for i, part in enumerate(parts) if part.endswith('%'))
+                    # Everything before the last three elements is the filename
+                    filename = ' '.join(parts[:confidence_idx-2])
+                    true_class = parts[confidence_idx-2]
+                    pred_class = parts[confidence_idx-1]
+                    confidence = float(parts[confidence_idx].rstrip('%')) / 100
+                    
+                    misclassified_files.append({
+                        'filename': filename,
+                        'true_class': true_class,
+                        'predicted_class': pred_class,
+                        'confidence': confidence
+                    })
+                except Exception as e:
+                    print(f"Warning: Could not parse line: {line.strip()}")
+                    continue
+    
+    metrics['class_accuracies'] = class_accuracies
+    metrics['misclassified_files'] = misclassified_files
+    return metrics
+
+def analyze_trial(trial_dir):
+    """Analyze all visualization directories in a trial and aggregate results"""
+    trial_metrics = {
+        'overall_accuracy': 0,
+        'total_samples': 0,
+        'class_accuracies': defaultdict(lambda: {'correct': 0, 'total': 0}),
+        'misclassified_files': []
+    }
+    
+    # Find all visualization directories
+    vis_dirs = [d for d in trial_dir.iterdir() if d.is_dir() and d.name.startswith('visualization_')]
+    if not vis_dirs:
+        return None
+        
+    for vis_dir in vis_dirs:
+        try:
+            metrics = parse_error_analysis(vis_dir)
+            
+            # Add to total samples and weighted accuracy
+            samples = sum(m['samples'] for m in metrics['class_accuracies'].values())
+            trial_metrics['total_samples'] += samples
+            trial_metrics['overall_accuracy'] += metrics['overall_accuracy'] * samples
+            
+            # Aggregate per-class metrics
+            for class_name, class_metrics in metrics['class_accuracies'].items():
+                trial_metrics['class_accuracies'][class_name]['correct'] += (
+                    class_metrics['accuracy'] * class_metrics['samples']
+                )
+                trial_metrics['class_accuracies'][class_name]['total'] += class_metrics['samples']
+            
+            # Collect misclassified files with visualization directory info
+            for error in metrics['misclassified_files']:
+                error['vis_dir'] = vis_dir.name
+                trial_metrics['misclassified_files'].append(error)
+                
+        except Exception as e:
+            print(f"Error processing visualization directory {vis_dir}: {e}")
+    
+    # Calculate final metrics
+    if trial_metrics['total_samples'] > 0:
+        trial_metrics['overall_accuracy'] /= trial_metrics['total_samples']
+        
+        for class_metrics in trial_metrics['class_accuracies'].values():
+            if class_metrics['total'] > 0:
+                class_metrics['accuracy'] = class_metrics['correct'] / class_metrics['total']
+    
+    return trial_metrics
+
+def analyze_trials(hyperparam_dir):
+    results = {
+        'search_dirs': defaultdict(lambda: {
+            'best_overall': {'accuracy': 0, 'trial': None},
+            'best_per_class': defaultdict(lambda: {'accuracy': 0, 'trial': None}),
+            'misclassified_files': []
+        })
+    }
+    
+    # Process each search directory
+    for search_dir in Path(hyperparam_dir).iterdir():
+        if not search_dir.is_dir() or not search_dir.name.startswith('search_'):
+            continue
+            
+        # Process each trial directory
+        for trial_dir in search_dir.iterdir():
+            if not trial_dir.is_dir() or not trial_dir.name.startswith('trial_'):
+                continue
+            
+            trial_metrics = analyze_trial(trial_dir)
+            if trial_metrics is None:
+                continue
+                
+            search_results = results['search_dirs'][search_dir.name]
+            
+            # Update overall best for this search directory
+            if trial_metrics['overall_accuracy'] > search_results['best_overall']['accuracy']:
+                search_results['best_overall']['accuracy'] = trial_metrics['overall_accuracy']
+                search_results['best_overall']['trial'] = trial_dir.name
+            
+            # Update per-class bests for this search directory
+            for class_name, class_metrics in trial_metrics['class_accuracies'].items():
+                if class_metrics['accuracy'] > search_results['best_per_class'][class_name]['accuracy']:
+                    search_results['best_per_class'][class_name]['accuracy'] = class_metrics['accuracy']
+                    search_results['best_per_class'][class_name]['trial'] = trial_dir.name
+            
+            # Collect misclassified files
+            search_results['misclassified_files'].extend(trial_metrics['misclassified_files'])
+    
+    return results
+
+def save_analysis_report(results, hyperparam_dir):
+    output_file = os.path.join(hyperparam_dir, 'trial_analysis_report.txt')
+    
+    with open(output_file, 'w') as f:
+        for search_dir, search_results in results['search_dirs'].items():
+            f.write(f"\n=== Results for {search_dir} ===\n")
+            f.write("-" * 80 + "\n")
+            
+            # Best overall model
+            f.write("\nBest Overall Model:\n")
+            f.write(f"Trial: {search_results['best_overall']['trial']}\n")
+            f.write(f"Accuracy: {search_results['best_overall']['accuracy']:.2%}\n")
+            
+            # Best model per class
+            f.write("\nBest Model Per Class:\n")
+            f.write(f"{'Class':<20} {'Accuracy':<10} {'Trial'}\n")
+            f.write("-" * 60 + "\n")
+            for class_name, data in search_results['best_per_class'].items():
+                f.write(f"{class_name:<20} {data['accuracy']:.2%}    {data['trial']}\n")
+            
+            # Most frequently misclassified files
+            f.write("\nMost Frequently Misclassified Files:\n")
+            f.write(f"{'Filename':<40} {'True Class':<15} {'Predicted':<15} {'Confidence':<10} {'Dataset'}\n")
+            f.write("-" * 100 + "\n")
+            
+            # Sort misclassified files by confidence (ascending) to show most problematic cases first
+            misclassified = sorted(search_results['misclassified_files'], 
+                                 key=lambda x: x['confidence'])
+            for error in misclassified[:10]:  # Show top 10 most problematic
+                f.write(f"{error['filename']:<40} {error['true_class']:<15} "
+                       f"{error['predicted_class']:<15} {error['confidence']:<10.2%} {error['vis_dir']}\n")
+            
+            f.write("\n" + "=" * 80 + "\n")
+
+def print_results(results):
+    """Print a summary of the analysis results"""
+    for search_dir, search_results in results['search_dirs'].items():
+        print(f"\n=== Results for {search_dir} ===")
+        print("-" * 80)
+        
+        # Best overall model
+        print(f"\nBest Overall Model:")
+        print(f"Trial: {search_results['best_overall']['trial']}")
+        print(f"Accuracy: {search_results['best_overall']['accuracy']:.2%}")
+        
+        # Best model per class
+        print(f"\nBest Model Per Class:")
+        print(f"{'Class':<20} {'Accuracy':<10} {'Trial'}")
+        print("-" * 60)
+        for class_name, data in search_results['best_per_class'].items():
+            print(f"{class_name:<20} {data['accuracy']:.2%}    {data['trial']}")
+        
+        # Most frequently misclassified files (top 5)
+        print(f"\nTop 5 Most Problematic Files:")
+        print(f"{'Filename':<40} {'True Class':<15} {'Predicted':<15} {'Confidence'}")
+        print("-" * 80)
+        misclassified = sorted(search_results['misclassified_files'], 
+                             key=lambda x: x['confidence'])[:5]
+        for error in misclassified:
+            print(f"{error['filename']:<40} {error['true_class']:<15} "
+                  f"{error['predicted_class']:<15} {error['confidence']:.2%}")
+
+if __name__ == "__main__":
+    hyperparam_dir = "runs_hyperparam/hyperparam_20241106_124214"
+    results = analyze_trials(hyperparam_dir)
+    
+    # Print summary to console
+    print_results(results)
+    
+    # Save detailed results to file
+    save_analysis_report(results, hyperparam_dir) 

script/visualization/visualize.py

@@ -45,9 +45,10 @@ def generate_evaluation_metrics(model, data_loader, device, output_dir, class_la
     all_preds = []
     all_labels = []
     all_probs = []
+    all_files = []
 
     with torch.no_grad():
-        for frames, labels, _ in data_loader:
+        for frames, labels, filenames in data_loader:
             frames = frames.to(device)
             labels = labels.to(device)
             
@@ -58,11 +59,44 @@ def generate_evaluation_metrics(model, data_loader, device, output_dir, class_la
             all_preds.extend(predicted.cpu().numpy())
             all_labels.extend(labels.cpu().numpy())
             all_probs.extend(probs.cpu().numpy())
+            all_files.extend(filenames)
 
     all_labels = np.array(all_labels)
     all_preds = np.array(all_preds)
     all_probs = np.array(all_probs)
 
+    # Generate error analysis file
+    error_file = os.path.join(output_dir, 'error_analysis.txt')
+    with open(error_file, 'w') as f:
+        f.write(f"Error Analysis for {data_info}\n")
+        f.write("=" * 80 + "\n\n")
+        
+        # Overall accuracy
+        accuracy = (all_labels == all_preds).mean()
+        f.write(f"Overall Accuracy: {accuracy:.2%}\n\n")
+        
+        # Per-class accuracy
+        f.write("Per-Class Accuracy:\n")
+        for i, class_name in enumerate(class_labels):
+            class_mask = all_labels == i
+            if class_mask.sum() > 0:
+                class_acc = (all_preds[class_mask] == i).mean()
+                f.write(f"{class_name}: {class_acc:.2%} ({(class_mask).sum()} samples)\n")
+        f.write("\n")
+        
+        # Detailed error analysis
+        f.write("Misclassified Videos:\n")
+        f.write("-" * 80 + "\n")
+        f.write(f"{'Filename':<40} {'True Class':<20} {'Predicted Class':<20} Confidence\n")
+        f.write("-" * 80 + "\n")
+        
+        for i, (true_label, pred_label, probs, filename) in enumerate(zip(all_labels, all_preds, all_probs, all_files)):
+            if true_label != pred_label:
+                true_class = class_labels[true_label]
+                pred_class = class_labels[pred_label]
+                confidence = probs[pred_label]
+                f.write(f"{filename:<40} {true_class:<20} {pred_class:<20} {confidence:.2%}\n")
+
     # Compute and plot confusion matrix
     cm = confusion_matrix(all_labels, all_preds)
     plt.figure(figsize=(10, 8))
@@ -124,7 +158,11 @@ def run_visualization(run_dir, data_path=None, test_csv=None):
     
     class_labels = config['class_labels']
     num_classes = config['num_classes']
-    data_path = data_path or config['data_path']
+    
+    # Update the config's data_path if provided
+    if data_path:
+        config['data_path'] = data_path
+    data_path = config['data_path']
 
     # Paths
     log_file = os.path.join(run_dir, 'training_log.csv')
@@ -136,6 +174,8 @@ def run_visualization(run_dir, data_path=None, test_csv=None):
     # Get the last directory of data_path and the file name
     last_dir = os.path.basename(os.path.normpath(data_path))
     file_name = os.path.basename(test_csv)
+
+    print(f"Running visualization for {data_path} with {test_csv} from CWD {os.getcwd()}")
     
     # Create a directory for visualization outputs
     vis_dir = os.path.join(run_dir, f'visualization_{last_dir}_{file_name.split(".")[0]}')
@@ -164,8 +204,9 @@ def run_visualization(run_dir, data_path=None, test_csv=None):
 
 if __name__ == "__main__":
     # Find the most recent run directory
-    run_dir = get_latest_run_dir()
+    # run_dir = get_latest_run_dir()
+    run_dir = "/home/bawolf/workspace/break/clip/runs_hyperparam/hyperparam_20241106_124214/search_combined_adjusted/trial_combined_adjusted_20241106-195023/"
     # run_dir = "/home/bawolf/workspace/break/clip/runs/run_20241024-150232_otherpeopleval_large_model"
     # run_dir = "/home/bawolf/workspace/break/clip/runs/run_20241022-122939_3moves_balanced"
-    
-    run_visualization(run_dir)
+    data_path = "/home/bawolf/workspace/break/finetune/blog/combined/all"
+    run_visualization(run_dir, data_path=data_path)

script/visualization/viz_cross_compare.py

@@ -0,0 +1,54 @@
+import os
+from pathlib import Path
+from visualize import run_visualization
+
+def get_opposite_dataset_path(run_folder):
+    # Map run folders to their corresponding opposite dataset training files
+    dataset_mapping = {
+        'search_bryant_adjusted': '../finetune/blog/youtube/adjusted',
+        'search_bryant_random': '../finetune/blog/youtube/random',
+        'search_youtube_adjusted': '../finetune/blog/bryant/adjusted',
+        'search_youtube_random': '../finetune/blog/bryant/random'
+    }
+    
+    for folder_prefix, dataset_path in dataset_mapping.items():
+        if run_folder.startswith(folder_prefix):
+            return dataset_path
+    return None
+
+def process_runs(base_dir):
+    # Get the full path to the runs directory
+    runs_dir = Path(base_dir)
+    
+    # Process each search directory
+    for search_dir in runs_dir.iterdir():
+        if not search_dir.is_dir() or search_dir.name == 'visualization':
+            continue
+            
+        # Get the opposite dataset path for this search directory
+        opposite_dataset = get_opposite_dataset_path(search_dir.name)
+        
+        if opposite_dataset is not None:
+            print(f"Skipping {search_dir.name} - no matching dataset mapping")
+            continue
+        
+        # Process each trial directory within the search directory
+        for trial_dir in search_dir.iterdir():
+            if not trial_dir.is_dir() or not trial_dir.name.startswith('trial_'):
+                continue
+                
+            print(f"Processing {trial_dir} with {opposite_dataset}")
+            try:
+                vis_dir, cm = run_visualization(
+                    run_dir=str(trial_dir),
+                    data_path=opposite_dataset,
+                    test_csv=os.path.join(opposite_dataset, "train.csv")
+                )
+                print(f"Visualization complete: {vis_dir}")
+            except Exception as e:
+                print(f"Error processing {trial_dir}: {e}")
+
+if __name__ == "__main__":
+    # Example usage
+    runs_path = "runs_hyperparam/hyperparam_20241106_124214"
+    process_runs(runs_path)

src/dataset/dataset.py

@@ -48,6 +48,11 @@ class VideoDataset(Dataset):
     def __getitem__(self, idx):
         video_path, label = self.data[idx]
         
+        if not os.path.exists(video_path):
+            print(f"File not found: {video_path}")
+            print(f"Absolute path attempt: {os.path.abspath(video_path)}")
+            raise FileNotFoundError(f"File not found: {video_path}")
+        
         frames, success = extract_frames(video_path, 
                                       {"max_frames": self.max_frames, "sigma": self.sigma}, 
                                       self.transform)