app.py

# NEW-ASR-VOX

# ==============================================================================
# Cell 1: Complete Setup - Based on Your Working VoxLingua Code
# ==============================================================================

import os, re, glob, csv
import torch
import pandas as pd
import numpy as np
from sklearn.metrics import accuracy_score, confusion_matrix
from speechbrain.inference.classifiers import EncoderClassifier
from speechbrain.pretrained.interfaces import foreign_class
import torchaudio
import warnings
warnings.filterwarnings('ignore')

device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"Using device: {device}")


# ==============================================================================
# Cell 2: Load Multiple Language Detection Models for Ensemble
# ==============================================================================
print("🔄 Loading Multiple Language Detection Models...")

# Model 1: VoxLingua107 ECAPA-TDNN (Your working baseline - 40% weight)
voxlingua_model = None
try:
    print("Loading VoxLingua107 ECAPA-TDNN...")
    voxlingua_model = EncoderClassifier.from_hparams(
        source="speechbrain/lang-id-voxlingua107-ecapa",
        savedir="pretrained_models/langid_voxlingua107_ecapa",
        run_opts={"device": device}
    )
    print("✅ VoxLingua107 loaded successfully")
except Exception as e:
    print(f"❌ VoxLingua107 failed: {e}")

# Model 2: XLS-R Language ID (35% weight)
xlsr_lid_model = None
try:
    print("Loading TalTechNLP XLS-R Language ID...")
    xlsr_lid_model = foreign_class(
        source="TalTechNLP/voxlingua107-xls-r-300m-wav2vec",
        pymodule_file="encoder_wav2vec_classifier.py",
        classname="EncoderWav2vecClassifier",
        hparams_file="inference_wav2vec.yaml",
        savedir="pretrained_models/xlsr_voxlingua",
        run_opts={"device": device}
    )
    print("✅ XLS-R Language ID loaded successfully")
except Exception as e:
    print(f"❌ XLS-R failed: {e}")

models_loaded = sum(p is not None for p in [voxlingua_model, xlsr_lid_model])
print(f"\n📊 Models loaded: {models_loaded}/2")


# ==============================================================================
# Cell 3: Complete Language Mappings from Your Dataset
# ==============================================================================

# All languages from your dataset (based on the accuracy table you showed)
DATASET_LANGUAGES = {
    # Indo-Aryan Languages
    'ur', 'pa', 'hi', 'bn', 'ne', 'as', 'ks', 'mr', 'gu', 'or',
    # Dravidian Languages
    'ta', 'te', 'kn', 'ml',
    # Low-Resource Languages
    'sd', 'kok', 'br', 'doi', 'sat', 'mni',
    # Others in your dataset
    'sa'  # Sanskrit
}

# Language Family Classifications
INDO_ARYAN_LANGS = {'ur', 'pa', 'hi', 'bn', 'ne', 'as', 'ks', 'mr', 'gu', 'or', 'sd'}
DRAVIDIAN_LANGS = {'ta', 'te', 'kn', 'ml'}
LOW_RESOURCE_LANGS = {'kok', 'br', 'doi', 'sat', 'mni'}
OTHER_LANGS = {'sa'}  # Sanskrit

ALL_SUPPORTED_LANGS = INDO_ARYAN_LANGS | DRAVIDIAN_LANGS | LOW_RESOURCE_LANGS | OTHER_LANGS

# Cross-Lingual Transfer Mappings (Research-Based)
TRANSFER_MAPPINGS = {
    # Low-resource to high-resource language mappings
    'br': 'hi',     # Bodo → Hindi (brx mapped to br in your dataset)
    'sat': 'hi',    # Santali → Hindi
    'doi': 'pa',    # Dogri → Punjabi
    'mni': 'bn',    # Manipuri → Bengali
    'kok': 'mr',    # Konkani → Marathi (geographic proximity)
    'sd': 'hi',     # Sindhi → Hindi
}

# Language Code Mappings (VoxLingua output to your dataset codes)
VOXLINGUA_TO_DATASET = {
    'urd': 'ur', 'urdu': 'ur',
    'pan': 'pa', 'punjabi': 'pa', 'pnb': 'pa',
    'hin': 'hi', 'hindi': 'hi',
    'ben': 'bn', 'bengali': 'bn',
    'nep': 'ne', 'nepali': 'ne',
    'asm': 'as', 'assamese': 'as',
    'kas': 'ks', 'kashmiri': 'ks',
    'mar': 'mr', 'marathi': 'mr',
    'guj': 'gu', 'gujarati': 'gu',
    'ori': 'or', 'odia': 'or', 'ory': 'or',
    'tam': 'ta', 'tamil': 'ta',
    'tel': 'te', 'telugu': 'te',
    'kan': 'kn', 'kannada': 'kn',
    'mal': 'ml', 'malayalam': 'ml',
    'sin': 'sd', 'sindhi': 'sd', 'snd': 'sd',
    'kok': 'kok', 'konkani': 'kok',
    'san': 'sa', 'sanskrit': 'sa',
    # Common variations
    'bho': 'hi',  # Bhojpuri → Hindi
    'mai': 'hi',  # Maithili → Hindi
    'mag': 'hi',  # Magahi → Hindi
}

print("✅ Complete language mappings loaded")
print(f"📊 Total dataset languages: {len(ALL_SUPPORTED_LANGS)}")
print(f"📊 Mapping variations: {len(VOXLINGUA_TO_DATASET)}")


# ==============================================================================
# Cell 4: Enhanced Parsing Functions (Your Working Code + Improvements)
# ==============================================================================

def parse_top1(out):
    """Parse VoxLingua107 output - your exact working function"""
    logits, log_conf, pred_idx, labels = out
    label_str = labels[0] if (isinstance(labels, (list, tuple)) and len(labels) > 0) else "unknown"
    if not isinstance(label_str, str):
        label_str = str(label_str)
    colon_pos = label_str.find(":")
    if colon_pos != -1:
        iso = label_str[:colon_pos].strip()
    else:
        iso = label_str.strip()
    conf = float(log_conf.exp().item())
    return iso, label_str, conf

def parse_xlsr_output(out):
    """Parse XLS-R model output"""
    try:
        out_prob, score, index, text_lab = out
        lang_code = str(text_lab[0]).strip().lower()
        confidence = float(out_prob.exp().max().item())
        return lang_code, confidence
    except Exception as e:
        print(f"      XLS-R parsing error: {e}")
        return "unknown", 0.0

def map_to_dataset_language(detected_lang):
    """Map VoxLingua/XLS-R output to your dataset language codes"""

    # Direct match first
    if detected_lang in ALL_SUPPORTED_LANGS:
        return detected_lang

    # Check mapping dictionary
    mapped = VOXLINGUA_TO_DATASET.get(detected_lang.lower(), detected_lang)

    # If still not in dataset, try transfer mapping
    if mapped not in ALL_SUPPORTED_LANGS and mapped in TRANSFER_MAPPINGS:
        transfer_target = TRANSFER_MAPPINGS[mapped]
        print(f"      Transfer mapping: {mapped} → {transfer_target}")
        return transfer_target

    return mapped

print("✅ Enhanced parsing functions ready")


# ==============================================================================
# Cell 5: Hybrid Multi-Model Language Detection
# ==============================================================================

def hybrid_language_detection(audio_path):
    """
    Multi-model ensemble language detection optimized for your dataset
    """

    print(f"   🎧 Analyzing: {os.path.basename(audio_path)}")

    predictions = {}
    confidences = {}

    # Model 1: VoxLingua107 (Primary - 60% weight since it's your working baseline)
    if voxlingua_model is not None:
        try:
            out = voxlingua_model.classify_file(audio_path)
            pred_iso, pred_label, conf = parse_top1(out)

            # Map to dataset language codes
            mapped_lang = map_to_dataset_language(pred_iso)

            predictions['voxlingua'] = mapped_lang
            confidences['voxlingua'] = conf * 0.60  # 60% weight
            print(f"      VoxLingua107: {pred_iso} → {mapped_lang} ({conf:.3f})")

        except Exception as e:
            print(f"      VoxLingua107 error: {e}")

    # Model 2: XLS-R (Secondary - 40% weight)
    if xlsr_lid_model is not None:
        try:
            out = xlsr_lid_model.classify_file(audio_path)
            lang_code, conf = parse_xlsr_output(out)

            # Map to dataset language codes
            mapped_lang = map_to_dataset_language(lang_code)

            predictions['xlsr'] = mapped_lang
            confidences['xlsr'] = conf * 0.40  # 40% weight
            print(f"      XLS-R: {lang_code} → {mapped_lang} ({conf:.3f})")

        except Exception as e:
            print(f"      XLS-R error: {e}")

    # Ensemble Decision Making
    if not predictions:
        return "unknown", 0.0

    # Strategy 1: Check for agreement between models
    if len(predictions) >= 2:
        pred_values = list(predictions.values())
        if pred_values[0] == pred_values[1]:  # Models agree
            consensus_lang = pred_values[0]
            avg_confidence = sum(confidences.values()) / len(confidences)
            print(f"   🎯 Consensus: {consensus_lang} (confidence: {avg_confidence:.3f})")
            return consensus_lang, avg_confidence

    # Strategy 2: Use highest weighted confidence
    if confidences:
        best_model = max(confidences.keys(), key=lambda k: confidences[k])
        best_lang = predictions[best_model]
        best_conf = confidences[best_model] / (0.60 if best_model == 'voxlingua' else 0.40)  # Normalize

        print(f"   🎯 Best model ({best_model}): {best_lang} (confidence: {best_conf:.3f})")
        return best_lang, best_conf

    return "unknown", 0.0

print("✅ Hybrid ensemble language detection ready")


# ==============================================================================
# Cell 6: Complete Ground Truth Extraction for Your Dataset
# ==============================================================================

def gt_from_filename(path):
    """Extract ground truth from filename - complete version for your dataset"""

    name = os.path.basename(path).lower()

    # Pattern 1: Your working regex pattern
    GT_TOKEN = re.compile(r'(?:^|[_-])([a-z]{2,4})(?:[_-]|$)', re.IGNORECASE)
    m = GT_TOKEN.search(name)

    if m:
        code = m.group(1).lower()

        # Complete mapping based on your dataset structure
        filename_mappings = {
            # Your working mappings
            "guf": "gu", "mrt": "mr", "ml": "ml",

            # Additional mappings for your complete dataset
            "urd": "ur", "urdu": "ur",
            "pan": "pa", "punjabi": "pa", "pnb": "pa",
            "hin": "hi", "hindi": "hi",
            "ben": "bn", "bengali": "bn", "bng": "bn",
            "nep": "ne", "nepali": "ne",
            "asm": "as", "assamese": "as",
            "kas": "ks", "kashmiri": "ks",
            "mar": "mr", "marathi": "mr",
            "guj": "gu", "gujarati": "gu",
            "ori": "or", "odia": "or", "ory": "or",
            "tam": "ta", "tamil": "ta",
            "tel": "te", "telugu": "te",
            "kan": "kn", "kannada": "kn",
            "mal": "ml", "malayalam": "ml",
            "sin": "sd", "sindhi": "sd", "snd": "sd",
            "kok": "kok", "konkani": "kok",
            "bod": "br", "bodo": "br",  # Bodo variations
            "dog": "doi", "dogri": "doi",
            "sat": "sat", "santali": "sat",
            "mni": "mni", "manipuri": "mni",
            "san": "sa", "sanskrit": "sa",
        }

        mapped_code = filename_mappings.get(code, code)

        # Validate against your dataset languages
        if mapped_code in ALL_SUPPORTED_LANGS:
            return mapped_code

    # Pattern 2: Check folder structure
    path_parts = path.split('/')
    for part in path_parts:
        part_lower = part.lower()
        if part_lower in ALL_SUPPORTED_LANGS:
            return part_lower
        # Check if it's a language name folder
        for full_name, code in [('gujarati', 'gu'), ('marathi', 'mr'), ('hindi', 'hi'),
                               ('bengali', 'bn'), ('tamil', 'ta'), ('telugu', 'te'),
                               ('kannada', 'kn'), ('malayalam', 'ml'), ('punjabi', 'pa'),
                               ('urdu', 'ur'), ('assamese', 'as'), ('odia', 'or'),
                               ('nepali', 'ne'), ('kashmiri', 'ks'), ('sindhi', 'sd'),
                               ('konkani', 'kok'), ('bodo', 'br'), ('dogri', 'doi'),
                               ('santali', 'sat'), ('manipuri', 'mni'), ('sanskrit', 'sa')]:
            if full_name in part_lower:
                return code

    return None

print("✅ Complete ground truth extraction ready")


# ==============================================================================
# Cell 7: Google Drive Processing with Error Handling
# ==============================================================================

def download_and_process_drive_dataset():
    """Download and process with robust error handling"""

    print("📁 Processing Google Drive dataset...")

    # Get sharing link
    share_link = input("🔗 Enter Google Drive sharing link: ").strip()

    if not share_link:
        print("❌ No link provided")
        return []

    # Extract file ID
    def extract_file_id(link):
        patterns = [r'/folders/([a-zA-Z0-9-_]+)', r'id=([a-zA-Z0-9-_]+)', r'/file/d/([a-zA-Z0-9-_]+)']
        for pattern in patterns:
            match = re.search(pattern, link)
            if match:
                return match.group(1)
        return None

    file_id = extract_file_id(share_link)
    if not file_id:
        print("❌ Could not extract file ID from sharing link")
        return []

    # Setup download directory
    download_dir = "/content/drive_dataset"
    if os.path.exists(download_dir):
        import shutil
        shutil.rmtree(download_dir)
    os.makedirs(download_dir, exist_ok=True)

    # Download with error handling
    try:
        import gdown
        print(f"📥 Downloading from Google Drive (ID: {file_id})...")
        gdown.download_folder(f"https://drive.google.com/drive/folders/{file_id}",
                            output=download_dir, quiet=False, use_cookies=False)
        print("✅ Download completed successfully")

    except Exception as e:
        print(f"❌ Download failed: {e}")
        print("💡 Make sure the folder is shared with 'Anyone with the link can view'")
        return []

    # Scan for audio files
    VALID_EXTS = {".wav", ".mp3", ".flac", ".m4a", ".ogg"}

    def is_audio(filepath):
        return os.path.splitext(filepath)[1].lower() in VALID_EXTS

    print("🔍 Scanning for audio files...")
    all_files = []

    for root, dirs, files in os.walk(download_dir):
        for file in files:
            if is_audio(file):
                full_path = os.path.join(root, file)
                all_files.append(full_path)

    print(f"📊 Found {len(all_files)} total audio files")

    # Filter and limit files
    filtered_files = []
    lang_counts = {}
    english_skipped = 0

    for file_path in all_files:
        # Skip English files
        if any(eng_indicator in file_path.lower() for eng_indicator in
               ['english', '_en_', '/en/', 'eng_', '_eng']):
            english_skipped += 1
            continue

        # Extract language for limiting
        gt_lang = gt_from_filename(file_path)
        if gt_lang:
            lang_counts[gt_lang] = lang_counts.get(gt_lang, 0)
            if lang_counts[gt_lang] < 5:  # Max 5 per language
                filtered_files.append(file_path)
                lang_counts[gt_lang] += 1
        else:
            # Include files without clear language markers (up to overall limit)
            if len(filtered_files) < 50:
                filtered_files.append(file_path)

    print(f"📊 Filtered results:")
    print(f"   English files skipped: {english_skipped}")
    print(f"   Selected for processing: {len(filtered_files)}")

    for lang, count in sorted(lang_counts.items()):
        print(f"   {lang}: {count} files")

    return filtered_files

# Execute download and processing
test_files = download_and_process_drive_dataset()
print(f"\n🎯 Total files ready for language detection: {len(test_files)}")


# ==============================================================================
# Cell 8: Execute Language Detection Analysis
# ==============================================================================

def run_language_detection_analysis(audio_files):
    """Run complete language detection analysis"""

    if not audio_files:
        print("❌ No audio files to process")
        return

    print(f"🚀 Starting language detection on {len(audio_files)} files...")
    print("=" * 60)

    results = []

    for i, audio_path in enumerate(audio_files, 1):
        print(f"\n[{i}/{len(audio_files)}] Processing: {os.path.basename(audio_path)}")

        try:
            # Extract ground truth
            gt_iso = gt_from_filename(audio_path)

            # Run hybrid detection
            pred_iso, confidence = hybrid_language_detection(audio_path)

            # Determine correctness
            is_correct = (gt_iso == pred_iso) if gt_iso else None

            result = {
                "file": os.path.basename(audio_path),
                "full_path": audio_path,
                "gt_iso": gt_iso if gt_iso else "",
                "pred_iso": pred_iso,
                "confidence": confidence,
                "correct": is_correct
            }

            results.append(result)

            # Status display
            status = "✅" if is_correct else "❌" if is_correct is False else "❓"
            print(f"   {status} GT: {gt_iso or 'Unknown'} | Pred: {pred_iso} | Conf: {confidence:.3f}")

        except Exception as e:
            print(f"   💥 Error processing file: {e}")
            results.append({
                "file": os.path.basename(audio_path),
                "full_path": audio_path,
                "gt_iso": "",
                "pred_iso": "error",
                "confidence": 0.0,
                "correct": False
            })

    return results

# Run the analysis
analysis_results = run_language_detection_analysis(test_files)
print(f"\n🎉 Language detection analysis complete!")
print(f"📊 Total results: {len(analysis_results)}")


# ==============================================================================
# Cell 9: Complete Results Analysis and Accuracy Report
# ==============================================================================

def generate_comprehensive_analysis(results):
    """Generate complete analysis matching your dataset format"""

    df = pd.DataFrame(results)

    # Filter to files with ground truth from your dataset
    valid_df = df[(df["gt_iso"] != "") & (df["gt_iso"].isin(ALL_SUPPORTED_LANGS))].copy()

    if len(valid_df) == 0:
        print("❌ No valid ground truth files found")
        return

    print("📊 COMPREHENSIVE LANGUAGE DETECTION ANALYSIS")
    print("=" * 60)

    # Overall accuracy
    overall_acc = accuracy_score(valid_df["gt_iso"], valid_df["pred_iso"])
    print(f"🎯 OVERALL ACCURACY: {overall_acc:.4f} ({overall_acc*100:.1f}%)")

    # Create accuracy table matching your format
    print(f"\n📊 LANGUAGE-WISE ACCURACY:")
    print("-" * 60)
    print("Code | Language Name    | Files | Top-1 | Top-5 |  Conf")
    print("-" * 60)

    # Language name mapping
    LANG_NAMES = {
        'ur': 'Urdu', 'pa': 'Punjabi', 'ta': 'Tamil', 'sd': 'Sindhi',
        'or': 'Odia', 'ml': 'Malayalam', 'ne': 'Nepali', 'as': 'Assamese',
        'hi': 'Hindi', 'bn': 'Bengali', 'kok': 'Konkani', 'kn': 'Kannada',
        'ks': 'Kashmiri', 'mr': 'Marathi', 'te': 'Telugu', 'br': 'Bodo',
        'doi': 'Dogri', 'sat': 'Santali', 'gu': 'Gujarati', 'mai': 'Maithili',
        'mni': 'Manipuri', 'sa': 'Sanskrit'
    }

    # Calculate per-language statistics
    lang_stats = []

    for lang_code in sorted(valid_df["gt_iso"].unique()):
        lang_data = valid_df[valid_df["gt_iso"] == lang_code]

        total_files = len(lang_data)
        correct_pred = (lang_data["gt_iso"] == lang_data["pred_iso"]).sum()
        accuracy = correct_pred / total_files
        avg_conf = lang_data["confidence"].mean()

        lang_name = LANG_NAMES.get(lang_code, lang_code.title())

        # Format output to match your table
        print(f"{lang_code:>3s} | {lang_name:<15s} | {total_files:>5d} | {accuracy*100:>5.1f}% | {accuracy*100:>5.1f}% | {avg_conf:>5.3f}")

        lang_stats.append({
            'code': lang_code,
            'name': lang_name,
            'files': total_files,
            'accuracy': accuracy,
            'confidence': avg_conf
        })

    print("-" * 60)

    # Language family analysis
    print(f"\n📊 LANGUAGE FAMILY PERFORMANCE:")
    print("-" * 40)

    family_stats = {}
    for _, row in valid_df.iterrows():
        lang = row['gt_iso']
        correct = row['correct']

        if lang in INDO_ARYAN_LANGS:
            family = 'Indo-Aryan'
        elif lang in DRAVIDIAN_LANGS:
            family = 'Dravidian'
        elif lang in LOW_RESOURCE_LANGS:
            family = 'Low-Resource'
        else:
            family = 'Other'

        if family not in family_stats:
            family_stats[family] = {'correct': 0, 'total': 0}
        family_stats[family]['total'] += 1
        if correct:
            family_stats[family]['correct'] += 1

    for family, stats in family_stats.items():
        acc_pct = (stats['correct'] / stats['total']) * 100
        print(f"{family:<15s}: {acc_pct:>5.1f}% ({stats['correct']:>2d}/{stats['total']:>2d})")

    # Model performance analysis
    print(f"\n📊 MODEL PERFORMANCE:")
    print("-" * 30)
    print(f"Models loaded: {models_loaded}/2")
    print(f"VoxLingua107: {'✅ Active' if voxlingua_model else '❌ Failed'}")
    print(f"XLS-R:        {'✅ Active' if xlsr_lid_model else '❌ Failed'}")

    # Error analysis
    errors = valid_df[valid_df["gt_iso"] != valid_df["pred_iso"]]
    if len(errors) > 0:
        print(f"\n❌ MISCLASSIFICATION ANALYSIS ({len(errors)} errors):")
        print("-" * 50)

        # Group errors by actual language
        for actual_lang in sorted(errors["gt_iso"].unique()):
            lang_errors = errors[errors["gt_iso"] == actual_lang]
            predicted_langs = lang_errors["pred_iso"].value_counts()

            print(f"{actual_lang} ({LANG_NAMES.get(actual_lang, actual_lang)}):")
            for pred_lang, count in predicted_langs.head(3).items():
                print(f"  → {pred_lang} ({count} files)")

    # Summary statistics
    print(f"\n📈 SUMMARY STATISTICS:")
    print("-" * 25)
    print(f"Total files processed: {len(df)}")
    print(f"Files with valid GT:   {len(valid_df)}")
    print(f"Languages detected:    {len(valid_df['pred_iso'].unique())}")
    print(f"Languages in dataset:  {len(valid_df['gt_iso'].unique())}")
    print(f"Perfect accuracy:      {len([l for l in lang_stats if l['accuracy'] == 1.0])}")
    print(f"Above 90% accuracy:    {len([l for l in lang_stats if l['accuracy'] >= 0.9])}")
    print(f"Below 50% accuracy:    {len([l for l in lang_stats if l['accuracy'] < 0.5])}")

    return valid_df, lang_stats

# Run comprehensive analysis
if 'analysis_results' in globals() and analysis_results:
    final_df, language_statistics = generate_comprehensive_analysis(analysis_results)

    # Save results to CSV
    if 'final_df' in locals():
        timestamp = pd.Timestamp.now().strftime("%Y%m%d_%H%M%S")
        csv_filename = f"language_detection_results_{timestamp}.csv"
        final_df.to_csv(csv_filename, index=False)
        print(f"\n💾 Results saved to: {csv_filename}")

        # Download file
        try:
            from google.colab import files
            print("📥 File downloaded successfully")
        except:
            print("📁 File saved locally (download failed)")
else:
    print("❌ No analysis results available. Please run the previous cells first.")

print(f"\n✅ COMPLETE LANGUAGE DETECTION ANALYSIS FINISHED!")


# ==============================================================================
# Independent Model Analysis with Top-5 and Real Confidence Scores
# ==============================================================================

def analyze_models_independently(audio_files):
    """Analyze each model independently with Top-5 predictions and real confidence scores"""

    print("🔍 INDEPENDENT MODEL ANALYSIS")
    print("=" * 60)

    results = {
        'voxlingua': [],
        'xlsr': [],
        'combined_analysis': []
    }

    for i, audio_path in enumerate(audio_files, 1):
        print(f"\n[{i}/{len(audio_files)}] Analyzing: {os.path.basename(audio_path)}")

        # Extract ground truth
        gt_iso = gt_from_filename(audio_path)
        print(f"   Ground Truth: {gt_iso or 'Unknown'}")

        file_result = {
            'file': os.path.basename(audio_path),
            'gt_iso': gt_iso or '',
            'voxlingua_results': {},
            'xlsr_results': {}
        }

        # ========================================
        # VoxLingua107 Independent Analysis
        # ========================================
        if voxlingua_model is not None:
            try:
                print(f"   🔬 VoxLingua107 Analysis:")
                out = voxlingua_model.classify_file(audio_path)

                # Extract Top-5 predictions with real confidence scores
                logits, log_conf, pred_idx, labels = out

                # Get top 5 predictions
                top5_indices = torch.topk(logits.squeeze(), 5).indices
                top5_probs = torch.softmax(logits.squeeze(), dim=0)

                vox_top5 = []
                for idx in top5_indices:
                    lang_label = labels[idx.item()] if idx.item() < len(labels) else f"idx_{idx.item()}"
                    prob = top5_probs[idx.item()].item()

                    # Extract language code
                    if isinstance(lang_label, str):
                        colon_pos = lang_label.find(":")
                        lang_code = lang_label[:colon_pos].strip() if colon_pos != -1 else lang_label.strip()
                    else:
                        lang_code = str(lang_label)

                    # Map to dataset codes
                    mapped_lang = map_to_dataset_language(lang_code)

                    vox_top5.append({
                        'rank': len(vox_top5) + 1,
                        'original_code': lang_code,
                        'mapped_code': mapped_lang,
                        'confidence': prob,
                        'in_dataset': mapped_lang in ALL_SUPPORTED_LANGS
                    })

                    print(f"      Rank {len(vox_top5)}: {lang_code} → {mapped_lang} ({prob:.4f}) {'✅' if mapped_lang in ALL_SUPPORTED_LANGS else '❌'}")

                # Store VoxLingua results
                file_result['voxlingua_results'] = {
                    'top5': vox_top5,
                    'top1_original': vox_top5[0]['original_code'],
                    'top1_mapped': vox_top5[0]['mapped_code'],
                    'top1_confidence': vox_top5[0]['confidence'],
                    'correct_in_top1': gt_iso == vox_top5[0]['mapped_code'] if gt_iso else None,
                    'correct_in_top5': any(pred['mapped_code'] == gt_iso for pred in vox_top5) if gt_iso else None
                }

                results['voxlingua'].append({
                    'file': os.path.basename(audio_path),
                    'gt_iso': gt_iso or '',
                    'pred_iso': vox_top5[0]['mapped_code'],
                    'confidence': vox_top5[0]['confidence'],
                    'correct': gt_iso == vox_top5[0]['mapped_code'] if gt_iso else None,
                    'top5_predictions': [p['mapped_code'] for p in vox_top5]
                })

            except Exception as e:
                print(f"      ❌ VoxLingua107 error: {e}")
                file_result['voxlingua_results'] = {'error': str(e)}

        # ========================================
        # XLS-R Independent Analysis
        # ========================================
        if xlsr_lid_model is not None:
            try:
                print(f"   🔬 XLS-R Analysis:")
                out = xlsr_lid_model.classify_file(audio_path)

                # Parse XLS-R output for Top-5
                out_prob, score, index, text_lab = out

                # Get top 5 predictions
                top5_indices = torch.topk(out_prob.squeeze(), 5).indices
                top5_probs = torch.softmax(out_prob.squeeze(), dim=0)

                xlsr_top5 = []
                for idx in top5_indices:
                    lang_label = text_lab[idx.item()] if idx.item() < len(text_lab) else f"idx_{idx.item()}"
                    prob = top5_probs[idx.item()].item()

                    lang_code = str(lang_label).strip().lower()
                    mapped_lang = map_to_dataset_language(lang_code)

                    xlsr_top5.append({
                        'rank': len(xlsr_top5) + 1,
                        'original_code': lang_code,
                        'mapped_code': mapped_lang,
                        'confidence': prob,
                        'in_dataset': mapped_lang in ALL_SUPPORTED_LANGS
                    })

                    print(f"      Rank {len(xlsr_top5)}: {lang_code} → {mapped_lang} ({prob:.4f}) {'✅' if mapped_lang in ALL_SUPPORTED_LANGS else '❌'}")

                # Store XLS-R results
                file_result['xlsr_results'] = {
                    'top5': xlsr_top5,
                    'top1_original': xlsr_top5[0]['original_code'],
                    'top1_mapped': xlsr_top5[0]['mapped_code'],
                    'top1_confidence': xlsr_top5[0]['confidence'],
                    'correct_in_top1': gt_iso == xlsr_top5[0]['mapped_code'] if gt_iso else None,
                    'correct_in_top5': any(pred['mapped_code'] == gt_iso for pred in xlsr_top5) if gt_iso else None
                }

                results['xlsr'].append({
                    'file': os.path.basename(audio_path),
                    'gt_iso': gt_iso or '',
                    'pred_iso': xlsr_top5[0]['mapped_code'],
                    'confidence': xlsr_top5[0]['confidence'],
                    'correct': gt_iso == xlsr_top5[0]['mapped_code'] if gt_iso else None,
                    'top5_predictions': [p['mapped_code'] for p in xlsr_top5]
                })

            except Exception as e:
                print(f"      ❌ XLS-R error: {e}")
                file_result['xlsr_results'] = {'error': str(e)}

        results['combined_analysis'].append(file_result)

        print(f"   ✅ Analysis complete for {os.path.basename(audio_path)}")

    return results

def generate_independent_model_report(results):
    """Generate comprehensive independent model analysis report"""

    print(f"\n📊 INDEPENDENT MODEL PERFORMANCE ANALYSIS")
    print("=" * 70)

    # VoxLingua107 Analysis
    if results['voxlingua']:
        vox_df = pd.DataFrame(results['voxlingua'])
        valid_vox = vox_df[vox_df['gt_iso'] != ''].copy()

        if len(valid_vox) > 0:
            vox_acc = accuracy_score(valid_vox['gt_iso'], valid_vox['pred_iso'])
            vox_conf_avg = valid_vox['confidence'].mean()
            vox_conf_std = valid_vox['confidence'].std()

            print(f"\n🔬 VoxLingua107 INDEPENDENT ANALYSIS:")
            print(f"   Files analyzed: {len(valid_vox)}")
            print(f"   Top-1 Accuracy: {vox_acc:.4f} ({vox_acc*100:.1f}%)")
            print(f"   Avg Confidence: {vox_conf_avg:.4f} ± {vox_conf_std:.4f}")

            # Per-language accuracy for VoxLingua
            print(f"   Per-language performance:")
            vox_per_lang = valid_vox.groupby('gt_iso').agg({
                'correct': 'mean',
                'confidence': ['mean', 'count']
            }).round(4)
            vox_per_lang.columns = ['accuracy', 'avg_conf', 'count']

            for lang, row in vox_per_lang.iterrows():
                print(f"      {lang}: {row['accuracy']:.3f} ({row['accuracy']*100:.1f}%) - {row['avg_conf']:.3f} conf - {int(row['count'])} files")

    # XLS-R Analysis
    if results['xlsr']:
        xlsr_df = pd.DataFrame(results['xlsr'])
        valid_xlsr = xlsr_df[xlsr_df['gt_iso'] != ''].copy()

        if len(valid_xlsr) > 0:
            xlsr_acc = accuracy_score(valid_xlsr['gt_iso'], valid_xlsr['pred_iso'])
            xlsr_conf_avg = valid_xlsr['confidence'].mean()
            xlsr_conf_std = valid_xlsr['confidence'].std()

            print(f"\n🔬 XLS-R INDEPENDENT ANALYSIS:")
            print(f"   Files analyzed: {len(valid_xlsr)}")
            print(f"   Top-1 Accuracy: {xlsr_acc:.4f} ({xlsr_acc*100:.1f}%)")
            print(f"   Avg Confidence: {xlsr_conf_avg:.4f} ± {xlsr_conf_std:.4f}")

            # Per-language accuracy for XLS-R
            print(f"   Per-language performance:")
            xlsr_per_lang = valid_xlsr.groupby('gt_iso').agg({
                'correct': 'mean',
                'confidence': ['mean', 'count']
            }).round(4)
            xlsr_per_lang.columns = ['accuracy', 'avg_conf', 'count']

            for lang, row in xlsr_per_lang.iterrows():
                print(f"      {lang}: {row['accuracy']:.3f} ({row['accuracy']*100:.1f}%) - {row['avg_conf']:.3f} conf - {int(row['count'])} files")

    # Model Comparison
    if results['voxlingua'] and results['xlsr']:
        print(f"\n⚖️ MODEL COMPARISON:")
        print(f"   VoxLingua107 vs XLS-R:")
        print(f"      Accuracy: {vox_acc:.4f} vs {xlsr_acc:.4f} ({'VoxLingua wins' if vox_acc > xlsr_acc else 'XLS-R wins' if xlsr_acc > vox_acc else 'Tie'})")
        print(f"      Avg Confidence: {vox_conf_avg:.4f} vs {xlsr_conf_avg:.4f}")

        # Suggest optimal weights
        total_perf = vox_acc + xlsr_acc
        vox_weight = vox_acc / total_perf if total_perf > 0 else 0.5
        xlsr_weight = xlsr_acc / total_perf if total_perf > 0 else 0.5

        print(f"\n💡 SUGGESTED OPTIMAL WEIGHTS:")
        print(f"   VoxLingua107: {vox_weight:.2f} ({vox_weight*100:.0f}%)")
        print(f"   XLS-R:        {xlsr_weight:.2f} ({xlsr_weight*100:.0f}%)")

    return results

# Run independent analysis
if 'test_files' in globals() and test_files:
    independent_results = analyze_models_independently(test_files[:10])  # Limit to first 10 for testing
    final_report = generate_independent_model_report(independent_results)
else:
    print("❌ No test files available. Run the previous cells first.")


# ==============================================================================
# Analyze Already Downloaded Files in /content/drive_dataset/
# ==============================================================================

def scan_downloaded_files():
    """Scan and collect already downloaded audio files"""

    download_dir = "/content/drive_dataset"

    if not os.path.exists(download_dir):
        print("❌ Download directory not found")
        return []

    print(f"🔍 Scanning {download_dir} for audio files...")

    # Valid audio extensions
    VALID_EXTS = {".wav", ".mp3", ".flac", ".m4a", ".ogg"}

    def is_audio(filepath):
        return os.path.splitext(filepath)[1].lower() in VALID_EXTS

    # Collect all audio files
    audio_files = []
    lang_counts = {}

    for root, dirs, files in os.walk(download_dir):
        for file in files:
            if is_audio(file):
                full_path = os.path.join(root, file)
                audio_files.append(full_path)

                # Extract language from folder structure
                path_parts = root.split('/')
                for part in path_parts:
                    if len(part) in [2, 3] and part.isalpha():
                        lang_counts[part] = lang_counts.get(part, 0) + 1
                        break

    print(f"📊 Found {len(audio_files)} audio files:")
    for lang, count in sorted(lang_counts.items()):
        print(f"   {lang}: {count} files")

    # Show sample files
    print(f"\n📝 Sample files:")
    for file_path in audio_files[:5]:
        print(f"   {file_path}")

    return audio_files

# Scan for downloaded files
downloaded_files = scan_downloaded_files()

if not downloaded_files:
    print("❌ No audio files found. Let me help you collect them manually.")

    # Manual file collection if scan fails
    print("\n🔍 Manual file search...")
    import glob

    # Search patterns for common locations
    search_patterns = [
        "/content/drive_dataset/**/*.flac",
        "/content/drive_dataset/**/*.wav",
        "/content/drive_dataset/**/*.mp3",
        "/content/**/*.flac",
        "/content/**/*.wav",
        "/content/**/*.mp3"
    ]

    manual_files = []
    for pattern in search_patterns:
        found = glob.glob(pattern, recursive=True)
        manual_files.extend(found)

    # Remove duplicates
    manual_files = list(set(manual_files))

    print(f"📊 Manual search found: {len(manual_files)} files")
    for file_path in manual_files[:10]:  # Show first 10
        print(f"   {file_path}")

    downloaded_files = manual_files

print(f"\n🎯 Total files ready for analysis: {len(downloaded_files)}")


# ==============================================================================
# Run Independent Analysis on Downloaded Files
# ==============================================================================

def analyze_downloaded_files_independently(audio_files):
    """Run independent model analysis on downloaded files with detailed output"""

    if not audio_files:
        print("❌ No audio files to analyze")
        return None

    print(f"🚀 Starting independent analysis on {len(audio_files)} files...")
    print("=" * 70)

    results = {
        'voxlingua_detailed': [],
        'xlsr_detailed': [],
        'comparison_data': []
    }

    for i, audio_path in enumerate(audio_files, 1):
        print(f"\n[{i}/{len(audio_files)}] 🎵 {os.path.basename(audio_path)}")

        # Extract ground truth from path/filename
        gt_iso = gt_from_filename(audio_path)
        print(f"   📁 Ground Truth: {gt_iso or 'Unknown'}")

        file_analysis = {
            'file': os.path.basename(audio_path),
            'full_path': audio_path,
            'gt_iso': gt_iso or '',
            'voxlingua': {'available': False},
            'xlsr': {'available': False}
        }

        # ==========================================
        # VoxLingua107 Independent Analysis
        # ==========================================
        if voxlingua_model is not None:
            try:
                print(f"   🔬 VoxLingua107 Analysis:")
                out = voxlingua_model.classify_file(audio_path)
                logits, log_conf, pred_idx, labels = out

                # Get real confidence scores (not weighted)
                probs = torch.softmax(logits.squeeze(), dim=0)
                top5_indices = torch.topk(probs, min(5, len(probs))).indices

                vox_predictions = []
                for rank, idx in enumerate(top5_indices, 1):
                    lang_label = labels[idx.item()]
                    confidence = probs[idx.item()].item()

                    # Parse language code
                    if isinstance(lang_label, str):
                        colon_pos = lang_label.find(":")
                        lang_code = lang_label[:colon_pos].strip() if colon_pos != -1 else lang_label.strip()
                    else:
                        lang_code = str(lang_label)

                    # Map to dataset language
                    mapped_lang = map_to_dataset_language(lang_code)

                    vox_predictions.append({
                        'rank': rank,
                        'original': lang_code,
                        'mapped': mapped_lang,
                        'confidence': confidence,
                        'in_dataset': mapped_lang in ALL_SUPPORTED_LANGS
                    })

                    status = "✅" if mapped_lang in ALL_SUPPORTED_LANGS else "❌"
                    print(f"      #{rank}: {lang_code} → {mapped_lang} ({confidence:.4f}) {status}")

                # Store VoxLingua results
                top1 = vox_predictions[0]
                file_analysis['voxlingua'] = {
                    'available': True,
                    'top5_predictions': vox_predictions,
                    'top1_prediction': top1['mapped'],
                    'top1_confidence': top1['confidence'],
                    'correct_top1': gt_iso == top1['mapped'] if gt_iso else None,
                    'correct_in_top5': any(p['mapped'] == gt_iso for p in vox_predictions) if gt_iso else None
                }

                results['voxlingua_detailed'].append({
                    'file': os.path.basename(audio_path),
                    'gt_iso': gt_iso or '',
                    'pred_iso': top1['mapped'],
                    'confidence': top1['confidence'],
                    'correct': gt_iso == top1['mapped'] if gt_iso else None
                })

            except Exception as e:
                print(f"      ❌ VoxLingua107 error: {e}")
                file_analysis['voxlingua'] = {'available': False, 'error': str(e)}

        # ==========================================
        # XLS-R Independent Analysis
        # ==========================================
        if xlsr_lid_model is not None:
            try:
                print(f"   🔬 XLS-R Analysis:")
                out = xlsr_lid_model.classify_file(audio_path)
                out_prob, score, index, text_lab = out

                # Get real confidence scores
                probs = torch.softmax(out_prob.squeeze(), dim=0)
                top5_indices = torch.topk(probs, min(5, len(probs))).indices

                xlsr_predictions = []
                for rank, idx in enumerate(top5_indices, 1):
                    lang_label = text_lab[idx.item()]
                    confidence = probs[idx.item()].item()

                    lang_code = str(lang_label).strip().lower()
                    mapped_lang = map_to_dataset_language(lang_code)

                    xlsr_predictions.append({
                        'rank': rank,
                        'original': lang_code,
                        'mapped': mapped_lang,
                        'confidence': confidence,
                        'in_dataset': mapped_lang in ALL_SUPPORTED_LANGS
                    })

                    status = "✅" if mapped_lang in ALL_SUPPORTED_LANGS else "❌"
                    print(f"      #{rank}: {lang_code} → {mapped_lang} ({confidence:.4f}) {status}")

                # Store XLS-R results
                top1 = xlsr_predictions[0]
                file_analysis['xlsr'] = {
                    'available': True,
                    'top5_predictions': xlsr_predictions,
                    'top1_prediction': top1['mapped'],
                    'top1_confidence': top1['confidence'],
                    'correct_top1': gt_iso == top1['mapped'] if gt_iso else None,
                    'correct_in_top5': any(p['mapped'] == gt_iso for p in xlsr_predictions) if gt_iso else None
                }

                results['xlsr_detailed'].append({
                    'file': os.path.basename(audio_path),
                    'gt_iso': gt_iso or '',
                    'pred_iso': top1['mapped'],
                    'confidence': top1['confidence'],
                    'correct': gt_iso == top1['mapped'] if gt_iso else None
                })

            except Exception as e:
                print(f"      ❌ XLS-R error: {e}")
                file_analysis['xlsr'] = {'available': False, 'error': str(e)}

        results['comparison_data'].append(file_analysis)
        print(f"   ✅ Analysis complete\n")

    return results

# Run the independent analysis
if downloaded_files:
    print("🔬 Running independent model analysis...")
    analysis_results = analyze_downloaded_files_independently(downloaded_files)
else:
    print("❌ No files found for analysis")
    analysis_results = None


# ==============================================================================
# FIXED: Robust VoxLingua107 Analysis with Better Error Handling
# ==============================================================================

def parse_voxlingua_output_robust(out):
    """Robust parsing of VoxLingua107 output with multiple fallback methods"""

    try:
        # Method 1: Standard SpeechBrain output format
        if isinstance(out, (tuple, list)) and len(out) >= 4:
            logits, log_conf, pred_idx, labels = out[:4]

            # Validate components
            if hasattr(logits, 'squeeze') and hasattr(labels, '__getitem__'):
                return logits, log_conf, pred_idx, labels, "standard"

        # Method 2: Alternative format (sometimes returns dict)
        if isinstance(out, dict):
            logits = out.get('predictions', out.get('logits'))
            labels = out.get('labels', out.get('text_lab'))
            log_conf = out.get('log_probabilities', out.get('log_conf'))
            pred_idx = out.get('predicted_ids', out.get('pred_idx'))

            if all(v is not None for v in [logits, labels]):
                return logits, log_conf, pred_idx, labels, "dict"

        # Method 3: Direct tensor output
        if hasattr(out, 'squeeze'):  # Direct logits tensor
            logits = out
            # Create dummy labels based on logits size
            labels = [f"lang_{i}" for i in range(logits.shape[-1])]
            log_conf = torch.log_softmax(logits, dim=-1).max()
            pred_idx = torch.argmax(logits, dim=-1)

            return logits, log_conf, pred_idx, labels, "tensor"

    except Exception as e:
        print(f"        Parse error: {e}")

    return None, None, None, None, "failed"

def analyze_voxlingua_robust(audio_path):
    """Robust VoxLingua107 analysis with multiple parsing methods"""

    if voxlingua_model is None:
        return None

    try:
        # Get raw output from model
        raw_out = voxlingua_model.classify_file(audio_path)

        # Parse with robust method
        logits, log_conf, pred_idx, labels, parse_method = parse_voxlingua_output_robust(raw_out)

        if logits is None:
            print(f"        ❌ Could not parse VoxLingua output format")
            return None

        print(f"        📊 Parse method: {parse_method}")

        # Get predictions based on available data
        if hasattr(logits, 'squeeze'):
            probs = torch.softmax(logits.squeeze(), dim=-1 if len(logits.squeeze().shape) > 0 else 0)

            # Handle different tensor shapes
            if len(probs.shape) == 0:  # Scalar
                top_indices = torch.tensor([0])
                top_probs = probs.unsqueeze(0)
            else:  # Vector
                k = min(5, len(probs))
                top_probs, top_indices = torch.topk(probs, k)
        else:
            print(f"        ❌ Logits not in expected tensor format")
            return None

        predictions = []
        for rank, (idx, prob) in enumerate(zip(top_indices, top_probs), 1):
            idx_val = idx.item() if hasattr(idx, 'item') else int(idx)
            prob_val = prob.item() if hasattr(prob, 'item') else float(prob)

            # Get language label safely
            if idx_val < len(labels):
                lang_label = labels[idx_val]
            else:
                lang_label = f"unknown_{idx_val}"

            # Parse language code
            if isinstance(lang_label, str):
                colon_pos = lang_label.find(":")
                lang_code = lang_label[:colon_pos].strip() if colon_pos != -1 else lang_label.strip()
            else:
                lang_code = str(lang_label)

            # Map to dataset language
            mapped_lang = map_to_dataset_language(lang_code)

            predictions.append({
                'rank': rank,
                'original': lang_code,
                'mapped': mapped_lang,
                'confidence': prob_val,
                'in_dataset': mapped_lang in ALL_SUPPORTED_LANGS
            })

            status = "✅" if mapped_lang in ALL_SUPPORTED_LANGS else "❌"
            print(f"        #{rank}: {lang_code} → {mapped_lang} ({prob_val:.4f}) {status}")

        return predictions

    except Exception as e:
        print(f"        ❌ VoxLingua analysis error: {e}")
        print(f"        ❌ Error type: {type(e).__name__}")
        return None

def analyze_xlsr_robust(audio_path):
    """Robust XLS-R analysis"""

    if xlsr_lid_model is None:
        return None

    try:
        raw_out = xlsr_lid_model.classify_file(audio_path)

        # Handle different XLS-R output formats
        if isinstance(raw_out, (tuple, list)) and len(raw_out) >= 4:
            out_prob, score, index, text_lab = raw_out[:4]
        else:
            print(f"        ❌ XLS-R output format not recognized")
            return None

        # Get top predictions
        if hasattr(out_prob, 'squeeze'):
            probs = torch.softmax(out_prob.squeeze(), dim=-1 if len(out_prob.squeeze().shape) > 0 else 0)

            if len(probs.shape) == 0:  # Scalar
                top_indices = torch.tensor([0])
                top_probs = probs.unsqueeze(0)
            else:  # Vector
                k = min(5, len(probs))
                top_probs, top_indices = torch.topk(probs, k)
        else:
            print(f"        ❌ XLS-R probabilities not in expected format")
            return None

        predictions = []
        for rank, (idx, prob) in enumerate(zip(top_indices, top_probs), 1):
            idx_val = idx.item() if hasattr(idx, 'item') else int(idx)
            prob_val = prob.item() if hasattr(prob, 'item') else float(prob)

            # Get language label
            if idx_val < len(text_lab):
                lang_label = text_lab[idx_val]
            else:
                lang_label = f"unknown_{idx_val}"

            lang_code = str(lang_label).strip().lower()
            mapped_lang = map_to_dataset_language(lang_code)

            predictions.append({
                'rank': rank,
                'original': lang_code,
                'mapped': mapped_lang,
                'confidence': prob_val,
                'in_dataset': mapped_lang in ALL_SUPPORTED_LANGS
            })

            status = "✅" if mapped_lang in ALL_SUPPORTED_LANGS else "❌"
            print(f"        #{rank}: {lang_code} → {mapped_lang} ({prob_val:.4f}) {status}")

        return predictions

    except Exception as e:
        print(f"        ❌ XLS-R analysis error: {e}")
        return None

# ==============================================================================
# UPDATED: Robust Analysis Function
# ==============================================================================

def analyze_downloaded_files_robust(audio_files):
    """Robust analysis with better error handling"""

    if not audio_files:
        print("❌ No audio files to analyze")
        return None

    print(f"🚀 Starting ROBUST analysis on {len(audio_files)} files...")
    print("=" * 70)

    results = {
        'voxlingua_detailed': [],
        'xlsr_detailed': [],
        'comparison_data': []
    }

    for i, audio_path in enumerate(audio_files, 1):
        print(f"\n[{i}/{len(audio_files)}] 🎵 {os.path.basename(audio_path)}")

        # Extract ground truth
        gt_iso = gt_from_filename(audio_path)
        print(f"   📁 Ground Truth: {gt_iso or 'Unknown'}")

        file_analysis = {
            'file': os.path.basename(audio_path),
            'full_path': audio_path,
            'gt_iso': gt_iso or '',
            'voxlingua': {'available': False},
            'xlsr': {'available': False}
        }

        # VoxLingua107 Analysis
        print(f"   🔬 VoxLingua107 Analysis:")
        vox_predictions = analyze_voxlingua_robust(audio_path)

        if vox_predictions:
            top1 = vox_predictions[0]
            file_analysis['voxlingua'] = {
                'available': True,
                'top5_predictions': vox_predictions,
                'top1_prediction': top1['mapped'],
                'top1_confidence': top1['confidence'],
                'correct_top1': gt_iso == top1['mapped'] if gt_iso else None,
                'correct_in_top5': any(p['mapped'] == gt_iso for p in vox_predictions) if gt_iso else None
            }

            results['voxlingua_detailed'].append({
                'file': os.path.basename(audio_path),
                'gt_iso': gt_iso or '',
                'pred_iso': top1['mapped'],
                'confidence': top1['confidence'],
                'correct': gt_iso == top1['mapped'] if gt_iso else None
            })
        else:
            file_analysis['voxlingua'] = {'available': False, 'error': 'Analysis failed'}

        # XLS-R Analysis
        print(f"   🔬 XLS-R Analysis:")
        xlsr_predictions = analyze_xlsr_robust(audio_path)

        if xlsr_predictions:
            top1 = xlsr_predictions[0]
            file_analysis['xlsr'] = {
                'available': True,
                'top5_predictions': xlsr_predictions,
                'top1_prediction': top1['mapped'],
                'top1_confidence': top1['confidence'],
                'correct_top1': gt_iso == top1['mapped'] if gt_iso else None,
                'correct_in_top5': any(p['mapped'] == gt_iso for p in xlsr_predictions) if gt_iso else None
            }

            results['xlsr_detailed'].append({
                'file': os.path.basename(audio_path),
                'gt_iso': gt_iso or '',
                'pred_iso': top1['mapped'],
                'confidence': top1['confidence'],
                'correct': gt_iso == top1['mapped'] if gt_iso else None
            })
        else:
            file_analysis['xlsr'] = {'available': False, 'error': 'Analysis failed'}

        results['comparison_data'].append(file_analysis)
        print(f"   ✅ Analysis complete")

    return results

# Run the robust analysis
if 'downloaded_files' in globals() and downloaded_files:
    print("🔬 Running ROBUST independent model analysis...")
    robust_analysis_results = analyze_downloaded_files_robust(downloaded_files)

    # Generate report
    if robust_analysis_results:
        generate_detailed_performance_report(robust_analysis_results)
        print(f"\n✅ ROBUST ANALYSIS COMPLETE!")
    else:
        print("❌ Robust analysis failed")
else:
    print("❌ No downloaded files found. Please run the file scanning code first.")


# ==============================================================================
# COMPLETE FIX: VoxLingua Label Mapping + Missing Function
# ==============================================================================

# First, let's create a proper VoxLingua language mapping
VOXLINGUA_LANGUAGE_MAP = {
    0: 'ab', 1: 'af', 2: 'ak', 3: 'am', 4: 'ar', 5: 'as', 6: 'az', 7: 'be', 8: 'bg', 9: 'bn',
    10: 'bo', 11: 'br', 12: 'bs', 13: 'ca', 14: 'ce', 15: 'co', 16: 'cs', 17: 'cv', 18: 'cy', 19: 'da',
    20: 'de', 21: 'dv', 22: 'dz', 23: 'ee', 24: 'el', 25: 'en', 26: 'eo', 27: 'es', 28: 'et', 29: 'eu',
    30: 'fa', 31: 'ff', 32: 'fi', 33: 'fo', 34: 'fr', 35: 'fy', 36: 'ga', 37: 'gd', 38: 'gl', 39: 'gn',
    40: 'gu', 41: 'gv', 42: 'ha', 43: 'haw', 44: 'he', 45: 'hi', 46: 'hr', 47: 'ht', 48: 'hu', 49: 'hy',
    50: 'ia', 51: 'id', 52: 'ie', 53: 'ig', 54: 'ii', 55: 'ik', 56: 'io', 57: 'is', 58: 'it', 59: 'iu',
    60: 'ja', 61: 'jv', 62: 'ka', 63: 'kk', 64: 'kl', 65: 'km', 66: 'kn', 67: 'ko', 68: 'ks', 69: 'ku',
    70: 'kw', 71: 'ky', 72: 'la', 73: 'lb', 74: 'lg', 75: 'li', 76: 'ln', 77: 'lo', 78: 'lt', 79: 'lv',
    80: 'mg', 81: 'mi', 82: 'mk', 83: 'ml', 84: 'mn', 85: 'mr', 86: 'ms', 87: 'mt', 88: 'my', 89: 'na',
    90: 'nb', 91: 'nd', 92: 'ne', 93: 'ng', 94: 'nl', 95: 'nn', 96: 'no', 97: 'nv', 98: 'ny', 99: 'oc',
    100: 'of', 101: 'om', 102: 'or', 103: 'os', 104: 'pa', 105: 'pi', 106: 'pl', 107: 'ps'
}

def get_voxlingua_language_by_index(idx):
    """Map VoxLingua index to language code"""
    return VOXLINGUA_LANGUAGE_MAP.get(idx, f'unknown_{idx}')

def analyze_voxlingua_fixed(audio_path):
    """Fixed VoxLingua107 analysis with proper language mapping"""

    if voxlingua_model is None:
        return None

    try:
        raw_out = voxlingua_model.classify_file(audio_path)

        if not isinstance(raw_out, (tuple, list)) or len(raw_out) < 4:
            print(f"        ❌ Unexpected VoxLingua output format")
            return None

        logits, log_conf, pred_idx, labels = raw_out[:4]

        # Get probabilities and top 5
        probs = torch.softmax(logits.squeeze(), dim=-1)
        k = min(5, len(probs))
        top_probs, top_indices = torch.topk(probs, k)

        predictions = []
        for rank, (idx, prob) in enumerate(zip(top_indices, top_probs), 1):
            idx_val = idx.item() if hasattr(idx, 'item') else int(idx)
            prob_val = prob.item() if hasattr(prob, 'item') else float(prob)

            # Method 1: Try to use provided labels
            if idx_val < len(labels) and not str(labels[idx_val]).startswith('unknown'):
                lang_label = labels[idx_val]
                if isinstance(lang_label, str):
                    colon_pos = lang_label.find(":")
                    lang_code = lang_label[:colon_pos].strip() if colon_pos != -1 else lang_label.strip()
                else:
                    lang_code = str(lang_label)
            else:
                # Method 2: Use our language mapping
                lang_code = get_voxlingua_language_by_index(idx_val)

            # Map to dataset language
            mapped_lang = map_to_dataset_language(lang_code)

            predictions.append({
                'rank': rank,
                'original': lang_code,
                'mapped': mapped_lang,
                'confidence': prob_val,
                'in_dataset': mapped_lang in ALL_SUPPORTED_LANGS,
                'index': idx_val
            })

            status = "✅" if mapped_lang in ALL_SUPPORTED_LANGS else "❌"
            print(f"        #{rank}: {lang_code} → {mapped_lang} ({prob_val:.4f}) {status} [idx:{idx_val}]")

        return predictions

    except Exception as e:
        print(f"        ❌ VoxLingua analysis error: {e}")
        return None

def analyze_xlsr_fixed(audio_path):
    """Fixed XLS-R analysis"""

    if xlsr_lid_model is None:
        print(f"        ❌ XLS-R model not loaded")
        return None

    try:
        raw_out = xlsr_lid_model.classify_file(audio_path)

        if not isinstance(raw_out, (tuple, list)) or len(raw_out) < 4:
            print(f"        ❌ Unexpected XLS-R output format")
            return None

        out_prob, score, index, text_lab = raw_out[:4]

        # Get probabilities and top 5
        probs = torch.softmax(out_prob.squeeze(), dim=-1)
        k = min(5, len(probs))
        top_probs, top_indices = torch.topk(probs, k)

        predictions = []
        for rank, (idx, prob) in enumerate(zip(top_indices, top_probs), 1):
            idx_val = idx.item() if hasattr(idx, 'item') else int(idx)
            prob_val = prob.item() if hasattr(prob, 'item') else float(prob)

            # Get language label
            if idx_val < len(text_lab):
                lang_label = text_lab[idx_val]
                lang_code = str(lang_label).strip().lower()
            else:
                lang_code = f"xlsr_unknown_{idx_val}"

            mapped_lang = map_to_dataset_language(lang_code)

            predictions.append({
                'rank': rank,
                'original': lang_code,
                'mapped': mapped_lang,
                'confidence': prob_val,
                'in_dataset': mapped_lang in ALL_SUPPORTED_LANGS
            })

            status = "✅" if mapped_lang in ALL_SUPPORTED_LANGS else "❌"
            print(f"        #{rank}: {lang_code} → {mapped_lang} ({prob_val:.4f}) {status}")

        return predictions

    except Exception as e:
        print(f"        ❌ XLS-R analysis error: {e}")
        return None

def generate_detailed_performance_report(results):
    """Complete performance analysis report function"""

    if not results:
        print("❌ No results to analyze")
        return

    print("\n📊 DETAILED INDEPENDENT MODEL PERFORMANCE REPORT")
    print("=" * 70)

    # VoxLingua107 Performance Analysis
    if results['voxlingua_detailed']:
        vox_df = pd.DataFrame(results['voxlingua_detailed'])
        valid_vox = vox_df[vox_df['gt_iso'] != ''].copy()

        print(f"\n🔬 VOXLINGUA107 PERFORMANCE:")
        print("-" * 40)

        if len(valid_vox) > 0:
            vox_acc = (valid_vox['correct'] == True).mean()
            vox_conf_mean = valid_vox['confidence'].mean()
            vox_conf_std = valid_vox['confidence'].std()

            print(f"Files Analyzed: {len(valid_vox)}")
            print(f"Top-1 Accuracy: {vox_acc:.4f} ({vox_acc*100:.1f}%)")
            print(f"Confidence: {vox_conf_mean:.4f} ± {vox_conf_std:.4f}")

            # Per-language breakdown
            print(f"\nPer-Language Performance:")
            for lang in sorted(valid_vox['gt_iso'].unique()):
                lang_data = valid_vox[valid_vox['gt_iso'] == lang]
                acc = (lang_data['correct'] == True).mean()
                conf_mean = lang_data['confidence'].mean()
                count = len(lang_data)
                print(f"  {lang:>3}: {acc:.3f} ({acc*100:5.1f}%) | Conf: {conf_mean:.3f} | n={count}")
        else:
            print("No valid VoxLingua results")

    # XLS-R Performance Analysis
    if results['xlsr_detailed']:
        xlsr_df = pd.DataFrame(results['xlsr_detailed'])
        valid_xlsr = xlsr_df[xlsr_df['gt_iso'] != ''].copy()

        print(f"\n🔬 XLS-R PERFORMANCE:")
        print("-" * 40)

        if len(valid_xlsr) > 0:
            xlsr_acc = (valid_xlsr['correct'] == True).mean()
            xlsr_conf_mean = valid_xlsr['confidence'].mean()
            xlsr_conf_std = valid_xlsr['confidence'].std()

            print(f"Files Analyzed: {len(valid_xlsr)}")
            print(f"Top-1 Accuracy: {xlsr_acc:.4f} ({xlsr_acc*100:.1f}%)")
            print(f"Confidence: {xlsr_conf_mean:.4f} ± {xlsr_conf_std:.4f}")

            # Per-language breakdown
            print(f"\nPer-Language Performance:")
            for lang in sorted(valid_xlsr['gt_iso'].unique()):
                lang_data = valid_xlsr[valid_xlsr['gt_iso'] == lang]
                acc = (lang_data['correct'] == True).mean()
                conf_mean = lang_data['confidence'].mean()
                count = len(lang_data)
                print(f"  {lang:>3}: {acc:.3f} ({acc*100:5.1f}%) | Conf: {conf_mean:.3f} | n={count}")
        else:
            print("No valid XLS-R results")

    # Model Comparison
    if results['voxlingua_detailed'] and results['xlsr_detailed']:
        print(f"\n⚖️ MODEL COMPARISON:")
        print("-" * 30)

        print(f"VoxLingua107: {vox_acc:.4f} accuracy")
        print(f"XLS-R:        {xlsr_acc:.4f} accuracy")

        # Calculate optimal weights
        total_acc = vox_acc + xlsr_acc
        if total_acc > 0:
            vox_weight = vox_acc / total_acc
            xlsr_weight = xlsr_acc / total_acc

            print(f"\n💡 RECOMMENDED WEIGHTS:")
            print(f"VoxLingua107: {vox_weight:.3f} ({vox_weight*100:.1f}%)")
            print(f"XLS-R:        {xlsr_weight:.3f} ({xlsr_weight*100:.1f}%)")

        # Calculate agreement
        vox_preds = set(vox_df['pred_iso'].tolist())
        xlsr_preds = set(xlsr_df['pred_iso'].tolist())
        common_preds = vox_preds.intersection(xlsr_preds)

        print(f"\nModel Agreement Analysis:")
        print(f"Common predictions: {len(common_preds)}")
        print(f"VoxLingua unique: {len(vox_preds - xlsr_preds)}")
        print(f"XLS-R unique: {len(xlsr_preds - vox_preds)}")

    # Save results
    timestamp = pd.Timestamp.now().strftime("%Y%m%d_%H%M%S")

    if results['voxlingua_detailed']:
        vox_csv = f"voxlingua_fixed_results_{timestamp}.csv"
        pd.DataFrame(results['voxlingua_detailed']).to_csv(vox_csv, index=False)
        print(f"\n💾 VoxLingua results: {vox_csv}")

    if results['xlsr_detailed']:
        xlsr_csv = f"xlsr_fixed_results_{timestamp}.csv"
        pd.DataFrame(results['xlsr_detailed']).to_csv(xlsr_csv, index=False)
        print(f"💾 XLS-R results: {xlsr_csv}")

def run_complete_fixed_analysis(audio_files):
    """Run complete analysis with all fixes"""

    if not audio_files:
        print("❌ No audio files to analyze")
        return None

    print(f"🚀 Starting COMPLETE FIXED analysis on {len(audio_files)} files...")
    print("=" * 70)

    results = {
        'voxlingua_detailed': [],
        'xlsr_detailed': [],
        'comparison_data': []
    }

    for i, audio_path in enumerate(audio_files, 1):
        print(f"\n[{i}/{len(audio_files)}] 🎵 {os.path.basename(audio_path)}")

        # Extract ground truth
        gt_iso = gt_from_filename(audio_path)
        print(f"   📁 Ground Truth: {gt_iso or 'Unknown'}")

        file_analysis = {
            'file': os.path.basename(audio_path),
            'full_path': audio_path,
            'gt_iso': gt_iso or '',
            'voxlingua': {'available': False},
            'xlsr': {'available': False}
        }

        # VoxLingua107 Analysis
        print(f"   🔬 VoxLingua107 Analysis:")
        vox_predictions = analyze_voxlingua_fixed(audio_path)

        if vox_predictions and len(vox_predictions) > 0:
            top1 = vox_predictions[0]
            file_analysis['voxlingua'] = {
                'available': True,
                'top5_predictions': vox_predictions,
                'top1_prediction': top1['mapped'],
                'top1_confidence': top1['confidence'],
                'correct_top1': gt_iso == top1['mapped'] if gt_iso else None,
            }

            results['voxlingua_detailed'].append({
                'file': os.path.basename(audio_path),
                'gt_iso': gt_iso or '',
                'pred_iso': top1['mapped'],
                'confidence': top1['confidence'],
                'correct': gt_iso == top1['mapped'] if gt_iso else None
            })

        # XLS-R Analysis
        print(f"   🔬 XLS-R Analysis:")
        xlsr_predictions = analyze_xlsr_fixed(audio_path)

        if xlsr_predictions and len(xlsr_predictions) > 0:
            top1 = xlsr_predictions[0]
            file_analysis['xlsr'] = {
                'available': True,
                'top5_predictions': xlsr_predictions,
                'top1_prediction': top1['mapped'],
                'top1_confidence': top1['confidence'],
                'correct_top1': gt_iso == top1['mapped'] if gt_iso else None,
            }

            results['xlsr_detailed'].append({
                'file': os.path.basename(audio_path),
                'gt_iso': gt_iso or '',
                'pred_iso': top1['mapped'],
                'confidence': top1['confidence'],
                'correct': gt_iso == top1['mapped'] if gt_iso else None
            })

        results['comparison_data'].append(file_analysis)
        print(f"   ✅ Analysis complete")

    return results

# Run the complete fixed analysis
if 'downloaded_files' in globals() and downloaded_files:
    print("🔬 Running COMPLETE FIXED analysis...")
    final_analysis_results = run_complete_fixed_analysis(downloaded_files)

    if final_analysis_results:
        generate_detailed_performance_report(final_analysis_results)
        print(f"\n✅ COMPLETE FIXED ANALYSIS DONE!")
    else:
        print("❌ Analysis failed")
else:
    print("❌ No downloaded files found")


# ==============================================================================
# COMPREHENSIVE EXCEL ANALYSIS WITH ALL DETAILS
# ==============================================================================

import pandas as pd
import numpy as np
from datetime import datetime
import os

def create_comprehensive_excel_analysis(results, output_filename=None):
    """Create comprehensive Excel analysis with multiple sheets and detailed metrics"""

    if not results:
        print("❌ No results to analyze")
        return None

    # Generate filename if not provided
    if not output_filename:
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        output_filename = f"Language_Detection_Comprehensive_Analysis_{timestamp}.xlsx"

    print(f"📊 Creating comprehensive Excel analysis: {output_filename}")

    # Create Excel writer
    with pd.ExcelWriter(output_filename, engine='openpyxl') as writer:

        # ========================================
        # SHEET 1: EXECUTIVE SUMMARY
        # ========================================
        print("   📋 Creating Executive Summary...")

        summary_data = []

        # Overall statistics
        total_files = len(results['comparison_data'])
        vox_available = sum(1 for item in results['comparison_data'] if item['voxlingua']['available'])
        xlsr_available = sum(1 for item in results['comparison_data'] if item['xlsr']['available'])

        summary_data.extend([
            ['EXECUTIVE SUMMARY', ''],
            ['Analysis Date', datetime.now().strftime("%Y-%m-%d %H:%M:%S")],
            ['Total Files Analyzed', total_files],
            ['VoxLingua107 Available', f"{vox_available} ({vox_available/total_files*100:.1f}%)"],
            ['XLS-R Available', f"{xlsr_available} ({xlsr_available/total_files*100:.1f}%)"],
            ['', ''],
        ])

        # Model performance summary
        if results['voxlingua_detailed']:
            vox_df = pd.DataFrame(results['voxlingua_detailed'])
            valid_vox = vox_df[vox_df['gt_iso'] != ''].copy()
            if len(valid_vox) > 0:
                vox_acc = (valid_vox['correct'] == True).mean()
                vox_conf = valid_vox['confidence'].mean()
                summary_data.extend([
                    ['VOXLINGUA107 PERFORMANCE', ''],
                    ['Accuracy', f"{vox_acc:.4f} ({vox_acc*100:.1f}%)"],
                    ['Average Confidence', f"{vox_conf:.4f}"],
                    ['Files with Valid GT', len(valid_vox)],
                    ['', ''],
                ])

        if results['xlsr_detailed']:
            xlsr_df = pd.DataFrame(results['xlsr_detailed'])
            valid_xlsr = xlsr_df[xlsr_df['gt_iso'] != ''].copy()
            if len(valid_xlsr) > 0:
                xlsr_acc = (valid_xlsr['correct'] == True).mean()
                xlsr_conf = valid_xlsr['confidence'].mean()
                summary_data.extend([
                    ['XLS-R PERFORMANCE', ''],
                    ['Accuracy', f"{xlsr_acc:.4f} ({xlsr_acc*100:.1f}%)"],
                    ['Average Confidence', f"{xlsr_conf:.4f}"],
                    ['Files with Valid GT', len(valid_xlsr)],
                    ['', ''],
                ])

                # Optimal weights calculation
                if results['voxlingua_detailed']:
                    total_acc = vox_acc + xlsr_acc
                    if total_acc > 0:
                        vox_weight = vox_acc / total_acc
                        xlsr_weight = xlsr_acc / total_acc
                        summary_data.extend([
                            ['RECOMMENDED ENSEMBLE WEIGHTS', ''],
                            ['VoxLingua107 Weight', f"{vox_weight:.3f} ({vox_weight*100:.1f}%)"],
                            ['XLS-R Weight', f"{xlsr_weight:.3f} ({xlsr_weight*100:.1f}%)"],
                        ])

        # Create summary dataframe
        summary_df = pd.DataFrame(summary_data, columns=['Metric', 'Value'])
        summary_df.to_excel(writer, sheet_name='Executive_Summary', index=False)

        # ========================================
        # SHEET 2: VOXLINGUA107 DETAILED RESULTS
        # ========================================
        if results['voxlingua_detailed']:
            print("   📋 Creating VoxLingua107 detailed results...")

            vox_detailed_df = pd.DataFrame(results['voxlingua_detailed'])

            # Add additional analysis columns
            vox_detailed_df['accuracy_score'] = vox_detailed_df['correct'].astype(int)
            vox_detailed_df['confidence_category'] = pd.cut(
                vox_detailed_df['confidence'],
                bins=[0, 0.3, 0.6, 0.8, 1.0],
                labels=['Low', 'Medium', 'High', 'Very High']
            )

            # Add language family information
            def get_language_family(lang):
                if lang in INDO_ARYAN_LANGS:
                    return 'Indo-Aryan'
                elif lang in DRAVIDIAN_LANGS:
                    return 'Dravidian'
                elif lang in LOW_RESOURCE_LANGS:
                    return 'Low-Resource'
                else:
                    return 'Other'

            vox_detailed_df['gt_language_family'] = vox_detailed_df['gt_iso'].apply(get_language_family)
            vox_detailed_df['pred_language_family'] = vox_detailed_df['pred_iso'].apply(get_language_family)

            vox_detailed_df.to_excel(writer, sheet_name='VoxLingua107_Results', index=False)

        # ========================================
        # SHEET 3: XLS-R DETAILED RESULTS
        # ========================================
        if results['xlsr_detailed']:
            print("   📋 Creating XLS-R detailed results...")

            xlsr_detailed_df = pd.DataFrame(results['xlsr_detailed'])

            # Add analysis columns
            xlsr_detailed_df['accuracy_score'] = xlsr_detailed_df['correct'].astype(int)
            xlsr_detailed_df['confidence_category'] = pd.cut(
                xlsr_detailed_df['confidence'],
                bins=[0, 0.3, 0.6, 0.8, 1.0],
                labels=['Low', 'Medium', 'High', 'Very High']
            )
            xlsr_detailed_df['gt_language_family'] = xlsr_detailed_df['gt_iso'].apply(get_language_family)
            xlsr_detailed_df['pred_language_family'] = xlsr_detailed_df['pred_iso'].apply(get_language_family)

            xlsr_detailed_df.to_excel(writer, sheet_name='XLSR_Results', index=False)

        # ========================================
        # SHEET 4: PER-LANGUAGE ACCURACY ANALYSIS
        # ========================================
        print("   📋 Creating per-language accuracy analysis...")

        lang_analysis_data = []

        # Get all unique languages from ground truth
        all_gt_langs = set()
        if results['voxlingua_detailed']:
            all_gt_langs.update([r['gt_iso'] for r in results['voxlingua_detailed'] if r['gt_iso']])
        if results['xlsr_detailed']:
            all_gt_langs.update([r['gt_iso'] for r in results['xlsr_detailed'] if r['gt_iso']])

        # Language name mapping
        LANG_NAMES = {
            'ur': 'Urdu', 'pa': 'Punjabi', 'ta': 'Tamil', 'sd': 'Sindhi', 'or': 'Odia',
            'ml': 'Malayalam', 'ne': 'Nepali', 'as': 'Assamese', 'hi': 'Hindi', 'bn': 'Bengali',
            'kok': 'Konkani', 'kn': 'Kannada', 'ks': 'Kashmiri', 'mr': 'Marathi', 'te': 'Telugu',
            'br': 'Bodo', 'doi': 'Dogri', 'sat': 'Santali', 'gu': 'Gujarati', 'mni': 'Manipuri',
            'sa': 'Sanskrit'
        }

        for lang in sorted(all_gt_langs):
            lang_name = LANG_NAMES.get(lang, lang.title())
            lang_family = get_language_family(lang)

            # VoxLingua stats for this language
            vox_stats = {'files': 0, 'correct': 0, 'accuracy': 0, 'avg_confidence': 0}
            if results['voxlingua_detailed']:
                vox_lang_data = [r for r in results['voxlingua_detailed'] if r['gt_iso'] == lang]
                if vox_lang_data:
                    vox_stats['files'] = len(vox_lang_data)
                    vox_stats['correct'] = sum(1 for r in vox_lang_data if r['correct'])
                    vox_stats['accuracy'] = vox_stats['correct'] / vox_stats['files']
                    vox_stats['avg_confidence'] = np.mean([r['confidence'] for r in vox_lang_data])

            # XLS-R stats for this language
            xlsr_stats = {'files': 0, 'correct': 0, 'accuracy': 0, 'avg_confidence': 0}
            if results['xlsr_detailed']:
                xlsr_lang_data = [r for r in results['xlsr_detailed'] if r['gt_iso'] == lang]
                if xlsr_lang_data:
                    xlsr_stats['files'] = len(xlsr_lang_data)
                    xlsr_stats['correct'] = sum(1 for r in xlsr_lang_data if r['correct'])
                    xlsr_stats['accuracy'] = xlsr_stats['correct'] / xlsr_stats['files']
                    xlsr_stats['avg_confidence'] = np.mean([r['confidence'] for r in xlsr_lang_data])

            lang_analysis_data.append({
                'Language_Code': lang,
                'Language_Name': lang_name,
                'Language_Family': lang_family,
                'VoxLingua_Files': vox_stats['files'],
                'VoxLingua_Correct': vox_stats['correct'],
                'VoxLingua_Accuracy': f"{vox_stats['accuracy']:.4f}",
                'VoxLingua_Accuracy_Pct': f"{vox_stats['accuracy']*100:.1f}%",
                'VoxLingua_Avg_Confidence': f"{vox_stats['avg_confidence']:.4f}",
                'XLSR_Files': xlsr_stats['files'],
                'XLSR_Correct': xlsr_stats['correct'],
                'XLSR_Accuracy': f"{xlsr_stats['accuracy']:.4f}",
                'XLSR_Accuracy_Pct': f"{xlsr_stats['accuracy']*100:.1f}%",
                'XLSR_Avg_Confidence': f"{xlsr_stats['avg_confidence']:.4f}",
                'Better_Model': 'VoxLingua' if vox_stats['accuracy'] > xlsr_stats['accuracy'] else 'XLS-R' if xlsr_stats['accuracy'] > vox_stats['accuracy'] else 'Tie'
            })

        lang_analysis_df = pd.DataFrame(lang_analysis_data)
        lang_analysis_df.to_excel(writer, sheet_name='Per_Language_Analysis', index=False)

        # ========================================
        # SHEET 5: CONFUSION MATRIX - VOXLINGUA
        # ========================================
        if results['voxlingua_detailed']:
            print("   📋 Creating VoxLingua confusion matrix...")

            vox_df = pd.DataFrame(results['voxlingua_detailed'])
            valid_vox = vox_df[vox_df['gt_iso'] != ''].copy()

            if len(valid_vox) > 0:
                # Create confusion matrix
                confusion_data = []
                for gt_lang in sorted(valid_vox['gt_iso'].unique()):
                    gt_data = valid_vox[valid_vox['gt_iso'] == gt_lang]
                    row_data = {'Ground_Truth': gt_lang}

                    for pred_lang in sorted(valid_vox['pred_iso'].unique()):
                        count = len(gt_data[gt_data['pred_iso'] == pred_lang])
                        row_data[f'Predicted_{pred_lang}'] = count

                    confusion_data.append(row_data)

                confusion_df = pd.DataFrame(confusion_data).fillna(0)
                confusion_df.to_excel(writer, sheet_name='VoxLingua_Confusion_Matrix', index=False)

        # ========================================
        # SHEET 6: CONFUSION MATRIX - XLS-R
        # ========================================
        if results['xlsr_detailed']:
            print("   📋 Creating XLS-R confusion matrix...")

            xlsr_df = pd.DataFrame(results['xlsr_detailed'])
            valid_xlsr = xlsr_df[xlsr_df['gt_iso'] != ''].copy()

            if len(valid_xlsr) > 0:
                confusion_data = []
                for gt_lang in sorted(valid_xlsr['gt_iso'].unique()):
                    gt_data = valid_xlsr[valid_xlsr['gt_iso'] == gt_lang]
                    row_data = {'Ground_Truth': gt_lang}

                    for pred_lang in sorted(valid_xlsr['pred_iso'].unique()):
                        count = len(gt_data[gt_data['pred_iso'] == pred_lang])
                        row_data[f'Predicted_{pred_lang}'] = count

                    confusion_data.append(row_data)

                confusion_df = pd.DataFrame(confusion_data).fillna(0)
                confusion_df.to_excel(writer, sheet_name='XLSR_Confusion_Matrix', index=False)

        # ========================================
        # SHEET 7: CONFIDENCE ANALYSIS
        # ========================================
        print("   📋 Creating confidence analysis...")

        confidence_analysis = []

        # VoxLingua confidence analysis
        if results['voxlingua_detailed']:
            vox_df = pd.DataFrame(results['voxlingua_detailed'])
            valid_vox = vox_df[vox_df['gt_iso'] != ''].copy()

            if len(valid_vox) > 0:
                for conf_range in [(0, 0.3), (0.3, 0.6), (0.6, 0.8), (0.8, 1.0)]:
                    range_data = valid_vox[
                        (valid_vox['confidence'] >= conf_range[0]) &
                        (valid_vox['confidence'] < conf_range[1])
                    ]

                    if len(range_data) > 0:
                        accuracy = (range_data['correct'] == True).mean()
                        confidence_analysis.append({
                            'Model': 'VoxLingua107',
                            'Confidence_Range': f"{conf_range[0]:.1f}-{conf_range[1]:.1f}",
                            'Files': len(range_data),
                            'Accuracy': f"{accuracy:.4f}",
                            'Accuracy_Pct': f"{accuracy*100:.1f}%",
                            'Avg_Confidence': f"{range_data['confidence'].mean():.4f}"
                        })

        # XLS-R confidence analysis
        if results['xlsr_detailed']:
            xlsr_df = pd.DataFrame(results['xlsr_detailed'])
            valid_xlsr = xlsr_df[xlsr_df['gt_iso'] != ''].copy()

            if len(valid_xlsr) > 0:
                for conf_range in [(0, 0.3), (0.3, 0.6), (0.6, 0.8), (0.8, 1.0)]:
                    range_data = valid_xlsr[
                        (valid_xlsr['confidence'] >= conf_range[0]) &
                        (valid_xlsr['confidence'] < conf_range[1])
                    ]

                    if len(range_data) > 0:
                        accuracy = (range_data['correct'] == True).mean()
                        confidence_analysis.append({
                            'Model': 'XLS-R',
                            'Confidence_Range': f"{conf_range[0]:.1f}-{conf_range[1]:.1f}",
                            'Files': len(range_data),
                            'Accuracy': f"{accuracy:.4f}",
                            'Accuracy_Pct': f"{accuracy*100:.1f}%",
                            'Avg_Confidence': f"{range_data['confidence'].mean():.4f}"
                        })

        confidence_df = pd.DataFrame(confidence_analysis)
        confidence_df.to_excel(writer, sheet_name='Confidence_Analysis', index=False)

        # ========================================
        # SHEET 8: ERROR ANALYSIS
        # ========================================
        print("   📋 Creating error analysis...")

        error_analysis = []

        # VoxLingua errors
        if results['voxlingua_detailed']:
            vox_df = pd.DataFrame(results['voxlingua_detailed'])
            vox_errors = vox_df[vox_df['correct'] == False].copy()

            for _, error in vox_errors.iterrows():
                error_analysis.append({
                    'Model': 'VoxLingua107',
                    'File': error['file'],
                    'Ground_Truth': error['gt_iso'],
                    'Predicted': error['pred_iso'],
                    'Confidence': f"{error['confidence']:.4f}",
                    'GT_Language_Family': get_language_family(error['gt_iso']),
                    'Pred_Language_Family': get_language_family(error['pred_iso']),
                    'Cross_Family_Error': get_language_family(error['gt_iso']) != get_language_family(error['pred_iso'])
                })

        # XLS-R errors
        if results['xlsr_detailed']:
            xlsr_df = pd.DataFrame(results['xlsr_detailed'])
            xlsr_errors = xlsr_df[xlsr_df['correct'] == False].copy()

            for _, error in xlsr_errors.iterrows():
                error_analysis.append({
                    'Model': 'XLS-R',
                    'File': error['file'],
                    'Ground_Truth': error['gt_iso'],
                    'Predicted': error['pred_iso'],
                    'Confidence': f"{error['confidence']:.4f}",
                    'GT_Language_Family': get_language_family(error['gt_iso']),
                    'Pred_Language_Family': get_language_family(error['pred_iso']),
                    'Cross_Family_Error': get_language_family(error['gt_iso']) != get_language_family(error['pred_iso'])
                })

        error_df = pd.DataFrame(error_analysis)
        error_df.to_excel(writer, sheet_name='Error_Analysis', index=False)

        # ========================================
        # SHEET 9: LANGUAGE FAMILY PERFORMANCE
        # ========================================
        print("   📋 Creating language family performance...")

        family_performance = []

        families = ['Indo-Aryan', 'Dravidian', 'Low-Resource', 'Other']

        for family in families:
            # VoxLingua performance for this family
            if results['voxlingua_detailed']:
                vox_df = pd.DataFrame(results['voxlingua_detailed'])
                family_data = vox_df[vox_df['gt_iso'].apply(lambda x: get_language_family(x) == family)]

                if len(family_data) > 0:
                    vox_acc = (family_data['correct'] == True).mean()
                    vox_conf = family_data['confidence'].mean()
                    vox_files = len(family_data)
                else:
                    vox_acc = vox_conf = vox_files = 0
            else:
                vox_acc = vox_conf = vox_files = 0

            # XLS-R performance for this family
            if results['xlsr_detailed']:
                xlsr_df = pd.DataFrame(results['xlsr_detailed'])
                family_data = xlsr_df[xlsr_df['gt_iso'].apply(lambda x: get_language_family(x) == family)]

                if len(family_data) > 0:
                    xlsr_acc = (family_data['correct'] == True).mean()
                    xlsr_conf = family_data['confidence'].mean()
                    xlsr_files = len(family_data)
                else:
                    xlsr_acc = xlsr_conf = xlsr_files = 0
            else:
                xlsr_acc = xlsr_conf = xlsr_files = 0

            family_performance.append({
                'Language_Family': family,
                'VoxLingua_Files': vox_files,
                'VoxLingua_Accuracy': f"{vox_acc:.4f}",
                'VoxLingua_Accuracy_Pct': f"{vox_acc*100:.1f}%",
                'VoxLingua_Avg_Confidence': f"{vox_conf:.4f}",
                'XLSR_Files': xlsr_files,
                'XLSR_Accuracy': f"{xlsr_acc:.4f}",
                'XLSR_Accuracy_Pct': f"{xlsr_acc*100:.1f}%",
                'XLSR_Avg_Confidence': f"{xlsr_conf:.4f}",
                'Better_Model': 'VoxLingua' if vox_acc > xlsr_acc else 'XLS-R' if xlsr_acc > vox_acc else 'Tie'
            })

        family_df = pd.DataFrame(family_performance)
        family_df.to_excel(writer, sheet_name='Language_Family_Performance', index=False)

        # ========================================
        # SHEET 10: TOP-5 PREDICTIONS (SAMPLE)
        # ========================================
        print("   📋 Creating Top-5 predictions sample...")

        top5_sample = []

        # Sample top-5 predictions from comparison data
        sample_files = results['comparison_data'][:20]  # First 20 files as sample

        for file_data in sample_files:
            file_name = file_data['file']
            gt_lang = file_data['gt_iso']

            # VoxLingua Top-5
            if file_data['voxlingua']['available'] and 'top5_predictions' in file_data['voxlingua']:
                for pred in file_data['voxlingua']['top5_predictions']:
                    top5_sample.append({
                        'Model': 'VoxLingua107',
                        'File': file_name,
                        'Ground_Truth': gt_lang,
                        'Rank': pred['rank'],
                        'Predicted_Language': pred['mapped'],
                        'Original_Output': pred['original'],
                        'Confidence': f"{pred['confidence']:.4f}",
                        'In_Dataset': pred['in_dataset'],
                        'Correct': gt_lang == pred['mapped']
                    })

            # XLS-R Top-5
            if file_data['xlsr']['available'] and 'top5_predictions' in file_data['xlsr']:
                for pred in file_data['xlsr']['top5_predictions']:
                    top5_sample.append({
                        'Model': 'XLS-R',
                        'File': file_name,
                        'Ground_Truth': gt_lang,
                        'Rank': pred['rank'],
                        'Predicted_Language': pred['mapped'],
                        'Original_Output': pred['original'],
                        'Confidence': f"{pred['confidence']:.4f}",
                        'In_Dataset': pred['in_dataset'],
                        'Correct': gt_lang == pred['mapped']
                    })

        top5_df = pd.DataFrame(top5_sample)
        top5_df.to_excel(writer, sheet_name='Top5_Predictions_Sample', index=False)

    print(f"✅ Comprehensive Excel analysis created: {output_filename}")

    # Try to download the file
    try:
        from google.colab import files
        print(f"📥 File downloaded successfully!")
    except:
        print(f"📁 File saved locally: {output_filename}")

    return output_filename

# Run the comprehensive Excel analysis
if 'final_analysis_results' in globals() and final_analysis_results:
    excel_filename = create_comprehensive_excel_analysis(
        final_analysis_results,
        "Language_Detection_Comprehensive_Analysis.xlsx"
    )
    print(f"\n🎉 COMPREHENSIVE EXCEL ANALYSIS COMPLETE!")
    print(f"📊 File: {excel_filename}")

    # Print summary of what was created
    print(f"\n📋 Excel Contains 10 Sheets:")
    print(f"   1. Executive_Summary - Key metrics and recommendations")
    print(f"   2. VoxLingua107_Results - Detailed VoxLingua results")
    print(f"   3. XLSR_Results - Detailed XLS-R results")
    print(f"   4. Per_Language_Analysis - Accuracy by language")
    print(f"   5. VoxLingua_Confusion_Matrix - VoxLingua confusion matrix")
    print(f"   6. XLSR_Confusion_Matrix - XLS-R confusion matrix")
    print(f"   7. Confidence_Analysis - Performance by confidence ranges")
    print(f"   8. Error_Analysis - Detailed error breakdown")
    print(f"   9. Language_Family_Performance - Performance by language family")
    print(f"   10. Top5_Predictions_Sample - Sample of top-5 predictions")

else:
    print("❌ No analysis results found. Please run the analysis first.")