backend/analyze_transcripts.py

"""
Analyze Call Transcript Datasets to Improve Linguistic Features
Datasets: 
1. https://www.kaggle.com/datasets/mealss/call-transcripts-scam-determinations
2. https://www.kaggle.com/datasets/teeconnie/scam-and-non-scam-call-conversation-dataset

This will help improve:
1. SCAM_TERMS vocabulary
2. SCAM_PHRASES contextual patterns
3. LTS scoring weights
"""
import os
import pandas as pd
from collections import Counter
import re
from dotenv import load_dotenv

load_dotenv()

def download_datasets():
    """Download both transcript datasets"""
    datasets = [
        "mealss/call-transcripts-scam-determinations",
        "teeconnie/scam-and-non-scam-call-conversation-dataset"
    ]
    
    download_path = os.path.join(os.path.dirname(__file__), "transcript_data")
    os.makedirs(download_path, exist_ok=True)
    
    for dataset_name in datasets:
        print(f"\n📥 Downloading {dataset_name}...")
        dataset_folder = dataset_name.split('/')[-1]
        dataset_path = os.path.join(download_path, dataset_folder)
        os.makedirs(dataset_path, exist_ok=True)
        os.system(f'kaggle datasets download -d {dataset_name} -p "{dataset_path}" --unzip')
    
    print("\n✅ All downloads complete!")
    return download_path

def analyze_transcripts(data_path):
    """Analyze transcripts from multiple datasets to find scam patterns"""
    
    # Find all CSV files in all subdirectories
    csv_files = []
    for root, dirs, files in os.walk(data_path):
        for file in files:
            if file.endswith('.csv'):
                csv_files.append(os.path.join(root, file))
    
    if not csv_files:
        print("❌ No CSV files found")
        return
    
    print(f"\n📄 Found {len(csv_files)} CSV file(s)")
    
    # Combine all datasets
    all_scam_texts = []
    all_legit_texts = []
    total_scam = 0
    total_legit = 0
    
    for csv_file in csv_files:
        print(f"\n📊 Processing: {os.path.basename(csv_file)}")
        
        try:
            df = pd.read_csv(csv_file)
            print(f"   Rows: {len(df)}")
            print(f"   Columns: {list(df.columns)}")
            
            # Identify transcript and label columns
            transcript_col = None
            label_col = None
            
            for col in df.columns:
                col_lower = col.lower()
                if 'transcript' in col_lower or 'text' in col_lower or 'call' in col_lower or 'conversation' in col_lower or 'message' in col_lower:
                    transcript_col = col
                if 'scam' in col_lower or 'label' in col_lower or 'fraud' in col_lower or 'determination' in col_lower or 'class' in col_lower:
                    label_col = col
            
            if not transcript_col:
                print("   ⚠️ Could not find transcript column, skipping...")
                continue
                
            if not label_col:
                print("   ⚠️ Could not find label column, skipping...")
                continue
            
            print(f"   Using: {transcript_col} (text) + {label_col} (label)")
            
            # Separate scam and legitimate
            scam_mask = df[label_col].astype(str).str.lower().str.contains('scam|fraud|1|yes|true|positive', na=False, regex=True)
            legit_mask = df[label_col].astype(str).str.lower().str.contains('legit|normal|0|no|false|negative', na=False, regex=True)
            
            scam_df = df[sall_scam_texts]
    legit_texts = ' '.join(all_legit_texts)
            
            print(f"   Scam: {len(scam_df)}, Legitimate: {len(legit_df)}")
            
            # Collect texts
            all_scam_texts.extend(scam_df[transcript_col].astype(str).str.lower().tolist())
            all_legit_texts.extend(legit_df[transcript_col].astype(str).str.lower().tolist())
            
            total_scam += len(scam_df)
            total_legit += len(legit_df)
            
        except Exception as e:
            print(f"   ⚠️ Error processing file: {e}")
            continue
    
    print(f"\n\n📈 Combined Dataset Statistics:")
    print(f"   Total Scam calls: {total_scam}")
    print(f"   Total Legitimate calls: {total_legit}")
    print(f"   Datasets processed: {len(csv_files)}")
    
    if total_scam == 0 or total_legit == 0:
        print("\n❌ Not enough data to analyze")
        return
    
    # Analyze scam transcripts for common patterns
    print("\n🔍 Analyzing scam call patterns...")
    
    scam_texts = ' '.join(scam_df[transcript_col].astype(str).str.lower())
    legit_texts = ' '.join(legit_df[transcript_col].astype(str).str.lower())
    
    # Extract common words in scam calls
    scam_words = re.findall(r'\b\w+\b', scam_texts)
    legit_words = re.findall(r'\b\w+\b', legit_texts)
    
    scam_counter = Counter(scam_words)
    legit_counter = Counter(legit_words)
    
    # Find words that appear much more in scam calls
    scam_specific = {}
    for word, count in scam_counter.items():
        if len(word) > 3:  # Skip short words
            scam_freq = count / len(scam_words)
            legit_freq = legit_counter.get(word, 0) / max(len(legit_words), 1)
            
            if scam_freq > legit_freq * 3:  # 3x more common in scams
                scam_specific[word] = (count, scam_freq / (legit_freq + 0.0001))
    
    # Sort by ratio
    sorted_scam_words = sorted(scam_specific.items(), key=lambda x: x[1][1], reverse=True)[:50]
    
    print("\n🎯 Top 50 Scam-Specific Words (not in current SCAM_TERMS):")
    print("   (Words appearing 3x+ more in scam calls)")
    
    # Load current SCAM_TERMS to avoid duplicates
    import processor
    current_terms = set()
    for category in processor.SCAM_TERMS.values():
        current_terms.update(category)
    current_terms = {term.lower() for term in current_terms}
    
    new_terms = []
    for word, (count, ratio) in sorted_scam_words:
        if word not in current_terms:
            new_terms.append((word, count, ratio))
    
    print("\n   Word (Count, Scam/Legit Ratio)")
    print("   " + "-" * 40)
    for word, count, ratio in new_terms[:30]:
        print(f"   {word:20s} ({count:4d}, {ratio:.1f}x)")
    
    # Extract common phrases (2-4 words)
    print("\n\n🔍 Analyzing common scam phrases...")
    
    scam_phrases = []
    for text in all_scam_texts:
        text_lower = text.lower()
        # Extract 2-4 word phrases
        words = text_lower.split()
        for i in range(len(words) - 1):
            for length in [2, 3, 4]:
                if i + length <= len(words):
                    phrase = ' '.join(words[i:i+length])
                    if len(phrase) > 10:  # Skip very short phrases
                        scam_phrases.append(phrase)
    
    phrase_counter = Counter(scam_phrases)
    
    # Load current SCAM_PHRASES
    current_phrases = {phrase.lower() for phrase in processor.SCAM_PHRASES}
    
    print("\n🎯 Top 30 New Scam Phrases (not in current SCAM_PHRASES):")
    print("   " + "-" * 50)
    
    new_phrases = []
    for phrase, count in phrase_counter.most_common(100):
        if phrase not in current_phrases and count > 5:  # Appears 5+ times
            new_phrases.append((phrase, count))
    
    for phrase, count in new_phrases[:30]:
        print(f"   [{count:3d}x] {phrase}")
    
    # Save suggestions
    output_file = os.path.join(os.path.dirname(__file__), "linguistic_improvements.txt")
    
    with open(output_file, 'w') as f:
        f.write("VocalGuard Linguistic Feature Improvements\n")
        f.write("=" * 60 + "\n\n")
        
        f.write("NEW SCAM TERMS TO ADD:\n")
        f.write("-" * 60 + "\n")
        for word, count, ratio in new_terms[:50]:
            f.write(f"{word} (appears {count}x, {ratio:.1f}x more in scams)\n")
        
        f.write("\n\nNEW SCAM PHRASES TO ADD:\n")
        f.write("-" * 60 + "\n")
        for phrase, count in new_phrases[:50]:
            f.write(f"[{count}x] {phrase}\n")
        
        f.write("\n\nRECOMMENDATIONS:\n")
        f.write("-" * 60 + "\n")
        f.write("1. Add the top 20-30 new terms to SCAM_TERMS in processor.py\n")
        f.write("2. Add the top 10-15 new phrases to SCAM_PHRASES\n")
        f.write("3. Test on validation set to ensure no false positives\n")
        f.write("4. Re-evaluate LTS scoring weights if needed\n")
    
    print(f"\n\n✅ Analysis complete!")
    print(f"   Results saved to: {output_file}")
    print(f"\n💡 Next Steps:")
    print(f"   1. Review {output_file}")
    print(f"   2. Add relevant terms/phrases to processor.py")
    print(f"   3. Test improved model")
    
    return new_terms, new_phrases

def main():
    print("=" * 60)
    print("🤖 VocalGuard - Multi-Dataset Transcript Analysis")
    print("Improving Linguistic Threat Detection")
    print("=" * 60)
    
    # Check if data already exists
    data_path = os.path.join(os.path.dirname(__file__), "transcript_data")
    
    if not os.path.exists(data_path) or len(os.listdir(data_path)) < 2:
        print("\n📥 Downloading both transcript datasets...")
        download_datasets()
    else:
        print(f"✅ Transcript data exists: {data_path}")
    
    # Analyze
    analyze_transcripts(data_path)

if __name__ == "__main__":
    main()