""" Advanced Analysis Module Provides additional analysis features for AI papers across the intelligence spectrum """ from typing import Dict, List from collections import Counter import pandas as pd class AdvancedAnalyzer: """Advanced analysis for AGI/ASI papers""" def __init__(self): pass def analyze_keyword_frequency(self, papers: List[Dict]) -> Dict: """ Analyze frequency of AGI/ASI keywords across all papers Args: papers: List of classified papers Returns: Dictionary with keyword frequency statistics """ agi_keywords = [] asi_keywords = [] related_keywords = [] for paper in papers: if 'classification_result' in paper: agi_keywords.extend(paper['classification_result'].get('matched_agi_keywords', [])) asi_keywords.extend(paper['classification_result'].get('matched_asi_keywords', [])) related_keywords.extend(paper['classification_result'].get('matched_related_keywords', [])) return { 'agi_keyword_frequency': dict(Counter(agi_keywords).most_common(20)), 'asi_keyword_frequency': dict(Counter(asi_keywords).most_common(20)), 'related_keyword_frequency': dict(Counter(related_keywords).most_common(20)), 'total_agi_keywords': len(agi_keywords), 'total_asi_keywords': len(asi_keywords), 'total_related_keywords': len(related_keywords) } def analyze_paper_length(self, papers: List[Dict]) -> Dict: """ Analyze paper title and summary length statistics Args: papers: List of papers Returns: Dictionary with length statistics """ title_lengths = [len(p.get('title', '')) for p in papers] summary_lengths = [len(p.get('summary', '')) for p in papers] return { 'title_length': { 'mean': sum(title_lengths) / len(title_lengths) if title_lengths else 0, 'min': min(title_lengths) if title_lengths else 0, 'max': max(title_lengths) if title_lengths else 0, 'median': sorted(title_lengths)[len(title_lengths)//2] if title_lengths else 0 }, 'summary_length': { 'mean': sum(summary_lengths) / len(summary_lengths) if summary_lengths else 0, 'min': min(summary_lengths) if summary_lengths else 0, 'max': max(summary_lengths) if summary_lengths else 0, 'median': sorted(summary_lengths)[len(summary_lengths)//2] if summary_lengths else 0 } } def analyze_score_distribution(self, papers: List[Dict]) -> Dict: """ Analyze distribution of different scores Args: papers: List of classified papers Returns: Dictionary with score distribution statistics """ combined_scores = [] agi_scores = [] asi_scores = [] final_ranks = [] for paper in papers: if 'classification_result' in paper: combined_scores.append(paper['classification_result']['combined_score']) agi_scores.append(paper['classification_result']['agi_score']) asi_scores.append(paper['classification_result']['asi_score']) if 'final_rank' in paper: final_ranks.append(paper['final_rank']) def calculate_stats(scores): if not scores: return {'mean': 0, 'min': 0, 'max': 0, 'std': 0} return { 'mean': sum(scores) / len(scores), 'min': min(scores), 'max': max(scores), 'std': (sum((x - sum(scores)/len(scores))**2 for x in scores) / len(scores))**0.5 } return { 'combined_score_distribution': calculate_stats(combined_scores), 'agi_score_distribution': calculate_stats(agi_scores), 'asi_score_distribution': calculate_stats(asi_scores), 'final_rank_distribution': calculate_stats(final_ranks) } def analyze_classification_patterns(self, papers: List[Dict]) -> Dict: """ Analyze patterns in classifications Args: papers: List of classified papers Returns: Dictionary with classification patterns """ classifications = [] papers_with_both_agi_asi = 0 papers_with_only_agi = 0 papers_with_only_asi = 0 papers_with_neither = 0 for paper in papers: if 'classification_result' in paper: classification = paper['classification_result']['classification'] classifications.append(classification) agi_count = len(paper['classification_result'].get('matched_agi_keywords', [])) asi_count = len(paper['classification_result'].get('matched_asi_keywords', [])) if agi_count > 0 and asi_count > 0: papers_with_both_agi_asi += 1 elif agi_count > 0: papers_with_only_agi += 1 elif asi_count > 0: papers_with_only_asi += 1 else: papers_with_neither += 1 return { 'classification_counts': dict(Counter(classifications)), 'papers_with_both_agi_asi': papers_with_both_agi_asi, 'papers_with_only_agi': papers_with_only_agi, 'papers_with_only_asi': papers_with_only_asi, 'papers_with_neither': papers_with_neither } def generate_analysis_report(self, papers: List[Dict]) -> str: """ Generate comprehensive analysis report Args: papers: List of classified papers Returns: Formatted markdown report """ report = "# 🔬 Advanced Analysis Report\n\n" # Keyword frequency analysis keyword_freq = self.analyze_keyword_frequency(papers) report += "## 📊 Keyword Frequency Analysis\n\n" report += f"**Total AGI Keywords Found**: {keyword_freq['total_agi_keywords']}\n" report += f"**Total ASI Keywords Found**: {keyword_freq['total_asi_keywords']}\n" report += f"**Total Related Keywords Found**: {keyword_freq['total_related_keywords']}\n\n" report += "### Top AGI Keywords\n" for keyword, count in list(keyword_freq['agi_keyword_frequency'].items())[:10]: report += f"- {keyword}: {count} occurrences\n" report += "\n" report += "### Top ASI Keywords\n" for keyword, count in list(keyword_freq['asi_keyword_frequency'].items())[:10]: report += f"- {keyword}: {count} occurrences\n" report += "\n" # Score distribution score_dist = self.analyze_score_distribution(papers) report += "## 📈 Score Distribution\n\n" report += f"**Combined Score**: Mean={score_dist['combined_score_distribution']['mean']:.2f}, " report += f"Std={score_dist['combined_score_distribution']['std']:.2f}\n" report += f"**AGI Score**: Mean={score_dist['agi_score_distribution']['mean']:.2f}\n" report += f"**ASI Score**: Mean={score_dist['asi_score_distribution']['mean']:.2f}\n" report += f"**Final Rank**: Mean={score_dist['final_rank_distribution']['mean']:.2f}\n\n" # Classification patterns patterns = self.analyze_classification_patterns(papers) report += "## 🎯 Classification Patterns\n\n" report += f"**Papers with both AGI and ASI keywords**: {patterns['papers_with_both_agi_asi']}\n" report += f"**Papers with only AGI keywords**: {patterns['papers_with_only_agi']}\n" report += f"**Papers with only ASI keywords**: {patterns['papers_with_only_asi']}\n" report += f"**Papers with neither**: {patterns['papers_with_neither']}\n\n" # Paper length analysis length_analysis = self.analyze_paper_length(papers) report += "## 📝 Paper Length Analysis\n\n" report += f"**Average Title Length**: {length_analysis['title_length']['mean']:.1f} characters\n" report += f"**Average Summary Length**: {length_analysis['summary_length']['mean']:.1f} characters\n\n" return report # Test the analyzer if __name__ == "__main__": analyzer = AdvancedAnalyzer() # Test with sample papers test_papers = [ { 'title': 'Neural Computers: A New Computing Paradigm', 'summary': 'Researchers propose Neural Computers that unify computation and memory, potentially leading to artificial general intelligence.', 'classification_result': { 'classification': 'Core AGI/ASI', 'combined_score': 85.0, 'agi_score': 3, 'asi_score': 0, 'matched_agi_keywords': ['general intelligence', 'AGI'], 'matched_asi_keywords': [], 'matched_related_keywords': ['deep learning'] }, 'final_rank': 75.0 }, { 'title': 'AI Safety Research', 'summary': 'Analysis of existential risks from superintelligent AI systems.', 'classification_result': { 'classification': 'Strongly Related', 'combined_score': 60.0, 'agi_score': 0, 'asi_score': 2, 'matched_agi_keywords': [], 'matched_asi_keywords': ['AI safety', 'existential risk'], 'matched_related_keywords': ['reinforcement learning'] }, 'final_rank': 50.0 } ] print("Testing advanced analyzer...") report = analyzer.generate_analysis_report(test_papers) print(report)