test/test_frequency_flexible.py

\n#!/usr/bin/env python3\n\"\"\"\nTest script for the enhanced FrequencyAnalyzer with flexible column mapping.\nThis demonstrates the new functionality with sample data.\n\"\"\"\n\nimport pandas as pd\nimport numpy as np\nfrom io import StringIO\nimport sys\nimport os\n\n# Add the text_analyzer to path\nsys.path.append('text_analyzer')\n\nfrom frequency_analyzer import FrequencyAnalyzer\n\ndef create_sample_data():\n    \"\"\"Create sample frequency data in the new format.\"\"\"\n    sample_data = \"\"\"rank\tlForm\tlemma\tpos\tsubLemma\twType\tfrequency\tpmw\tPB_frequency\tPB_pmw\tPM_frequency\tPM_pmw\tcore_frequency\tcore_pmw\n1\tノ\tの\t助詞-格助詞\t\t和\t5061558\t48383.9\t1473494\t51791.5\t208748\t47179.3\t1398950\t51737.2\n2\tニ\tに\t助詞-格助詞\t\t和\t3576558\t34188.7\t1036653\t36437.1\t140178\t31681.7\t985766\t36456.5\n3\tテ\tて\t助詞-接続助詞\t\t和\t3493117\t33391.0\t948430\t33336.1\t124241\t28079.8\t902379\t33372.6\n4\tハ\tは\t助詞-係助詞\t\t和\t3289932\t31448.8\t945084\t33218.5\t129378\t29240.8\t899776\t33276.3\n5\tガ\tが\t助詞-格助詞\t\t和\t2518164\t24070.6\t743621\t26131.8\t103456\t23390.2\t707331\t26139.9\"\"\"\n    return sample_data\n\ndef test_file_format_detection():\n    \"\"\"Test file format detection functionality.\"\"\"\n    print(\"=== Testing File Format Detection ===\")\n    \n    analyzer = FrequencyAnalyzer(file_size_limit_mb=300)\n    sample_data = create_sample_data()\n    \n    format_info = analyzer.detect_file_format(sample_data)\n    print(f\"Detected separator: '{format_info['separator']}'\")\n    print(f\"Has header: {format_info['has_header']}\")\n    print(f\"Estimated columns: {format_info['estimated_columns']}\")\n    print(f\"Sample lines: {format_info['sample_lines'][:2]}\")\n    print()\n\ndef test_column_detection():\n    \"\"\"Test column detection and categorization.\"\"\"\n    print(\"=== Testing Column Detection ===\")\n    \n    analyzer = FrequencyAnalyzer()\n    sample_data = create_sample_data()\n    \n    # Read sample data for column detection\n    df = pd.read_csv(StringIO(sample_data), sep='\\t')\n    detected_cols = analyzer.detect_columns(df)\n    \n    print(\"Detected columns:\")\n    for category, columns in detected_cols.items():\n        print(f\"  {category}: {columns}\")\n    print()\n\ndef test_flexible_loading():\n    \"\"\"Test flexible data loading with column configuration.\"\"\"\n    print(\"=== Testing Flexible Data Loading ===\")\n    \n    analyzer = FrequencyAnalyzer()\n    sample_data = create_sample_data()\n    \n    # Test with different column configurations\n    configs = [\n        {\n            'word_column': 'lForm',\n            'frequency_column': 'frequency',\n            'pos_column': 'pos'\n        },\n        {\n            'word_column': 'lemma',\n            'frequency_column': 'pmw',\n            'pos_column': 'pos'\n        },\n        {\n            'word_column': 'lForm',\n            'frequency_column': 'PB_frequency'\n        }\n    ]\n    \n    for i, config in enumerate(configs, 1):\n        print(f\"Configuration {i}: {config}\")\n        try:\n            df = analyzer.load_frequency_data(sample_data, config)\n            print(f\"  ✓ Successfully loaded {len(df)} entries\")\n            print(f\"  ✓ Available frequency columns: {analyzer.get_available_frequency_columns()}\")\n            print(f\"  ✓ Available word columns: {analyzer.get_available_word_columns()}\")\n        except Exception as e:\n            print(f\"  ✗ Error: {e}\")\n        print()\n\ndef test_multi_frequency_analysis():\n    \"\"\"Test multi-frequency analysis functionality.\"\"\"\n    print(\"=== Testing Multi-Frequency Analysis ===\")\n    \n    analyzer = FrequencyAnalyzer()\n    sample_data = create_sample_data()\n    \n    config = {\n        'word_column': 'lForm',\n        'frequency_column': 'frequency',\n        'pos_column': 'pos'\n    }\n    \n    df = analyzer.load_frequency_data(sample_data, config)\n    \n    # Test analysis with multiple frequency columns\n    freq_columns = ['frequency', 'pmw', 'PB_frequency']\n    \n    try:\n        results = analyzer.create_multi_frequency_analysis(freq_columns, bin_size=2)\n        \n        print(f\"Multi-frequency analysis results:\")\n        for col, result in results.items():\n            print(f\"  {col}: {len(result['group_labels'])} groups\")\n            print(f\"    Sample frequencies: {result['avg_frequencies'][:3]}\")\n        \n    except Exception as e:\n        print(f\"Error in multi-frequency analysis: {e}\")\n    print()\n\ndef test_rank_based_visualization():\n    \"\"\"Test rank-based visualization with flexible columns.\"\"\"\n    print(\"=== Testing Rank-Based Visualization ===\")\n    \n    analyzer = FrequencyAnalyzer()\n    sample_data = create_sample_data()\n    \n    config = {\n        'word_column': 'lForm',\n        'frequency_column': 'frequency'\n    }\n    \n    df = analyzer.load_frequency_data(sample_data, config)\n    \n    try:\n        # Test with different frequency columns\n        for col in ['frequency', 'pmw', 'PB_frequency']:\n            result = analyzer.create_rank_based_visualization_flexible(\n                column=col, \n                bin_size=2, \n                log_transform=False\n            )\n            \n            print(f\"Analysis for column '{col}':\")\n            print(f\"  Groups: {len(result['group_labels'])}\")\n            print(f\"  Sample words: {[w['word'] for w in result['sample_words'].get(0, [])]}\")\n            print(f\"  Avg frequencies: {result['avg_frequencies']}\")\n            \n    except Exception as e:\n        print(f\"Error in rank-based visualization: {e}\")\n    print()\n\ndef test_backward_compatibility():\n    \"\"\"Test backward compatibility with legacy interface.\"\"\"\n    print(\"=== Testing Backward Compatibility ===\")\n    \n    analyzer = FrequencyAnalyzer()\n    sample_data = create_sample_data()\n    \n    # Test with flexible loading first\n    config = {\n        'word_column': 'lForm',\n        'frequency_column': 'frequency'\n    }\n    \n    df = analyzer.load_frequency_data(sample_data, config)\n    \n    # Then test legacy methods\n    try:\n        legacy_cols = analyzer.get_available_columns()\n        print(f\"Legacy available columns: {legacy_cols}\")\n        \n        if legacy_cols:\n            stats = analyzer.calculate_statistics(legacy_cols[0])\n            print(f\"Statistics for {legacy_cols[0]}: mean={stats['mean']:.1f}, count={stats['count']}\")\n            \n            top_words = analyzer.get_top_words(legacy_cols[0], n=3)\n            print(f\"Top 3 words: {[w['word'] for w in top_words]}\")\n        \n    except Exception as e:\n        print(f\"Error in backward compatibility test: {e}\")\n    print()\n\nif __name__ == \"__main__\":\n    print(\"Testing Enhanced FrequencyAnalyzer with Flexible Column Mapping\")\n    print(\"=\" * 60)\n    \n    test_file_format_detection()\n    test_column_detection()\n    test_flexible_loading()\n    test_multi_frequency_analysis()\n    test_rank_based_visualization()\n    test_backward_compatibility()\n    \n    print(\"All tests completed!\")\n