Spaces:
Building
Building
File size: 5,184 Bytes
e7279e4 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 |
#!/usr/bin/env python3
"""
Test script to verify the fix for bigram/trigram plot sample words
"""
import sys
import os
# Add the project root to the path
sys.path.insert(0, os.getcwd())
from web_app.config_manager import ConfigManager
from text_analyzer.lexical_sophistication import LexicalSophisticationAnalyzer
def test_plot_fix():
print("=== Testing Plot Fix ===")
# Load config and create reference lists
config = ConfigManager.load_reference_config()
english_config = config.get('english', {})
reference_lists = {}
# Load a unigram, bigram, and trigram reference
unigram_config = english_config['unigrams']['COCA_spoken_frequency_token']
bigram_config = english_config['bigrams']['COCA_spoken_bigram_frequency_token']
trigram_config = english_config['trigrams']['COCA_trigram_frequency_token']
reference_lists['COCA_spoken_frequency_token'] = ConfigManager.load_reference_list_data(unigram_config)
reference_lists['COCA_spoken_bigram_frequency_token'] = ConfigManager.load_reference_list_data(bigram_config)
reference_lists['COCA_trigram_frequency_token'] = ConfigManager.load_reference_list_data(trigram_config)
# Create analyzer and analyze text
analyzer = LexicalSophisticationAnalyzer(language='en', model_size='md')
analyzer.load_reference_lists(reference_lists)
test_text = "The cat sat on the mat. The dog ran quickly."
results = analyzer.analyze_text(test_text, list(reference_lists.keys()), apply_log=False)
print("\n=== Testing Column Matching with Fixed Algorithm ===")
# Test the fixed algorithm for bigrams
for key in results['raw_scores'].keys():
if '_bigram_' in key:
print(f"\nTesting bigram key: {key}")
# Use the new algorithm: remove '_bigram' from the key
index_measure_col = key.replace('_bigram', '')
print(f" Fixed algorithm expects column: '{index_measure_col}'")
# Check if this column exists in bigram_details
if results['bigram_details']:
sample_bigram = results['bigram_details'][0]
if index_measure_col in sample_bigram:
print(f" ✅ Column found in bigram_details")
# Test if we can build word_score_map successfully
word_score_map = {}
for bigram_detail in results['bigram_details']:
if index_measure_col in bigram_detail and bigram_detail[index_measure_col] is not None:
bigram_text = bigram_detail.get('bigram', '')
word_score_map[bigram_text] = bigram_detail[index_measure_col]
print(f" ✅ Successfully built word_score_map with {len(word_score_map)} entries")
if word_score_map:
sample_entries = list(word_score_map.items())[:3]
print(f" Sample entries: {sample_entries}")
else:
print(f" ❌ Column still NOT found in bigram_details")
# Test the fixed algorithm for trigrams
for key in results['raw_scores'].keys():
if '_trigram_' in key:
print(f"\nTesting trigram key: {key}")
# Use the new algorithm: remove '_trigram' from the key
index_measure_col = key.replace('_trigram', '')
print(f" Fixed algorithm expects column: '{index_measure_col}'")
# Check if this column exists in trigram_details
if results['trigram_details']:
sample_trigram = results['trigram_details'][0]
if index_measure_col in sample_trigram:
print(f" ✅ Column found in trigram_details")
# Test if we can build word_score_map successfully
word_score_map = {}
for trigram_detail in results['trigram_details']:
if index_measure_col in trigram_detail and trigram_detail[index_measure_col] is not None:
trigram_text = trigram_detail.get('trigram', '')
word_score_map[trigram_text] = trigram_detail[index_measure_col]
print(f" ✅ Successfully built word_score_map with {len(word_score_map)} entries")
if word_score_map:
sample_entries = list(word_score_map.items())[:3]
print(f" Sample entries: {sample_entries}")
else:
print(f" ❌ Column still NOT found in trigram_details")
print("\n=== Fix Verification Complete ===")
if any('_bigram_' in key for key in results['raw_scores'].keys()) and any('_trigram_' in key for key in results['raw_scores'].keys()):
print("✅ Fix appears to be working correctly!")
print("Sample words should now appear in bigram and trigram plots.")
else:
print("❌ No bigram/trigram results found to test")
if __name__ == "__main__":
test_plot_fix()
|