more sophistication indice selection

clear_session.py

@@ -0,0 +1,34 @@
+
+import streamlit as st
+from web_app.session_manager import SessionManager
+
+st.title("🔄 Session State Reset")
+
+st.write("## Current Session State")
+st.write("Reference lists currently loaded:")
+
+if hasattr(st.session_state, 'reference_lists') and st.session_state.reference_lists:
+    for name, data in st.session_state.reference_lists.items():
+        st.write(f"- **{name}**")
+else:
+    st.write("No reference lists loaded")
+
+st.write("---")
+
+if st.button("🗑️ Clear All Session State", type="primary"):
+    # Clear all session state
+    for key in list(st.session_state.keys()):
+        del st.session_state[key]
+    
+    # Reinitialize
+    SessionManager.initialize_session_state()
+    
+    st.success("✅ Session state cleared! Please refresh the page.")
+    st.balloons()
+
+st.write("### Instructions:")
+st.write("1. Click 'Clear All Session State' above")
+st.write("2. Refresh your browser page")
+st.write("3. Go back to the Lexical Sophistication tool")
+st.write("4. Re-select your reference lists")
+st.write("5. You should now see smart defaults!")

config/reference_lists.yaml

@@ -1,17 +1,12 @@
-# Configuration for Default Reference Lists
-# Add new reference lists here and they'll automatically appear in the UI
-# Structure: language -> type -> list_name -> configuration
-
 english:
   unigrams:
-    COCA_spoken_frequency:
-      display_name: "COCA Spoken Frequency"
-      description: "Frequency and range data from COCA spoken subcorpus"
-      files:
-        token: "resources/reference_lists/en/COCA_spoken_unigram_list.csv"
-        lemma: "resources/reference_lists/en/COCA_spoken_unigram_list.csv"  # Using same file for now
-      format: "tsv"
-      columns:
+    COCA_magazine_frequency_token:
+      display_name: COCA Magazine Frequency (Token)
+      description: Frequency and range data from COCA magazine subcorpus - token-based
+        analysis
+      file: resources/reference_lists/en/COCA_magazine_unigram_list.csv
+      format: tsv
+      columns: &id001
         word: 0
         frequency: 1
         normalized_freq: 2
@@ -19,59 +14,269 @@ english:
         dispersion: 4
       has_header: false
       enabled: true
+      analysis_type: token
+      log_transformable:
+      - frequency
+      - normalized_freq
+      selectable_measures:
+      - frequency
+      - normalized_freq
+      - range
+      - dispersion
+      default_measures:
+      - frequency
+      - normalized_freq
+      - range
+      default_log_transforms:
+      - frequency
+      - normalized_freq
+      measure_classifications:
+        frequency: frequency
+        normalized_freq: frequency
+        range: range
+        dispersion: range
 
-    concreteness_ratings:
-      display_name: "Concreteness Ratings"
-      description: "Concreteness ratings for English words (1-5 scale)"
-      files:
-        token: "resources/reference_lists/en/concreteness_token.csv"
-        lemma: "resources/reference_lists/en/concreteness_lemma.csv"
-      format: "tsv"
-      columns:
+    COCA_magazine_frequency_lemma:
+      display_name: COCA Magazine Frequency (Lemma)
+      description: Frequency and range data from COCA magazine subcorpus - lemma-based
+        analysis
+      file: resources/reference_lists/en/COCA_magazine_unigram_list.csv
+      format: tsv
+      columns: *id001
+      has_header: false
+      enabled: true
+      analysis_type: lemma
+      log_transformable:
+      - frequency
+      - normalized_freq
+      selectable_measures:
+      - frequency
+      - normalized_freq
+      - range
+      - dispersion
+      default_measures:
+      - frequency
+      - normalized_freq
+      - range
+      default_log_transforms:
+      - frequency
+      - normalized_freq
+      measure_classifications:
+        frequency: frequency
+        normalized_freq: frequency
+        range: range
+        dispersion: range
+
+    concreteness_ratings_token:
+      display_name: Concreteness Ratings (Token)
+      description: Concreteness ratings for English words (1-5 scale) - token-based
+        analysis
+      file: resources/reference_lists/en/Concreteness_Brysbaert.txt
+      format: tsv
+      columns: &id002
         word: 0
         concreteness: 1
       has_header: true
-      header_prefix: "#"
-      enabled: false  # Disabled until files exist
-
-    academic_words:
-      display_name: "Academic Word List"
-      description: "Common academic vocabulary for research writing"
-      files:
-        token: "resources/reference_lists/en/academic_words_token.csv"
-        lemma: "resources/reference_lists/en/academic_words_lemma.csv"
-      format: "csv"
-      columns:
+      enabled: true
+      analysis_type: token
+      log_transformable: []
+      selectable_measures:
+      - concreteness
+      default_measures:
+      - concreteness
+      default_log_transforms: []
+      measure_classifications:
+        concreteness: psycholinguistic
+      header_prefix: '#'
+      
+    concreteness_ratings_lemma:
+      display_name: Concreteness Ratings (Lemma)
+      description: Concreteness ratings for English words (1-5 scale) - lemma-based
+        analysis
+      file: resources/reference_lists/en/Concreteness_Brysbaert.txt
+      format: tsv
+      columns: *id002
+      has_header: true
+      enabled: true
+      analysis_type: lemma
+      log_transformable: []
+      selectable_measures:
+      - concreteness
+      default_measures:
+      - concreteness
+      default_log_transforms: []
+      measure_classifications:
+        concreteness: psycholinguistic
+      header_prefix: '#'
+    academic_words_token:
+      display_name: Academic Word List (Token)
+      description: Common academic vocabulary for research writing - token-based analysis
+      file: resources/reference_lists/en/academic_words_token.csv
+      format: csv
+      columns: &id003
         word: 0
         frequency: 1
       has_header: true
-      enabled: false  # Disabled until files exist
-
-  bigrams:
-    COCA_bigram_frequency:
-      display_name: "COCA Bigram Frequency"
-      description: "Bigram frequencies and range data"
-      files:
-        token: "resources/reference_lists/en/COCA_spoken_bigram_list.csv"
-        lemma: "resources/reference_lists/en/COCA_spoken_bigram_list.csv"
-      format: "tsv"
+      enabled: false
+      analysis_type: token
+      log_transformable:
+      - frequency
+      selectable_measures:
+      - frequency
+      default_measures:
+      - frequency
+      default_log_transforms:
+      - frequency
+      measure_classifications:
+        frequency: frequency
+    academic_words_lemma:
+      display_name: Academic Word List (Lemma)
+      description: Common academic vocabulary for research writing - lemma-based analysis
+      file: resources/reference_lists/en/academic_words_lemma.csv
+      format: csv
+      columns: *id003
+      has_header: true
+      enabled: false
+      analysis_type: lemma
+      log_transformable:
+      - frequency
+      selectable_measures:
+      - frequency
+      default_measures:
+      - frequency
+      default_log_transforms:
+      - frequency
+      measure_classifications:
+        frequency: frequency
+    COCA_spoken_frequency_token:
+      display_name: COCA Spoken Frequency (Token)
+      description: Frequency and range data from COCA spoken subcorpus - token-based
+        analysis
+      file: resources/reference_lists/en/COCA_spoken_unigram_list.csv
+      format: tsv
+      columns:
+        word: 0
+        frequency: 1
+        normalized_freq: 2
+        range: 3
+        dispersion: 4
+      has_header: false
+      enabled: true
+      analysis_type: token
+      log_transformable:
+      - frequency
+      - normalized_freq
+      selectable_measures:
+      - frequency
+      - normalized_freq
+      - range
+      - dispersion
+      default_measures:
+      - frequency
+      - normalized_freq
+      default_log_transforms:
+      - frequency
+      - normalized_freq
+    COCA_spoken_frequency_lemma:
+      display_name: COCA Spoken Frequency (Lemma)
+      description: Frequency and range data from COCA spoken subcorpus - lemma-based
+        analysis
+      file: resources/reference_lists/en/COCA_spoken_unigram_list.csv
+      format: tsv
       columns:
+        word: 0
+        frequency: 1
+        normalized_freq: 2
+        range: 3
+        dispersion: 4
+      has_header: false
+      enabled: true
+      analysis_type: lemma
+      log_transformable:
+      - frequency
+      - normalized_freq
+      selectable_measures:
+      - frequency
+      - normalized_freq
+      - range
+      - dispersion
+      default_measures:
+      - frequency
+      - normalized_freq
+      default_log_transforms:
+      - frequency
+      - normalized_freq
+  bigrams:
+    COCA_spoken_bigram_frequency_token:
+      display_name: COCA Spoken Bigram Frequency (Token)
+      description: Bigram frequencies and range data - token-based analysis
+      file: resources/reference_lists/en/COCA_spoken_bigram_list.csv
+      format: tsv
+      columns: &id004
         bigram: 0
         frequency: 1
         normalized_freq: 2
         documents: 3
         range: 4
       has_header: false
-      enabled: true  # Disabled until files exist
-
-    COCA_bigram_association:
-      display_name: "COCA Bigram Associations"
-      description: "Bigram association measures (MI, T-score, Delta P)"
-      files:
-        token: "resources/reference_lists/en/COCA_bigram_association_token.csv"
-        lemma: "resources/reference_lists/en/COCA_bigram_association_lemma.csv"
-      format: "csv"
-      columns:
+      enabled: true
+      analysis_type: token
+      log_transformable:
+      - frequency
+      - normalized_freq
+      selectable_measures:
+      - frequency
+      - normalized_freq
+      - documents
+      - range
+      default_measures:
+      - frequency
+      - normalized_freq
+      - range
+      default_log_transforms:
+      - frequency
+      - normalized_freq
+      measure_classifications:
+        frequency: frequency
+        normalized_freq: frequency
+        documents: range
+        range: range
+    COCA_spoken_bigram_frequency_lemma:
+      display_name: COCA Spoken Bigram Frequency (Lemma)
+      description: Bigram frequencies and range data - lemma-based analysis
+      file: resources/reference_lists/en/COCA_spoken_bigram_list.csv
+      format: tsv
+      columns: *id004
+      has_header: false
+      enabled: true
+      analysis_type: lemma
+      log_transformable:
+      - frequency
+      - normalized_freq
+      selectable_measures:
+      - frequency
+      - normalized_freq
+      - documents
+      - range
+      default_measures:
+      - frequency
+      - normalized_freq
+      - range
+      default_log_transforms:
+      - frequency
+      - normalized_freq
+      measure_classifications:
+        frequency: frequency
+        normalized_freq: frequency
+        documents: range
+        range: range
+    COCA_spoken_bigram_association_token:
+      display_name: COCA Spoken Bigram Associations (Token)
+      description: Bigram association measures (MI, T-score, Delta P) - token-based
+        analysis
+      file: resources/reference_lists/en/spoken_bi_contingency.csv
+      format: csv
+      columns: &id005
         bigram: 0
         frequency: 1
         mi_score: 2
@@ -80,17 +285,199 @@ english:
         delta_p: 5
         ap_collex: 6
       has_header: true
-      enabled: false  # Disabled until files exist
-
+      enabled: true
+      analysis_type: token
+      log_transformable:
+      - frequency
+      selectable_measures:
+      - frequency
+      - mi_score
+      - mi_2_score
+      - t_score
+      - delta_p
+      - ap_collex
+      default_measures:
+      - frequency
+      - t_score
+      default_log_transforms:
+      - frequency
+      measure_classifications:
+        frequency: frequency
+        mi_score: association
+        mi_2_score: association
+        t_score: association
+        delta_p: association
+        ap_collex: association
+    COCA_spoken_bigram_association_lemma:
+      display_name: COCA Spoken Bigram Associations (Lemma)
+      description: Bigram association measures (MI, T-score, Delta P) - lemma-based
+        analysis
+      file: resources/reference_lists/en/spoken_bigram_lemma_contingency.csv
+      format: csv
+      columns: *id005
+      has_header: true
+      enabled: true
+      analysis_type: lemma
+      log_transformable:
+      - frequency
+      selectable_measures:
+      - frequency
+      - mi_score
+      - mi_2_score
+      - t_score
+      - delta_p
+      - ap_collex
+      default_measures:
+      - frequency
+      - t_score
+      default_log_transforms:
+      - frequency
+      measure_classifications:
+        frequency: frequency
+        mi_score: association
+        mi_2_score: association
+        t_score: association
+        delta_p: association
+        ap_collex: association
+    COCA_magazine_bigram_frequency_token:
+      display_name: COCA Magazine Bigram Frequency (Token)
+      description: Bigram frequencies and range data in Magazine - token-based analysis
+      file: resources/reference_lists/en/COCA_magazine_bigram_list.csv
+      format: tsv
+      columns: &id006
+        bigram: 0
+        frequency: 1
+        normalized_freq: 2
+        documents: 3
+        range: 4
+      has_header: false
+      enabled: true
+      analysis_type: token
+      log_transformable:
+      - frequency
+      - normalized_freq
+      selectable_measures:
+      - frequency
+      - normalized_freq
+      - documents
+      - range
+      default_measures:
+      - frequency
+      - normalized_freq
+      - range
+      default_log_transforms:
+      - frequency
+      - normalized_freq
+      measure_classifications:
+        frequency: frequency
+        normalized_freq: frequency
+        documents: range
+        range: range
+    COCA_magazine_bigram_frequency_lemma:
+      display_name: COCA Magazine Bigram Frequency (Lemma)
+      description: Bigram frequencies and range data in Magazine - lemma-based analysis
+      file: resources/reference_lists/en/COCA_spoken_bigram_list.csv
+      format: tsv
+      columns: *id006
+      has_header: false
+      enabled: true
+      analysis_type: lemma
+      log_transformable:
+      - frequency
+      - normalized_freq
+      selectable_measures:
+      - frequency
+      - normalized_freq
+      - documents
+      - range
+      default_measures:
+      - frequency
+      - normalized_freq
+      - range
+      default_log_transforms:
+      - frequency
+      - normalized_freq
+      measure_classifications:
+        frequency: frequency
+        normalized_freq: frequency
+        documents: range
+        range: range
+    COCA_magazine_bigram_association_token:
+      display_name: COCA Magazine Bigram Associations (Token)
+      description: Bigram association measures (MI, T-score, Delta P) - token-based
+        analysis
+      file: resources/reference_lists/en/magazine_bi_contingency.csv
+      format: csv
+      columns: &id007
+        bigram: 0
+        frequency: 1
+        mi_score: 2
+        mi_2_score: 3
+        t_score: 4
+        delta_p: 5
+        ap_collex: 6
+      has_header: true
+      enabled: true
+      analysis_type: token
+      log_transformable:
+      - frequency
+      selectable_measures:
+      - frequency
+      - mi_score
+      - mi_2_score
+      - t_score
+      - delta_p
+      - ap_collex
+      default_measures:
+      - frequency
+      - t_score
+      default_log_transforms:
+      - frequency
+      measure_classifications:
+        frequency: frequency
+        mi_score: association
+        mi_2_score: association
+        t_score: association
+        delta_p: association
+        ap_collex: association
+    COCA_magazine_bigram_association_lemma:
+      display_name: COCA Magazine Bigram Associations (Lemma)
+      description: Bigram association measures (MI, T-score, Delta P) - lemma-based
+        analysis
+      file: resources/reference_lists/en/magazine_bigram_lemma_contingency.csv
+      format: csv
+      columns: *id007
+      has_header: true
+      enabled: true
+      analysis_type: lemma
+      log_transformable:
+      - frequency
+      selectable_measures:
+      - frequency
+      - mi_score
+      - mi_2_score
+      - t_score
+      - delta_p
+      - ap_collex
+      default_measures:
+      - frequency
+      - t_score
+      default_log_transforms:
+      - frequency
+      measure_classifications:
+        frequency: frequency
+        mi_score: association
+        mi_2_score: association
+        t_score: association
+        delta_p: association
+        ap_collex: association
   trigrams:
-    COCA_trigram_frequency:
-      display_name: "COCA Trigram Frequency"
-      description: "Trigram frequencies and range data"
-      files:
-        token: "resources/reference_lists/en/COCA_spoken_trigram_list.csv"
-        lemma: "resources/reference_lists/en/COCA_spoken_trigram_list.csv"
-      format: "tsv"
-      columns:
+    COCA_trigram_frequency_token:
+      display_name: COCA Trigram Frequency (Token)
+      description: Trigram frequencies and range data - token-based analysis
+      file: resources/reference_lists/en/COCA_spoken_trigram_list.csv
+      format: tsv
+      columns: &id008
         trigram: 0
         frequency: 1
         normalized_freq: 2
@@ -98,15 +485,62 @@ english:
         dispersion: 4
       has_header: false
       enabled: true
-
-    COCA_trigram_assoc_uni_bi:
-      display_name: "COCA Trigram→Bigram Associations"
-      description: "Trigram to bigram association measures"
-      files:
-        token: "resources/reference_lists/en/COCA_trigram_assoc_uni_bi_token.csv"
-        lemma: "resources/reference_lists/en/COCA_trigram_assoc_uni_bi_lemma.csv"
-      format: "csv"
-      columns:
+      analysis_type: token
+      log_transformable:
+      - frequency
+      - normalized_freq
+      selectable_measures:
+      - frequency
+      - normalized_freq
+      - range
+      - dispersion
+      default_measures:
+      - frequency
+      - normalized_freq
+      - range
+      default_log_transforms:
+      - frequency
+      - normalized_freq
+      measure_classifications:
+        frequency: frequency
+        normalized_freq: frequency
+        range: range
+        dispersion: range
+    COCA_trigram_frequency_lemma:
+      display_name: COCA Trigram Frequency (Lemma)
+      description: Trigram frequencies and range data - lemma-based analysis
+      file: resources/reference_lists/en/COCA_spoken_trigram_list.csv
+      format: tsv
+      columns: *id008
+      has_header: false
+      enabled: true
+      analysis_type: lemma
+      log_transformable:
+      - frequency
+      - normalized_freq
+      selectable_measures:
+      - frequency
+      - normalized_freq
+      - range
+      - dispersion
+      default_measures:
+      - frequency
+      - normalized_freq
+      - range
+      default_log_transforms:
+      - frequency
+      - normalized_freq
+      measure_classifications:
+        frequency: frequency
+        normalized_freq: frequency
+        range: range
+        dispersion: range
+    COCA_trigram_assoc_uni_bi_token:
+      display_name: COCA Trigram→Bigram Associations (Token)
+      description: Trigram to bigram association measures - token-based analysis
+      file: resources/reference_lists/en/spoken_tri_contingency_1.csv
+      format: csv
+      columns: &id009
         trigram: 0
         frequency: 1
         mi_score: 2
@@ -115,16 +549,65 @@ english:
         delta_p: 5
         ap_collex: 6
       has_header: true
-      enabled: false  # Disabled until files exist
-
-    COCA_trigram_assoc_bi_uni:
-      display_name: "COCA Trigram→Unigram Associations"
-      description: "Trigram to unigram association measures"
-      files:
-        token: "resources/reference_lists/en/COCA_trigram_assoc_bi_uni_token.csv"
-        lemma: "resources/reference_lists/en/COCA_trigram_assoc_bi_uni_lemma.csv"
-      format: "csv"
-      columns:
+      enabled: true
+      analysis_type: token
+      log_transformable:
+      - frequency
+      selectable_measures:
+      - frequency
+      - mi_score
+      - mi_2_score
+      - t_score
+      - delta_p
+      - ap_collex
+      default_measures:
+      - frequency
+      - t_score
+      default_log_transforms:
+      - frequency
+      measure_classifications:
+        frequency: frequency
+        mi_score: association
+        mi_2_score: association
+        t_score: association
+        delta_p: association
+        ap_collex: association
+    COCA_trigram_assoc_uni_bi_lemma:
+      display_name: COCA Trigram→Bigram Associations (Lemma)
+      description: Trigram to bigram association measures - lemma-based analysis
+      file: resources/reference_lists/en/spoken_trigram_lemma_contingency_1.csv
+      format: csv
+      columns: *id009
+      has_header: true
+      enabled: true
+      analysis_type: lemma
+      log_transformable:
+      - frequency
+      selectable_measures:
+      - frequency
+      - mi_score
+      - mi_2_score
+      - t_score
+      - delta_p
+      - ap_collex
+      default_measures:
+      - frequency
+      - t_score
+      default_log_transforms:
+      - frequency
+      measure_classifications:
+        frequency: frequency
+        mi_score: association
+        mi_2_score: association
+        t_score: association
+        delta_p: association
+        ap_collex: association
+    COCA_trigram_assoc_bi_uni_token:
+      display_name: COCA Trigram→Unigram Associations (Token)
+      description: Trigram to unigram association measures - token-based analysis
+      file: resources/reference_lists/en/spoken_tri_contingency_2.csv
+      format: csv
+      columns: &id010
         trigram: 0
         frequency: 1
         mi_score: 2
@@ -133,118 +616,398 @@ english:
         delta_p: 5
         ap_collex: 6
       has_header: true
-      enabled: false  # Disabled until files exist
-
+      enabled: true
+      analysis_type: token
+      log_transformable:
+      - frequency
+      selectable_measures:
+      - frequency
+      - mi_score
+      - mi_2_score
+      - t_score
+      - delta_p
+      - ap_collex
+      default_measures:
+      - frequency
+      - t_score
+      default_log_transforms:
+      - frequency
+      measure_classifications:
+        frequency: frequency
+        mi_score: association
+        mi_2_score: association
+        t_score: association
+        delta_p: association
+        ap_collex: association
+    COCA_trigram_assoc_bi_uni_lemma:
+      display_name: COCA Trigram→Unigram Associations (Lemma)
+      description: Trigram to unigram association measures - lemma-based analysis
+      file: resources/reference_lists/en/spoken_trigram_lemma_contingency_2.csv
+      format: csv
+      columns: *id010
+      has_header: true
+      enabled: true
+      analysis_type: lemma
+      log_transformable:
+      - frequency
+      selectable_measures:
+      - frequency
+      - mi_score
+      - mi_2_score
+      - t_score
+      - delta_p
+      - ap_collex
+      default_measures:
+      - frequency
+      - t_score
+      default_log_transforms:
+      - frequency
+      measure_classifications:
+        frequency: frequency
+        mi_score: association
+        mi_2_score: association
+        t_score: association
+        delta_p: association
+        ap_collex: association
 japanese:
   unigrams:
-    BCCWJ_frequency:
-      display_name: "BCCWJ Written - Frequency"
-      description: "BCCWJ raw frequency counts for written Japanese"
-      files:
-        token: "resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv"
-        lemma: "resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv"
-      format: "tsv"
+    BCCWJ_frequency_token:
+      display_name: BCCWJ Written - Frequency (Token)
+      description: BCCWJ raw frequency counts for written Japanese - token-based analysis
+      file: resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv
+      format: tsv
+      columns: &id011
+        surface_form: 1
+        lemma: 2
+        pos: 3
+        frequency: 6
       has_header: true
       enabled: true
+      analysis_type: token
+      log_transformable:
+      - frequency
+      selectable_measures:
+      - pos
+      - frequency
+      default_measures:
+      - frequency
+      - pos
+      default_log_transforms:
+      - frequency
+      measure_classifications:
+        pos: unknown
+        frequency: frequency
       japanese_corpus: true
-      columns:
-        surface_form: 1    # lForm
-        lemma: 2          # lemma
-        pos: 3            # pos
-        frequency: 6      # primary measure column
-      
-    BCCWJ_pmw:
-      display_name: "BCCWJ Written - Per Million Words"
-      description: "BCCWJ normalized frequency for written Japanese"
-      files:
-        token: "resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv"
-        lemma: "resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv"
-      format: "tsv"
+    BCCWJ_frequency_lemma:
+      display_name: BCCWJ Written - Frequency (Lemma)
+      description: BCCWJ raw frequency counts for written Japanese - lemma-based analysis
+      file: resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv
+      format: tsv
+      columns: *id011
       has_header: true
       enabled: true
+      analysis_type: lemma
+      log_transformable:
+      - frequency
+      selectable_measures:
+      - pos
+      - frequency
+      default_measures:
+      - frequency
+      - pos
+      default_log_transforms:
+      - frequency
+      measure_classifications:
+        pos: unknown
+        frequency: frequency
       japanese_corpus: true
-      columns:
+    BCCWJ_pmw_token:
+      display_name: BCCWJ Written - Per Million Words (Token)
+      description: BCCWJ normalized frequency for written Japanese - token-based analysis
+      file: resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv
+      format: tsv
+      columns: &id012
         surface_form: 1
         lemma: 2
         pos: 3
-        frequency: 7      # pmw column
-        
-    BCCWJ_rank:
-      display_name: "BCCWJ Written - Frequency Rank"
-      description: "BCCWJ frequency ranking for written Japanese"
-      files:
-        token: "resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv"
-        lemma: "resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv"
-      format: "tsv"
+        frequency: 7
       has_header: true
       enabled: true
+      analysis_type: token
+      log_transformable:
+      - frequency
+      selectable_measures:
+      - pos
+      - frequency
+      default_measures:
+      - frequency
+      - pos
+      default_log_transforms:
+      - frequency
+      measure_classifications:
+        pos: unknown
+        frequency: frequency
       japanese_corpus: true
-      columns:
+    BCCWJ_pmw_lemma:
+      display_name: BCCWJ Written - Per Million Words (Lemma)
+      description: BCCWJ normalized frequency for written Japanese - lemma-based analysis
+      file: resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv
+      format: tsv
+      columns: *id012
+      has_header: true
+      enabled: true
+      analysis_type: lemma
+      log_transformable:
+      - frequency
+      selectable_measures:
+      - pos
+      - frequency
+      default_measures:
+      - frequency
+      - pos
+      default_log_transforms:
+      - frequency
+      measure_classifications:
+        pos: unknown
+        frequency: frequency
+      japanese_corpus: true
+    BCCWJ_rank_token:
+      display_name: BCCWJ Written - Frequency Rank (Token)
+      description: BCCWJ frequency ranking for written Japanese - token-based analysis
+      file: resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv
+      format: tsv
+      columns: &id013
         surface_form: 1
         lemma: 2
         pos: 3
-        frequency: 0      # rank column
-
-    CSJ_frequency:
-      display_name: "CSJ Spoken - Frequency"
-      description: "CSJ raw frequency counts for spoken Japanese"
-      files:
-        token: "resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv"
-        lemma: "resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv"
-      format: "tsv"
+        frequency: 0
       has_header: true
       enabled: true
+      analysis_type: token
+      log_transformable:
+      - frequency
+      selectable_measures:
+      - pos
+      - frequency
+      default_measures:
+      - frequency
+      - pos
+      default_log_transforms:
+      - frequency
+      measure_classifications:
+        pos: unknown
+        frequency: frequency
       japanese_corpus: true
-      columns:
+    BCCWJ_rank_lemma:
+      display_name: BCCWJ Written - Frequency Rank (Lemma)
+      description: BCCWJ frequency ranking for written Japanese - lemma-based analysis
+      file: resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv
+      format: tsv
+      columns: *id013
+      has_header: true
+      enabled: true
+      analysis_type: lemma
+      log_transformable:
+      - frequency
+      selectable_measures:
+      - pos
+      - frequency
+      default_measures:
+      - frequency
+      - pos
+      default_log_transforms:
+      - frequency
+      measure_classifications:
+        pos: unknown
+        frequency: frequency
+      japanese_corpus: true
+    CSJ_frequency_token:
+      display_name: CSJ Spoken - Frequency (Token)
+      description: CSJ raw frequency counts for spoken Japanese - token-based analysis
+      file: resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv
+      format: tsv
+      columns: &id014
         surface_form: 1
         lemma: 2
         pos: 3
         frequency: 6
-
-    CSJ_pmw:
-      display_name: "CSJ Spoken - Per Million Words"
-      description: "CSJ normalized frequency for spoken Japanese"
-      files:
-        token: "resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv"
-        lemma: "resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv"
-      format: "tsv"
       has_header: true
       enabled: true
+      analysis_type: token
+      log_transformable:
+      - frequency
+      selectable_measures:
+      - pos
+      - frequency
+      default_measures:
+      - frequency
+      - pos
+      default_log_transforms:
+      - frequency
+      measure_classifications:
+        pos: unknown
+        frequency: frequency
       japanese_corpus: true
-      columns:
+    CSJ_frequency_lemma:
+      display_name: CSJ Spoken - Frequency (Lemma)
+      description: CSJ raw frequency counts for spoken Japanese - lemma-based analysis
+      file: resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv
+      format: tsv
+      columns: *id014
+      has_header: true
+      enabled: true
+      analysis_type: lemma
+      log_transformable:
+      - frequency
+      selectable_measures:
+      - pos
+      - frequency
+      default_measures:
+      - frequency
+      - pos
+      default_log_transforms:
+      - frequency
+      measure_classifications:
+        pos: unknown
+        frequency: frequency
+      japanese_corpus: true
+    CSJ_pmw_token:
+      display_name: CSJ Spoken - Per Million Words (Token)
+      description: CSJ normalized frequency for spoken Japanese - token-based analysis
+      file: resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv
+      format: tsv
+      columns: &id015
         surface_form: 1
         lemma: 2
         pos: 3
         frequency: 7
-
-    CSJ_rank:
-      display_name: "CSJ Spoken - Frequency Rank"
-      description: "CSJ frequency ranking for spoken Japanese"
-      files:
-        token: "resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv"
-        lemma: "resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv"
-      format: "tsv"
       has_header: true
       enabled: true
+      analysis_type: token
+      log_transformable:
+      - frequency
+      selectable_measures:
+      - pos
+      - frequency
+      default_measures:
+      - frequency
+      - pos
+      default_log_transforms:
+      - frequency
+      measure_classifications:
+        pos: unknown
+        frequency: frequency
       japanese_corpus: true
-      columns:
+    CSJ_pmw_lemma:
+      display_name: CSJ Spoken - Per Million Words (Lemma)
+      description: CSJ normalized frequency for spoken Japanese - lemma-based analysis
+      file: resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv
+      format: tsv
+      columns: *id015
+      has_header: true
+      enabled: true
+      analysis_type: lemma
+      log_transformable:
+      - frequency
+      selectable_measures:
+      - pos
+      - frequency
+      default_measures:
+      - frequency
+      - pos
+      default_log_transforms:
+      - frequency
+      measure_classifications:
+        pos: unknown
+        frequency: frequency
+      japanese_corpus: true
+    CSJ_rank_token:
+      display_name: CSJ Spoken - Frequency Rank (Token)
+      description: CSJ frequency ranking for spoken Japanese - token-based analysis
+      file: resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv
+      format: tsv
+      columns: &id016
         surface_form: 1
         lemma: 2
         pos: 3
         frequency: 0
-
-    jp_frequency:
-      display_name: "Japanese Frequency List"
-      description: "Frequency data for Japanese words"
-      files:
-        token: "resources/reference_lists/ja/jp_frequency_token.csv"
-        lemma: "resources/reference_lists/ja/jp_frequency_lemma.csv"
-      format: "csv"
-      columns:
+      has_header: true
+      enabled: true
+      analysis_type: token
+      log_transformable:
+      - frequency
+      selectable_measures:
+      - pos
+      - frequency
+      default_measures:
+      - frequency
+      - pos
+      default_log_transforms:
+      - frequency
+      measure_classifications:
+        pos: unknown
+        frequency: frequency
+      japanese_corpus: true
+    CSJ_rank_lemma:
+      display_name: CSJ Spoken - Frequency Rank (Lemma)
+      description: CSJ frequency ranking for spoken Japanese - lemma-based analysis
+      file: resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv
+      format: tsv
+      columns: *id016
+      has_header: true
+      enabled: true
+      analysis_type: lemma
+      log_transformable:
+      - frequency
+      selectable_measures:
+      - pos
+      - frequency
+      default_measures:
+      - frequency
+      - pos
+      default_log_transforms:
+      - frequency
+      measure_classifications:
+        pos: unknown
+        frequency: frequency
+      japanese_corpus: true
+    jp_frequency_token:
+      display_name: Japanese Frequency List (Token)
+      description: Frequency data for Japanese words - token-based analysis
+      file: resources/reference_lists/ja/jp_frequency_token.csv
+      format: csv
+      columns: &id017
         word: 0
         frequency: 1
       has_header: true
-      enabled: false  # Disabled until files exist
-
-  # bigrams: {}
-  # trigrams: {}
+      enabled: false
+      analysis_type: token
+      log_transformable:
+      - frequency
+      selectable_measures:
+      - frequency
+      default_measures:
+      - frequency
+      default_log_transforms:
+      - frequency
+      measure_classifications:
+        frequency: frequency
+    jp_frequency_lemma:
+      display_name: Japanese Frequency List (Lemma)
+      description: Frequency data for Japanese words - lemma-based analysis
+      file: resources/reference_lists/ja/jp_frequency_lemma.csv
+      format: csv
+      columns: *id017
+      has_header: true
+      enabled: false
+      analysis_type: lemma
+      log_transformable:
+      - frequency
+      selectable_measures:
+      - frequency
+      default_measures:
+      - frequency
+      default_log_transforms:
+      - frequency
+      measure_classifications:
+        frequency: frequency

config/reference_lists.yaml.backup_20250727_220815

@@ -0,0 +1,301 @@
+# Configuration for Default Reference Lists
+# Add new reference lists here and they'll automatically appear in the UI
+# Structure: language -> type -> list_name -> configuration
+
+english:
+  unigrams:
+    COCA_spoken_frequency:
+      display_name: "COCA Spoken Frequency"
+      description: "Frequency and range data from COCA spoken subcorpus"
+      files:
+        token: "resources/reference_lists/en/COCA_spoken_unigram_list.csv"
+        lemma: "resources/reference_lists/en/COCA_spoken_unigram_list.csv"  # Using same file for now
+      format: "tsv"
+      columns:
+        word: 0
+        frequency: 1
+        normalized_freq: 2
+        range: 3
+        dispersion: 4
+      has_header: false
+      enabled: true
+
+    COCA_magazine_frequency:
+      display_name: "COCA Magazine Frequency"
+      description: "Frequency and range data from COCA magazine subcorpus"
+      files:
+        token: "resources/reference_lists/en/COCA_magazine_unigram_list.csv"
+        lemma: "resources/reference_lists/en/COCA_magazine_unigram_list.csv"  # Using same file for now
+      format: "tsv"
+      columns:
+        word: 0
+        frequency: 1
+        normalized_freq: 2
+        range: 3
+        dispersion: 4
+      has_header: false
+      enabled: true
+
+    concreteness_ratings:
+      display_name: "Concreteness Ratings"
+      description: "Concreteness ratings for English words (1-5 scale)"
+      files:
+        token: "resources/reference_lists/en/Concreteness_Brysbaert.txt"
+        lemma: "resources/reference_lists/en/Concreteness_Brysbaert.txt"
+      format: "tsv"
+      columns:
+        word: 0
+        concreteness: 1
+      has_header: true
+      header_prefix: "#"
+      enabled: true  # Disabled until files exist
+
+    academic_words:
+      display_name: "Academic Word List"
+      description: "Common academic vocabulary for research writing"
+      files:
+        token: "resources/reference_lists/en/academic_words_token.csv"
+        lemma: "resources/reference_lists/en/academic_words_lemma.csv"
+      format: "csv"
+      columns:
+        word: 0
+        frequency: 1
+      has_header: true
+      enabled: false  # Disabled until files exist
+
+  bigrams:
+    COCA_spoken_bigram_frequency:
+      display_name: "COCA Spoken Bigram Frequency"
+      description: "Bigram frequencies and range data"
+      files:
+        token: "resources/reference_lists/en/COCA_spoken_bigram_list.csv"
+        lemma: "resources/reference_lists/en/COCA_spoken_bigram_list.csv"
+      format: "tsv"
+      columns:
+        bigram: 0
+        frequency: 1
+        normalized_freq: 2
+        documents: 3
+        range: 4
+      has_header: false
+      enabled: true  # Disabled until files exist
+
+    COCA_spoken_bigram_association:
+      display_name: "COCA Spoken Bigram Associations"
+      description: "Bigram association measures (MI, T-score, Delta P)"
+      files:
+        token: "resources/reference_lists/en/spoken_bi_contingency.csv"
+        lemma: "resources/reference_lists/en/spoken_bigram_lemma_contingency.csv"
+      format: "csv"
+      columns:
+        bigram: 0
+        frequency: 1
+        mi_score: 2
+        mi_2_score: 3
+        t_score: 4
+        delta_p: 5
+        ap_collex: 6
+      has_header: true
+      enabled: true  # Disabled until files exist
+
+    COCA_magazine_bigram_frequency:
+      display_name: "COCA Magazine Bigram Frequency"
+      description: "Bigram frequencies and range data in Magazine"
+      files:
+        token: "resources/reference_lists/en/COCA_magazine_bigram_list.csv"
+        lemma: "resources/reference_lists/en/COCA_spoken_bigram_list.csv"
+      format: "tsv"
+      columns:
+        bigram: 0
+        frequency: 1
+        normalized_freq: 2
+        documents: 3
+        range: 4
+      has_header: false
+      enabled: true  # Disabled until files exist
+
+    COCA_magazine_bigram_association:
+      display_name: "COCA Magazine Bigram Associations"
+      description: "Bigram association measures (MI, T-score, Delta P)"
+      files:
+        token: "resources/reference_lists/en/magazine_bi_contingency.csv"
+        lemma: "resources/reference_lists/en/magazine_bigram_lemma_contingency.csv"
+      format: "csv"
+      columns:
+        bigram: 0
+        frequency: 1
+        mi_score: 2
+        mi_2_score: 3
+        t_score: 4
+        delta_p: 5
+        ap_collex: 6
+      has_header: true
+      enabled: true  # Disabled until files exist
+
+
+  trigrams:
+    COCA_trigram_frequency:
+      display_name: "COCA Trigram Frequency"
+      description: "Trigram frequencies and range data"
+      files:
+        token: "resources/reference_lists/en/COCA_spoken_trigram_list.csv"
+        lemma: "resources/reference_lists/en/COCA_spoken_trigram_list.csv"
+      format: "tsv"
+      columns:
+        trigram: 0
+        frequency: 1
+        normalized_freq: 2
+        range: 3
+        dispersion: 4
+      has_header: false
+      enabled: true
+
+    COCA_trigram_assoc_uni_bi:
+      display_name: "COCA Trigram→Bigram Associations"
+      description: "Trigram to bigram association measures"
+      files:
+        token: "resources/reference_lists/en/spoken_tri_contingency_1.csv"
+        lemma: "resources/reference_lists/en/spoken_trigram_lemma_contingency_1.csv"
+      format: "csv"
+      columns:
+        trigram: 0
+        frequency: 1
+        mi_score: 2
+        mi_2_score: 3
+        t_score: 4
+        delta_p: 5
+        ap_collex: 6
+      has_header: true
+      enabled: true  # Disabled until files exist
+
+    COCA_trigram_assoc_bi_uni:
+      display_name: "COCA Trigram→Unigram Associations"
+      description: "Trigram to unigram association measures"
+      files:
+        token: "resources/reference_lists/en/spoken_tri_contingency_2.csv"
+        lemma: "resources/reference_lists/en/spoken_trigram_lemma_contingency_2.csv"
+      format: "csv"
+      columns:
+        trigram: 0
+        frequency: 1
+        mi_score: 2
+        mi_2_score: 3
+        t_score: 4
+        delta_p: 5
+        ap_collex: 6
+      has_header: true
+      enabled: true  # Disabled until files exist
+
+japanese:
+  unigrams:
+    BCCWJ_frequency:
+      display_name: "BCCWJ Written - Frequency"
+      description: "BCCWJ raw frequency counts for written Japanese"
+      files:
+        token: "resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv"
+        lemma: "resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv"
+      format: "tsv"
+      has_header: true
+      enabled: true
+      japanese_corpus: true
+      columns:
+        surface_form: 1    # lForm
+        lemma: 2          # lemma
+        pos: 3            # pos
+        frequency: 6      # primary measure column
+      
+    BCCWJ_pmw:
+      display_name: "BCCWJ Written - Per Million Words"
+      description: "BCCWJ normalized frequency for written Japanese"
+      files:
+        token: "resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv"
+        lemma: "resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv"
+      format: "tsv"
+      has_header: true
+      enabled: true
+      japanese_corpus: true
+      columns:
+        surface_form: 1
+        lemma: 2
+        pos: 3
+        frequency: 7      # pmw column
+        
+    BCCWJ_rank:
+      display_name: "BCCWJ Written - Frequency Rank"
+      description: "BCCWJ frequency ranking for written Japanese"
+      files:
+        token: "resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv"
+        lemma: "resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv"
+      format: "tsv"
+      has_header: true
+      enabled: true
+      japanese_corpus: true
+      columns:
+        surface_form: 1
+        lemma: 2
+        pos: 3
+        frequency: 0      # rank column
+
+    CSJ_frequency:
+      display_name: "CSJ Spoken - Frequency"
+      description: "CSJ raw frequency counts for spoken Japanese"
+      files:
+        token: "resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv"
+        lemma: "resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv"
+      format: "tsv"
+      has_header: true
+      enabled: true
+      japanese_corpus: true
+      columns:
+        surface_form: 1
+        lemma: 2
+        pos: 3
+        frequency: 6
+
+    CSJ_pmw:
+      display_name: "CSJ Spoken - Per Million Words"
+      description: "CSJ normalized frequency for spoken Japanese"
+      files:
+        token: "resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv"
+        lemma: "resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv"
+      format: "tsv"
+      has_header: true
+      enabled: true
+      japanese_corpus: true
+      columns:
+        surface_form: 1
+        lemma: 2
+        pos: 3
+        frequency: 7
+
+    CSJ_rank:
+      display_name: "CSJ Spoken - Frequency Rank"
+      description: "CSJ frequency ranking for spoken Japanese"
+      files:
+        token: "resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv"
+        lemma: "resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv"
+      format: "tsv"
+      has_header: true
+      enabled: true
+      japanese_corpus: true
+      columns:
+        surface_form: 1
+        lemma: 2
+        pos: 3
+        frequency: 0
+
+    jp_frequency:
+      display_name: "Japanese Frequency List"
+      description: "Frequency data for Japanese words"
+      files:
+        token: "resources/reference_lists/ja/jp_frequency_token.csv"
+        lemma: "resources/reference_lists/ja/jp_frequency_lemma.csv"
+      format: "csv"
+      columns:
+        word: 0
+        frequency: 1
+      has_header: true
+      enabled: false  # Disabled until files exist
+
+  # bigrams: {}
+  # trigrams: {}

config/reference_lists.yaml.backup_20250727_230913

@@ -0,0 +1,319 @@
+english:
+  unigrams:
+    COCA_magazine_frequency:
+      display_name: COCA Magazine Frequency
+      description: Frequency and range data from COCA magazine subcorpus
+      files:
+        token: resources/reference_lists/en/COCA_magazine_unigram_list.csv
+        lemma: resources/reference_lists/en/COCA_magazine_unigram_list.csv
+      format: tsv
+      columns:
+        word: 0
+        frequency: 1
+        normalized_freq: 2
+        range: 3
+        dispersion: 4
+      has_header: false
+      enabled: true
+    concreteness_ratings:
+      display_name: Concreteness Ratings
+      description: Concreteness ratings for English words (1-5 scale)
+      files:
+        token: resources/reference_lists/en/Concreteness_Brysbaert.txt
+        lemma: resources/reference_lists/en/Concreteness_Brysbaert.txt
+      format: tsv
+      columns:
+        word: 0
+        concreteness: 1
+      has_header: true
+      header_prefix: '#'
+      enabled: true
+    academic_words:
+      display_name: Academic Word List
+      description: Common academic vocabulary for research writing
+      files:
+        token: resources/reference_lists/en/academic_words_token.csv
+        lemma: resources/reference_lists/en/academic_words_lemma.csv
+      format: csv
+      columns:
+        word: 0
+        frequency: 1
+      has_header: true
+      enabled: false
+    COCA_spoken_frequency_token:
+      display_name: COCA Spoken Frequency (Token)
+      description: Frequency and range data from COCA spoken subcorpus - token-based
+        analysis
+      file: resources/reference_lists/en/COCA_spoken_unigram_list.csv
+      format: tsv
+      columns:
+        word: 0
+        frequency: 1
+        normalized_freq: 2
+        range: 3
+        dispersion: 4
+      has_header: false
+      enabled: true
+      analysis_type: token
+      log_transformable:
+      - frequency
+      - normalized_freq
+      selectable_measures:
+      - frequency
+      - normalized_freq
+      - range
+      - dispersion
+      default_measures:
+      - frequency
+      - normalized_freq
+      default_log_transforms:
+      - frequency
+      - normalized_freq
+    COCA_spoken_frequency_lemma:
+      display_name: COCA Spoken Frequency (Lemma)
+      description: Frequency and range data from COCA spoken subcorpus - lemma-based
+        analysis
+      file: resources/reference_lists/en/COCA_spoken_unigram_list.csv
+      format: tsv
+      columns:
+        word: 0
+        frequency: 1
+        normalized_freq: 2
+        range: 3
+        dispersion: 4
+      has_header: false
+      enabled: true
+      analysis_type: lemma
+      log_transformable:
+      - frequency
+      - normalized_freq
+      selectable_measures:
+      - frequency
+      - normalized_freq
+      - range
+      - dispersion
+      default_measures:
+      - frequency
+      - normalized_freq
+      default_log_transforms:
+      - frequency
+      - normalized_freq
+  bigrams:
+    COCA_spoken_bigram_frequency:
+      display_name: COCA Spoken Bigram Frequency
+      description: Bigram frequencies and range data
+      files:
+        token: resources/reference_lists/en/COCA_spoken_bigram_list.csv
+        lemma: resources/reference_lists/en/COCA_spoken_bigram_list.csv
+      format: tsv
+      columns:
+        bigram: 0
+        frequency: 1
+        normalized_freq: 2
+        documents: 3
+        range: 4
+      has_header: false
+      enabled: true
+    COCA_spoken_bigram_association:
+      display_name: COCA Spoken Bigram Associations
+      description: Bigram association measures (MI, T-score, Delta P)
+      files:
+        token: resources/reference_lists/en/spoken_bi_contingency.csv
+        lemma: resources/reference_lists/en/spoken_bigram_lemma_contingency.csv
+      format: csv
+      columns:
+        bigram: 0
+        frequency: 1
+        mi_score: 2
+        mi_2_score: 3
+        t_score: 4
+        delta_p: 5
+        ap_collex: 6
+      has_header: true
+      enabled: true
+    COCA_magazine_bigram_frequency:
+      display_name: COCA Magazine Bigram Frequency
+      description: Bigram frequencies and range data in Magazine
+      files:
+        token: resources/reference_lists/en/COCA_magazine_bigram_list.csv
+        lemma: resources/reference_lists/en/COCA_spoken_bigram_list.csv
+      format: tsv
+      columns:
+        bigram: 0
+        frequency: 1
+        normalized_freq: 2
+        documents: 3
+        range: 4
+      has_header: false
+      enabled: true
+    COCA_magazine_bigram_association:
+      display_name: COCA Magazine Bigram Associations
+      description: Bigram association measures (MI, T-score, Delta P)
+      files:
+        token: resources/reference_lists/en/magazine_bi_contingency.csv
+        lemma: resources/reference_lists/en/magazine_bigram_lemma_contingency.csv
+      format: csv
+      columns:
+        bigram: 0
+        frequency: 1
+        mi_score: 2
+        mi_2_score: 3
+        t_score: 4
+        delta_p: 5
+        ap_collex: 6
+      has_header: true
+      enabled: true
+  trigrams:
+    COCA_trigram_frequency:
+      display_name: COCA Trigram Frequency
+      description: Trigram frequencies and range data
+      files:
+        token: resources/reference_lists/en/COCA_spoken_trigram_list.csv
+        lemma: resources/reference_lists/en/COCA_spoken_trigram_list.csv
+      format: tsv
+      columns:
+        trigram: 0
+        frequency: 1
+        normalized_freq: 2
+        range: 3
+        dispersion: 4
+      has_header: false
+      enabled: true
+    COCA_trigram_assoc_uni_bi:
+      display_name: COCA Trigram→Bigram Associations
+      description: Trigram to bigram association measures
+      files:
+        token: resources/reference_lists/en/spoken_tri_contingency_1.csv
+        lemma: resources/reference_lists/en/spoken_trigram_lemma_contingency_1.csv
+      format: csv
+      columns:
+        trigram: 0
+        frequency: 1
+        mi_score: 2
+        mi_2_score: 3
+        t_score: 4
+        delta_p: 5
+        ap_collex: 6
+      has_header: true
+      enabled: true
+    COCA_trigram_assoc_bi_uni:
+      display_name: COCA Trigram→Unigram Associations
+      description: Trigram to unigram association measures
+      files:
+        token: resources/reference_lists/en/spoken_tri_contingency_2.csv
+        lemma: resources/reference_lists/en/spoken_trigram_lemma_contingency_2.csv
+      format: csv
+      columns:
+        trigram: 0
+        frequency: 1
+        mi_score: 2
+        mi_2_score: 3
+        t_score: 4
+        delta_p: 5
+        ap_collex: 6
+      has_header: true
+      enabled: true
+japanese:
+  unigrams:
+    BCCWJ_frequency:
+      display_name: BCCWJ Written - Frequency
+      description: BCCWJ raw frequency counts for written Japanese
+      files:
+        token: resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv
+        lemma: resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv
+      format: tsv
+      has_header: true
+      enabled: true
+      japanese_corpus: true
+      columns:
+        surface_form: 1
+        lemma: 2
+        pos: 3
+        frequency: 6
+    BCCWJ_pmw:
+      display_name: BCCWJ Written - Per Million Words
+      description: BCCWJ normalized frequency for written Japanese
+      files:
+        token: resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv
+        lemma: resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv
+      format: tsv
+      has_header: true
+      enabled: true
+      japanese_corpus: true
+      columns:
+        surface_form: 1
+        lemma: 2
+        pos: 3
+        frequency: 7
+    BCCWJ_rank:
+      display_name: BCCWJ Written - Frequency Rank
+      description: BCCWJ frequency ranking for written Japanese
+      files:
+        token: resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv
+        lemma: resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv
+      format: tsv
+      has_header: true
+      enabled: true
+      japanese_corpus: true
+      columns:
+        surface_form: 1
+        lemma: 2
+        pos: 3
+        frequency: 0
+    CSJ_frequency:
+      display_name: CSJ Spoken - Frequency
+      description: CSJ raw frequency counts for spoken Japanese
+      files:
+        token: resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv
+        lemma: resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv
+      format: tsv
+      has_header: true
+      enabled: true
+      japanese_corpus: true
+      columns:
+        surface_form: 1
+        lemma: 2
+        pos: 3
+        frequency: 6
+    CSJ_pmw:
+      display_name: CSJ Spoken - Per Million Words
+      description: CSJ normalized frequency for spoken Japanese
+      files:
+        token: resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv
+        lemma: resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv
+      format: tsv
+      has_header: true
+      enabled: true
+      japanese_corpus: true
+      columns:
+        surface_form: 1
+        lemma: 2
+        pos: 3
+        frequency: 7
+    CSJ_rank:
+      display_name: CSJ Spoken - Frequency Rank
+      description: CSJ frequency ranking for spoken Japanese
+      files:
+        token: resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv
+        lemma: resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv
+      format: tsv
+      has_header: true
+      enabled: true
+      japanese_corpus: true
+      columns:
+        surface_form: 1
+        lemma: 2
+        pos: 3
+        frequency: 0
+    jp_frequency:
+      display_name: Japanese Frequency List
+      description: Frequency data for Japanese words
+      files:
+        token: resources/reference_lists/ja/jp_frequency_token.csv
+        lemma: resources/reference_lists/ja/jp_frequency_lemma.csv
+      format: csv
+      columns:
+        word: 0
+        frequency: 1
+      has_header: true
+      enabled: false

config/reference_lists.yaml.backup_20250727_231728

@@ -0,0 +1,319 @@
+english:
+  unigrams:
+    COCA_magazine_frequency:
+      display_name: COCA Magazine Frequency
+      description: Frequency and range data from COCA magazine subcorpus
+      files:
+        token: resources/reference_lists/en/COCA_magazine_unigram_list.csv
+        lemma: resources/reference_lists/en/COCA_magazine_unigram_list.csv
+      format: tsv
+      columns:
+        word: 0
+        frequency: 1
+        normalized_freq: 2
+        range: 3
+        dispersion: 4
+      has_header: false
+      enabled: true
+    concreteness_ratings:
+      display_name: Concreteness Ratings
+      description: Concreteness ratings for English words (1-5 scale)
+      files:
+        token: resources/reference_lists/en/Concreteness_Brysbaert.txt
+        lemma: resources/reference_lists/en/Concreteness_Brysbaert.txt
+      format: tsv
+      columns:
+        word: 0
+        concreteness: 1
+      has_header: true
+      header_prefix: '#'
+      enabled: true
+    academic_words:
+      display_name: Academic Word List
+      description: Common academic vocabulary for research writing
+      files:
+        token: resources/reference_lists/en/academic_words_token.csv
+        lemma: resources/reference_lists/en/academic_words_lemma.csv
+      format: csv
+      columns:
+        word: 0
+        frequency: 1
+      has_header: true
+      enabled: false
+    COCA_spoken_frequency_token:
+      display_name: COCA Spoken Frequency (Token)
+      description: Frequency and range data from COCA spoken subcorpus - token-based
+        analysis
+      file: resources/reference_lists/en/COCA_spoken_unigram_list.csv
+      format: tsv
+      columns:
+        word: 0
+        frequency: 1
+        normalized_freq: 2
+        range: 3
+        dispersion: 4
+      has_header: false
+      enabled: true
+      analysis_type: token
+      log_transformable:
+      - frequency
+      - normalized_freq
+      selectable_measures:
+      - frequency
+      - normalized_freq
+      - range
+      - dispersion
+      default_measures:
+      - frequency
+      - normalized_freq
+      default_log_transforms:
+      - frequency
+      - normalized_freq
+    COCA_spoken_frequency_lemma:
+      display_name: COCA Spoken Frequency (Lemma)
+      description: Frequency and range data from COCA spoken subcorpus - lemma-based
+        analysis
+      file: resources/reference_lists/en/COCA_spoken_unigram_list.csv
+      format: tsv
+      columns:
+        word: 0
+        frequency: 1
+        normalized_freq: 2
+        range: 3
+        dispersion: 4
+      has_header: false
+      enabled: true
+      analysis_type: lemma
+      log_transformable:
+      - frequency
+      - normalized_freq
+      selectable_measures:
+      - frequency
+      - normalized_freq
+      - range
+      - dispersion
+      default_measures:
+      - frequency
+      - normalized_freq
+      default_log_transforms:
+      - frequency
+      - normalized_freq
+  bigrams:
+    COCA_spoken_bigram_frequency:
+      display_name: COCA Spoken Bigram Frequency
+      description: Bigram frequencies and range data
+      files:
+        token: resources/reference_lists/en/COCA_spoken_bigram_list.csv
+        lemma: resources/reference_lists/en/COCA_spoken_bigram_list.csv
+      format: tsv
+      columns:
+        bigram: 0
+        frequency: 1
+        normalized_freq: 2
+        documents: 3
+        range: 4
+      has_header: false
+      enabled: true
+    COCA_spoken_bigram_association:
+      display_name: COCA Spoken Bigram Associations
+      description: Bigram association measures (MI, T-score, Delta P)
+      files:
+        token: resources/reference_lists/en/spoken_bi_contingency.csv
+        lemma: resources/reference_lists/en/spoken_bigram_lemma_contingency.csv
+      format: csv
+      columns:
+        bigram: 0
+        frequency: 1
+        mi_score: 2
+        mi_2_score: 3
+        t_score: 4
+        delta_p: 5
+        ap_collex: 6
+      has_header: true
+      enabled: true
+    COCA_magazine_bigram_frequency:
+      display_name: COCA Magazine Bigram Frequency
+      description: Bigram frequencies and range data in Magazine
+      files:
+        token: resources/reference_lists/en/COCA_magazine_bigram_list.csv
+        lemma: resources/reference_lists/en/COCA_spoken_bigram_list.csv
+      format: tsv
+      columns:
+        bigram: 0
+        frequency: 1
+        normalized_freq: 2
+        documents: 3
+        range: 4
+      has_header: false
+      enabled: true
+    COCA_magazine_bigram_association:
+      display_name: COCA Magazine Bigram Associations
+      description: Bigram association measures (MI, T-score, Delta P)
+      files:
+        token: resources/reference_lists/en/magazine_bi_contingency.csv
+        lemma: resources/reference_lists/en/magazine_bigram_lemma_contingency.csv
+      format: csv
+      columns:
+        bigram: 0
+        frequency: 1
+        mi_score: 2
+        mi_2_score: 3
+        t_score: 4
+        delta_p: 5
+        ap_collex: 6
+      has_header: true
+      enabled: true
+  trigrams:
+    COCA_trigram_frequency:
+      display_name: COCA Trigram Frequency
+      description: Trigram frequencies and range data
+      files:
+        token: resources/reference_lists/en/COCA_spoken_trigram_list.csv
+        lemma: resources/reference_lists/en/COCA_spoken_trigram_list.csv
+      format: tsv
+      columns:
+        trigram: 0
+        frequency: 1
+        normalized_freq: 2
+        range: 3
+        dispersion: 4
+      has_header: false
+      enabled: true
+    COCA_trigram_assoc_uni_bi:
+      display_name: COCA Trigram→Bigram Associations
+      description: Trigram to bigram association measures
+      files:
+        token: resources/reference_lists/en/spoken_tri_contingency_1.csv
+        lemma: resources/reference_lists/en/spoken_trigram_lemma_contingency_1.csv
+      format: csv
+      columns:
+        trigram: 0
+        frequency: 1
+        mi_score: 2
+        mi_2_score: 3
+        t_score: 4
+        delta_p: 5
+        ap_collex: 6
+      has_header: true
+      enabled: true
+    COCA_trigram_assoc_bi_uni:
+      display_name: COCA Trigram→Unigram Associations
+      description: Trigram to unigram association measures
+      files:
+        token: resources/reference_lists/en/spoken_tri_contingency_2.csv
+        lemma: resources/reference_lists/en/spoken_trigram_lemma_contingency_2.csv
+      format: csv
+      columns:
+        trigram: 0
+        frequency: 1
+        mi_score: 2
+        mi_2_score: 3
+        t_score: 4
+        delta_p: 5
+        ap_collex: 6
+      has_header: true
+      enabled: true
+japanese:
+  unigrams:
+    BCCWJ_frequency:
+      display_name: BCCWJ Written - Frequency
+      description: BCCWJ raw frequency counts for written Japanese
+      files:
+        token: resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv
+        lemma: resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv
+      format: tsv
+      has_header: true
+      enabled: true
+      japanese_corpus: true
+      columns:
+        surface_form: 1
+        lemma: 2
+        pos: 3
+        frequency: 6
+    BCCWJ_pmw:
+      display_name: BCCWJ Written - Per Million Words
+      description: BCCWJ normalized frequency for written Japanese
+      files:
+        token: resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv
+        lemma: resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv
+      format: tsv
+      has_header: true
+      enabled: true
+      japanese_corpus: true
+      columns:
+        surface_form: 1
+        lemma: 2
+        pos: 3
+        frequency: 7
+    BCCWJ_rank:
+      display_name: BCCWJ Written - Frequency Rank
+      description: BCCWJ frequency ranking for written Japanese
+      files:
+        token: resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv
+        lemma: resources/reference_lists/ja/BCCWJ_frequencylist_suw_ver1_1.tsv
+      format: tsv
+      has_header: true
+      enabled: true
+      japanese_corpus: true
+      columns:
+        surface_form: 1
+        lemma: 2
+        pos: 3
+        frequency: 0
+    CSJ_frequency:
+      display_name: CSJ Spoken - Frequency
+      description: CSJ raw frequency counts for spoken Japanese
+      files:
+        token: resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv
+        lemma: resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv
+      format: tsv
+      has_header: true
+      enabled: true
+      japanese_corpus: true
+      columns:
+        surface_form: 1
+        lemma: 2
+        pos: 3
+        frequency: 6
+    CSJ_pmw:
+      display_name: CSJ Spoken - Per Million Words
+      description: CSJ normalized frequency for spoken Japanese
+      files:
+        token: resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv
+        lemma: resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv
+      format: tsv
+      has_header: true
+      enabled: true
+      japanese_corpus: true
+      columns:
+        surface_form: 1
+        lemma: 2
+        pos: 3
+        frequency: 7
+    CSJ_rank:
+      display_name: CSJ Spoken - Frequency Rank
+      description: CSJ frequency ranking for spoken Japanese
+      files:
+        token: resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv
+        lemma: resources/reference_lists/ja/CSJ_frequencylist_suw_ver201803.tsv
+      format: tsv
+      has_header: true
+      enabled: true
+      japanese_corpus: true
+      columns:
+        surface_form: 1
+        lemma: 2
+        pos: 3
+        frequency: 0
+    jp_frequency:
+      display_name: Japanese Frequency List
+      description: Frequency data for Japanese words
+      files:
+        token: resources/reference_lists/ja/jp_frequency_token.csv
+        lemma: resources/reference_lists/ja/jp_frequency_lemma.csv
+      format: csv
+      columns:
+        word: 0
+        frequency: 1
+      has_header: true
+      enabled: false

text_analyzer/lexical_sophistication.py

@@ -484,16 +484,69 @@ class LexicalSophisticationAnalyzer(BaseAnalyzer):
         
         return score
     
-    def analyze_text(self, text: str, selected_indices: List[str], 
-                    apply_log: bool = False, word_type_filter: Optional[str] = None) -> Dict:
+    def _should_apply_log_transform(self, index_name: str, analysis_type: str, 
+                                  measure_name: str, log_transforms: Optional[Dict[str, List[str]]], 
+                                  apply_log_fallback: bool) -> bool:
+        """
+        Determine if a specific measure should be log-transformed.
+        
+        Args:
+            index_name: Name of the reference index
+            analysis_type: 'token' or 'lemma'
+            measure_name: Name of the measure (e.g., 'frequency', 'MI')
+            log_transforms: Dict mapping index names to lists of measures to log-transform
+            apply_log_fallback: Legacy fallback boolean
+            
+        Returns:
+            True if the measure should be log-transformed, False otherwise
+        """
+        # If new log_transforms parameter is provided, use it
+        if log_transforms is not None:
+            index_transforms = log_transforms.get(index_name, [])
+            return measure_name in index_transforms
+        
+        # Fallback to legacy apply_log behavior for backward compatibility
+        return apply_log_fallback
+    
+    def _should_compute_measure(self, index_name: str, measure_name: str, 
+                              selected_measures: Optional[Dict[str, List[str]]]) -> bool:
+        """
+        Determine if a specific measure should be computed.
+        
+        Args:
+            index_name: Name of the reference index
+            measure_name: Name of the measure (e.g., 'frequency', 'MI')
+            selected_measures: Dict mapping index names to lists of measures to compute
+            
+        Returns:
+            True if the measure should be computed, False otherwise
+        """
+        # If selected_measures is provided, use it for filtering
+        if selected_measures is not None:
+            index_measures = selected_measures.get(index_name, [])
+            return measure_name in index_measures
+        
+        # If not specified, compute all measures (backward compatibility)
+        return True
+    
+    def analyze_text(self, text: str, selected_indices: List[str],
+                    apply_log: bool = False, word_type_filter: Optional[str] = None,
+                    log_transforms: Optional[Dict[str, List[str]]] = None,
+                    selected_measures: Optional[Dict[str, List[str]]] = None) -> Dict:
         """
         Analyze text and return lexical sophistication scores.
         
         Args:
             text: Input text to analyze
             selected_indices: List of reference indices to apply
-            apply_log: Whether to apply log10 transformation
+            apply_log: Whether to apply log10 transformation (legacy parameter, superseded by log_transforms)
             word_type_filter: Filter by word type ('CW', 'FW', or None for all)
+            log_transforms: Dict mapping index names to list of measures that should be log-transformed
+                          e.g., {'COCA_spoken_frequency_token': ['frequency', 'normalized_freq']}
+                          If None, falls back to apply_log behavior for backward compatibility
+            selected_measures: Dict mapping index names to list of measures to compute
+                             e.g., {'COCA_spoken_frequency_token': ['frequency', 'range']}
+                             If None, computes all available measures for backward compatibility
             
         Returns:
             Dictionary containing analysis results
@@ -607,13 +660,21 @@ class LexicalSophisticationAnalyzer(BaseAnalyzer):
                     token_detail[f"{index_name}_token"] = token_score if token_score is not None else None
                     token_detail[f"{index_name}_lemma"] = lemma_score if lemma_score is not None else None
                 
-                # Collect for summary statistics
+                # Collect for summary statistics with selective log transformation
                 if token_score is not None:
-                    score_val = np.log10(token_score) if apply_log and token_score > 0 else token_score
+                    # Check if this specific measure should be log-transformed
+                    should_log_transform = self._should_apply_log_transform(
+                        index_name, 'token', 'frequency', log_transforms, apply_log
+                    )
+                    score_val = np.log10(token_score) if should_log_transform and token_score > 0 else token_score
                     all_scores[f"{index_name}_token_{word_type}"].append(score_val)
                 
                 if lemma_score is not None:
-                    score_val = np.log10(lemma_score) if apply_log and lemma_score > 0 else lemma_score
+                    # Check if this specific measure should be log-transformed
+                    should_log_transform = self._should_apply_log_transform(
+                        index_name, 'lemma', 'frequency', log_transforms, apply_log
+                    )
+                    score_val = np.log10(lemma_score) if should_log_transform and lemma_score > 0 else lemma_score
                     all_scores[f"{index_name}_lemma_{word_type}"].append(score_val)
             
             results['token_details'].append(token_detail)
@@ -664,10 +725,19 @@ class LexicalSophisticationAnalyzer(BaseAnalyzer):
                     # Get available measures
                     available_measures = ref_data.columns[1:].tolist()
                     
+                    # Filter measures based on selection
                     for measure in available_measures:
+                        # Check if this measure should be computed
+                        if not self._should_compute_measure(index_name, measure, selected_measures):
+                            continue
+                            
                         score = self._lookup_score(ngram, index_name, ngram_type, measure)
                         if score is not None:
-                            score_val = np.log10(score) if apply_log and score > 0 else score
+                            # Check if this measure should be log-transformed
+                            should_log_transform = self._should_apply_log_transform(
+                                index_name, ngram_type, measure, log_transforms, apply_log
+                            )
+                            score_val = np.log10(score) if should_log_transform and score > 0 else score
                             ngram_detail[f"{index_name}_{measure}"] = score_val
                         else:
                             ngram_detail[f"{index_name}_{measure}"] = None
@@ -686,12 +756,21 @@ class LexicalSophisticationAnalyzer(BaseAnalyzer):
                 # Get available measures (all columns except the first one)
                 available_measures = ref_data.columns[1:].tolist()
                 
+                # Filter measures based on selection and compute summary statistics
                 for measure in available_measures:
+                    # Check if this measure should be computed
+                    if not self._should_compute_measure(index_name, measure, selected_measures):
+                        continue
+                        
                     ngram_scores = []
                     for ngram in ngrams:
                         score = self._lookup_score(ngram, index_name, ngram_type, measure)
                         if score is not None:
-                            score_val = np.log10(score) if apply_log and score > 0 else score
+                            # Check if this measure should be log-transformed
+                            should_log_transform = self._should_apply_log_transform(
+                                index_name, ngram_type, measure, log_transforms, apply_log
+                            )
+                            score_val = np.log10(score) if should_log_transform and score > 0 else score
                             ngram_scores.append(score_val)
                     
                     if ngram_scores:

web_app/components/ui_components.py

@@ -173,20 +173,82 @@ class UIComponents:
     
     @staticmethod
     def render_analysis_options():
-        """Render analysis options UI."""
+        """Render enhanced analysis options UI with sophisticated hierarchical interface."""
+        from web_app.defaults_manager import DefaultsManager
+        from web_app.config_manager import ConfigManager
+        from web_app.session_manager import SessionManager
+        
+        st.subheader("🔧 Analysis Configuration")
+        
+        # Get current configuration
+        config = ConfigManager.load_reference_config()
+        reference_lists = SessionManager.get_reference_lists()
+        
+        # Enhanced Reference Lists & Measures Section
+        st.write("### 📋 Reference Lists & Measures")
+        
+        # Render the sophisticated hierarchical interface
+        selected_measures, log_transforms = UIComponents.render_enhanced_reference_selection(config, reference_lists)
+        
+        # Global Analysis Options
+        st.write("### 🎯 Analysis Types")
         col1, col2 = st.columns(2)
         
         with col1:
-            apply_log = st.checkbox("Apply log₁₀ transformation")
-        
+            token_analysis = st.checkbox("☑️ Token-based", value=True, key="token_analysis_enabled")
         with col2:
-            word_type_filter = st.selectbox(
-                "Word Type Filter",
-                options=[None, 'CW', 'FW'],
-                format_func=lambda x: 'All Words' if x is None else ('Content Words' if x == 'CW' else 'Function Words')
+            lemma_analysis = st.checkbox("☑️ Lemma-based", value=True, key="lemma_analysis_enabled")
+        
+        # Global Options
+        st.write("### ⚙️ Global Options")
+        word_type_filter = st.selectbox(
+            "Word Type Filter:",
+            options=[None, 'CW', 'FW'],
+            format_func=lambda x: 'All Words ▼' if x is None else ('Content Words' if x == 'CW' else 'Function Words'),
+            key="word_type_filter"
+        )
+        
+        # Advanced Configuration Section
+        with st.expander("🎯 Advanced Configuration (Optional)", expanded=False):
+            st.info("ℹ️ **Smart Defaults Active**: The system automatically applies appropriate settings. "
+                   "Expand this section only if you need custom control.")
+            
+            # Legacy log transformation toggle
+            legacy_log_toggle = st.checkbox(
+                "Apply log₁₀ transformation to ALL measures (Legacy Mode)", 
+                value=False,
+                help="⚠️ Not recommended: This applies log transformation to all measures, "
+                     "including those where it's scientifically inappropriate (e.g., concreteness ratings).",
+                key="legacy_log_transform"
             )
+            
+            if legacy_log_toggle:
+                st.warning("⚠️ Legacy mode enabled: Log transformation will be applied to ALL numerical measures. "
+                          "This may produce scientifically invalid results for psycholinguistic measures.")
         
-        return apply_log, word_type_filter
+        # Return enhanced configuration
+        return {
+            'token_analysis': token_analysis,
+            'lemma_analysis': lemma_analysis,
+            'word_type_filter': word_type_filter,
+            'selected_measures': selected_measures,
+            'log_transforms': log_transforms,
+            'use_smart_defaults': not st.session_state.get('legacy_log_transform', False),
+            'legacy_log_transform': st.session_state.get('legacy_log_transform', False)
+        }
+    
+    @staticmethod
+    def _find_entry_config(entry_name: str, config: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+        """Find configuration entry by name."""
+        for language, lang_data in config.items():
+            if not isinstance(lang_data, dict):
+                continue
+            for ngram_type, type_data in lang_data.items():
+                if not isinstance(type_data, dict):
+                    continue
+                if entry_name in type_data:
+                    return type_data[entry_name]
+        return None
     
     @staticmethod
     def display_configured_indices():
@@ -233,3 +295,53 @@ class UIComponents:
         
         if success_count == 0:
             st.error("No valid configurations found")
+    
+    @staticmethod
+    def render_enhanced_reference_selection(config: Dict[str, Any], reference_lists: Dict[str, Any]) -> Tuple[Dict[str, List[str]], Dict[str, List[str]]]:
+        """Render the enhanced reference list selection interface with hierarchical display."""
+        from web_app.defaults_manager import DefaultsManager
+        
+        # Initialize return values
+        selected_measures = {}
+        log_transforms = {}
+        
+        if not reference_lists:
+            st.info("No reference lists selected. Please configure reference lists first.")
+            return selected_measures, log_transforms
+        
+        # Simple hierarchical display showing selected lists with smart defaults info
+        for list_name in reference_lists.keys():
+            # Show smart defaults indicator
+            entry_config = UIComponents._find_entry_config(list_name, config)
+            if entry_config and entry_config.get('default_measures'):
+                defaults_info = f"📊 {len(entry_config['default_measures'])} measures selected"
+                log_info = f"🔄 {len(entry_config.get('default_log_transforms', []))} log-transformed"
+                
+                # Determine analysis type badges
+                analysis_badges = []
+                if entry_config.get('analysis_type') == 'token' or not entry_config.get('analysis_type'):
+                    analysis_badges.append("[Token ✓]")
+                if entry_config.get('analysis_type') == 'lemma' or not entry_config.get('analysis_type'):
+                    analysis_badges.append("[Lemma ✓]")
+                
+                analysis_info = " ".join(analysis_badges) if analysis_badges else ""
+                
+                st.write(f"├─ **{list_name}** {analysis_info} [ℹ️ Smart defaults]")
+                st.write(f"   {defaults_info}, {log_info}")
+                
+                # Apply smart defaults to return values
+                selected_measures[list_name] = entry_config.get('default_measures', [])
+                log_transforms[list_name] = entry_config.get('default_log_transforms', [])
+            else:
+                st.write(f"├─ **{list_name}** [Legacy configuration]")
+        
+        return selected_measures, log_transforms
+    
+    @staticmethod
+    def group_has_smart_defaults(group_entries: List[str], config: Dict[str, Any]) -> bool:
+        """Check if a group has smart defaults configured."""
+        for entry_name in group_entries:
+            entry_config = UIComponents._find_entry_config(entry_name, config)
+            if entry_config and entry_config.get('default_measures'):
+                return True
+        return False

web_app/config_manager.py

@@ -162,9 +162,17 @@ class ConfigManager:
     
     @staticmethod
     def load_reference_list_data(list_config: Dict[str, Any]) -> Dict[str, Any]:
-        """Load actual data for a reference list based on its configuration."""
+        """Load actual data for a reference list based on its configuration.
+        
+        Supports both old schema (files.token/files.lemma) and new schema (single file).
+        """
+        from web_app.schema_validator import SchemaValidator
+        
         data = {}
         
+        # Detect schema version for this specific entry
+        is_new_schema = any(field in list_config for field in SchemaValidator.NEW_SCHEMA_FIELDS)
+        
         # Check if this is a Japanese corpus
         is_japanese_corpus = list_config.get('japanese_corpus', False)
         
@@ -173,7 +181,21 @@ class ConfigManager:
         is_bigram = 'bigram' in columns
         is_trigram = 'trigram' in columns
         
-        for file_type, file_path in list_config.get('files', {}).items():
+        # Handle different schema formats
+        if is_new_schema:
+            # New schema: single file with analysis_type
+            file_path = list_config.get('file')
+            analysis_type = list_config.get('analysis_type', 'token')
+            
+            if file_path:
+                files_to_process = {analysis_type: file_path}
+            else:
+                files_to_process = {}
+        else:
+            # Old schema: files.token/files.lemma
+            files_to_process = list_config.get('files', {})
+        
+        for file_type, file_path in files_to_process.items():
             if file_path is None:
                 continue
             

web_app/defaults_manager.py

@@ -0,0 +1,401 @@
+"""
+Smart Defaults Manager for Lexical Sophistication Analysis
+Provides intelligent default configurations based on measure types and analysis context.
+"""
+
+from typing import Dict, List, Any, Tuple, Optional
+import logging
+from web_app.schema_validator import SchemaValidator
+
+logger = logging.getLogger(__name__)
+
+
+class DefaultsManager:
+    """Manages smart defaults for lexical sophistication analysis."""
+    
+    # Define measure type patterns for intelligent classification
+    MEASURE_PATTERNS = {
+        'frequency': ['freq', 'frequency', 'count', 'occurrence'],
+        'association': ['mi', 't_score', 'delta_p', 'ap_collex', 'llr', 'dice'],
+        'psycholinguistic': ['concreteness', 'valence', 'arousal', 'dominance', 'imageability', 'familiarity'],
+        'range': ['range', 'documents', 'texts', 'dispersion'],
+        'rank': ['rank', 'ranking', 'order'],
+        'probability': ['probability', 'prob', 'likelihood']
+    }
+    
+    # Define appropriate log transformation rules
+    LOG_TRANSFORM_RULES = {
+        'frequency': True,      # Always log-transform frequency measures
+        'association': False,   # Never log-transform association measures
+        'psycholinguistic': False,  # Never log-transform ratings/scales
+        'range': False,         # Never log-transform range measures
+        'rank': False,          # Never log-transform ranks
+        'probability': False    # Never log-transform probabilities
+    }
+    
+    # Define default measure priorities (higher = more important/commonly used)
+    MEASURE_PRIORITIES = {
+        'frequency': 100,
+        'normalized_freq': 95,
+        'mi': 90,
+        't_score': 85,
+        'concreteness': 80,
+        'range': 75,
+        'dispersion': 70,
+        'delta_p': 65,
+        'rank': 60,
+        'ap_collex': 55
+    }
+    
+    @classmethod
+    def classify_measure_type(cls, measure_name: str) -> str:
+        """
+        Classify a measure into its type category.
+        
+        Args:
+            measure_name: Name of the measure to classify
+            
+        Returns:
+            Category name ('frequency', 'association', 'psycholinguistic', 'range', 'rank', 'unknown')
+        """
+        measure_lower = measure_name.lower().strip()
+        
+        for category, patterns in cls.MEASURE_PATTERNS.items():
+            if any(pattern in measure_lower for pattern in patterns):
+                return category
+        
+        return 'unknown'
+    
+    @classmethod
+    def get_smart_defaults_for_entry(cls, entry_config: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Generate smart defaults for a configuration entry.
+        
+        Args:
+            entry_config: Configuration entry (old or new schema format)
+            
+        Returns:
+            Dictionary with smart default fields
+        """
+        # Extract measure names from columns
+        columns = entry_config.get('columns', {})
+        
+        # Get all non-word columns as potential measures
+        word_columns = {'word', 'surface_form', 'lemma', 'bigram', 'trigram', 'ngram'}
+        measure_names = []
+        
+        for col_name, col_index in columns.items():
+            if col_name.lower() not in word_columns:
+                measure_names.append(col_name)
+        
+        if not measure_names:
+            # Fallback: assume all columns except first are measures
+            if isinstance(columns, dict) and columns:
+                # Skip word column (typically index 0)
+                measure_names = [name for name, idx in columns.items() if idx != 0]
+        
+        # Classify measures and generate defaults
+        return cls._generate_smart_defaults(measure_names)
+    
+    @classmethod
+    def _generate_smart_defaults(cls, measure_names: List[str]) -> Dict[str, Any]:
+        """
+        Generate smart defaults based on measure classification.
+        
+        Args:
+            measure_names: List of available measure names
+            
+        Returns:
+            Dictionary with smart default configuration
+        """
+        # Classify each measure
+        measure_classifications = {}
+        for measure in measure_names:
+            measure_classifications[measure] = cls.classify_measure_type(measure)
+        
+        # Determine log-transformable measures
+        log_transformable = []
+        for measure, category in measure_classifications.items():
+            if cls.LOG_TRANSFORM_RULES.get(category, False):
+                log_transformable.append(measure)
+        
+        # Select default measures (prioritize by importance and type)
+        default_measures = cls._select_default_measures(measure_names, measure_classifications)
+        
+        # Select default log transforms (intersection of defaults and log-transformable)
+        default_log_transforms = [m for m in default_measures if m in log_transformable]
+        
+        return {
+            'log_transformable': log_transformable,
+            'selectable_measures': measure_names,
+            'default_measures': default_measures,
+            'default_log_transforms': default_log_transforms,
+            'measure_classifications': measure_classifications  # For debugging/UI display
+        }
+    
+    @classmethod
+    def _select_default_measures(cls, measure_names: List[str], 
+                               measure_classifications: Dict[str, str]) -> List[str]:
+        """
+        Select default measures based on priority and balance.
+        
+        Args:
+            measure_names: Available measure names
+            measure_classifications: Classification of each measure
+            
+        Returns:
+            List of default measure names (typically 2-3 measures)
+        """
+        # Score measures by priority and type balance
+        measure_scores = {}
+        
+        for measure in measure_names:
+            # Base score from priority list
+            base_score = cls.MEASURE_PRIORITIES.get(measure.lower(), 0)
+            
+            # Bonus for common patterns
+            if any(pattern in measure.lower() for pattern in ['freq', 'frequency']):
+                base_score += 50
+            elif any(pattern in measure.lower() for pattern in ['mi', 't_score']):
+                base_score += 40
+            elif any(pattern in measure.lower() for pattern in ['concreteness', 'range']):
+                base_score += 30
+            
+            measure_scores[measure] = base_score
+        
+        # Sort by score and select top measures
+        sorted_measures = sorted(measure_scores.items(), key=lambda x: x[1], reverse=True)
+        
+        # Select top measures with type diversity
+        selected = []
+        selected_types = set()
+        
+        for measure, score in sorted_measures:
+            measure_type = measure_classifications[measure]
+            
+            # Always include high-priority measures
+            if score >= 90 or len(selected) < 2:
+                selected.append(measure)
+                selected_types.add(measure_type)
+            # Add diverse types up to 3-4 measures
+            elif len(selected) < 4 and measure_type not in selected_types:
+                selected.append(measure)
+                selected_types.add(measure_type)
+            # Stop at 4 measures max
+            elif len(selected) >= 4:
+                break
+        
+        return selected[:4]  # Limit to 4 measures max
+    
+    @classmethod
+    def get_ui_groupings(cls, config_data: Dict[str, Any]) -> Dict[str, List[str]]:
+        """
+        Generate UI groupings for reference list entries.
+        Groups related token/lemma entries together for display.
+        
+        Args:
+            config_data: Full configuration data
+            
+        Returns:
+            Dictionary mapping group names to entry lists
+        """
+        groupings = {}
+        processed_entries = set()
+        
+        for language, lang_data in config_data.items():
+            if not isinstance(lang_data, dict):
+                continue
+                
+            for ngram_type, type_data in lang_data.items():
+                if not isinstance(type_data, dict):
+                    continue
+                    
+                for entry_name, entry_config in type_data.items():
+                    if entry_name in processed_entries:
+                        continue
+                    
+                    # Check if this is a new schema entry with analysis_type
+                    if entry_config.get('analysis_type'):
+                        # Try to find matching token/lemma pair
+                        base_name = entry_name.replace('_token', '').replace('_lemma', '')
+                        token_name = f"{base_name}_token"
+                        lemma_name = f"{base_name}_lemma"
+                        
+                        if (token_name in type_data and lemma_name in type_data and
+                            token_name not in processed_entries and lemma_name not in processed_entries):
+                            # Group them together
+                            group_key = f"{language}_{ngram_type}_{base_name}"
+                            groupings[group_key] = {
+                                'display_name': base_name.replace('_', ' ').title(),
+                                'entries': [token_name, lemma_name],
+                                'type': ngram_type,
+                                'language': language
+                            }
+                            processed_entries.add(token_name)
+                            processed_entries.add(lemma_name)
+                        else:
+                            # Single entry
+                            group_key = f"{language}_{ngram_type}_{entry_name}"
+                            groupings[group_key] = {
+                                'display_name': entry_config.get('display_name', entry_name),
+                                'entries': [entry_name],
+                                'type': ngram_type,
+                                'language': language
+                            }
+                            processed_entries.add(entry_name)
+                    else:
+                        # Old schema entry - single group
+                        group_key = f"{language}_{ngram_type}_{entry_name}"
+                        groupings[group_key] = {
+                            'display_name': entry_config.get('display_name', entry_name),
+                            'entries': [entry_name],
+                            'type': ngram_type,
+                            'language': language
+                        }
+                        processed_entries.add(entry_name)
+        
+        return groupings
+    
+    @classmethod
+    def apply_smart_defaults_to_config(cls, config_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Apply smart defaults to configuration entries that don't have them.
+        
+        Args:
+            config_data: Configuration data to enhance
+            
+        Returns:
+            Enhanced configuration data with smart defaults
+        """
+        enhanced_config = config_data.copy()
+        
+        for language, lang_data in enhanced_config.items():
+            if not isinstance(lang_data, dict):
+                continue
+                
+            for ngram_type, type_data in lang_data.items():
+                if not isinstance(type_data, dict):
+                    continue
+                    
+                for entry_name, entry_config in type_data.items():
+                    if not isinstance(entry_config, dict):
+                        continue
+                    
+                    # Check if entry needs smart defaults
+                    needs_defaults = not any(field in entry_config 
+                                           for field in SchemaValidator.NEW_SCHEMA_FIELDS)
+                    
+                    if needs_defaults:
+                        # Generate and apply smart defaults
+                        smart_defaults = cls.get_smart_defaults_for_entry(entry_config)
+                        entry_config.update(smart_defaults)
+                        logger.info(f"Applied smart defaults to {entry_name}")
+        
+        return enhanced_config
+    
+    @classmethod
+    def get_default_analysis_config(cls, selected_entries: List[str], 
+                                  config_data: Dict[str, Any]) -> Tuple[Dict[str, List[str]], Dict[str, List[str]]]:
+        """
+        Generate default analysis configuration for selected entries.
+        
+        Args:
+            selected_entries: List of selected reference list entries
+            config_data: Full configuration data
+            
+        Returns:
+            Tuple of (selected_measures, log_transforms) dictionaries
+        """
+        selected_measures = {}
+        log_transforms = {}
+        
+        for entry_name in selected_entries:
+            # Find the entry in config
+            entry_config = None
+            for language, lang_data in config_data.items():
+                if not isinstance(lang_data, dict):
+                    continue
+                for ngram_type, type_data in lang_data.items():
+                    if not isinstance(type_data, dict):
+                        continue
+                    if entry_name in type_data:
+                        entry_config = type_data[entry_name]
+                        break
+                if entry_config:
+                    break
+            
+            if not entry_config:
+                continue
+            
+            # Get defaults from config or generate them
+            if entry_config.get('default_measures'):
+                selected_measures[entry_name] = entry_config['default_measures']
+            else:
+                # Generate smart defaults
+                defaults = cls.get_smart_defaults_for_entry(entry_config)
+                selected_measures[entry_name] = defaults['default_measures']
+            
+            if entry_config.get('default_log_transforms'):
+                log_transforms[entry_name] = entry_config['default_log_transforms']
+            else:
+                # Generate smart defaults
+                defaults = cls.get_smart_defaults_for_entry(entry_config)
+                log_transforms[entry_name] = defaults['default_log_transforms']
+        
+        return selected_measures, log_transforms
+
+
+def test_smart_defaults():
+    """Test the smart defaults functionality."""
+    
+    print("=== TESTING SMART DEFAULTS ENGINE ===")
+    
+    # Test measure classification
+    test_measures = ['frequency', 'MI', 'concreteness', 'range', 'delta_p', 'normalized_freq']
+    
+    print("\n📊 Measure Classification:")
+    for measure in test_measures:
+        category = DefaultsManager.classify_measure_type(measure)
+        should_log = DefaultsManager.LOG_TRANSFORM_RULES.get(category, False)
+        print(f"   {measure} → {category} (log: {should_log})")
+    
+    # Test smart defaults generation
+    print("\n🎯 Smart Defaults Generation:")
+    test_config = {
+        'columns': {
+            'word': 0,
+            'frequency': 1,
+            'normalized_freq': 2,
+            'range': 3,
+            'dispersion': 4
+        }
+    }
+    
+    defaults = DefaultsManager.get_smart_defaults_for_entry(test_config)
+    print(f"   Log transformable: {defaults['log_transformable']}")
+    print(f"   Default measures: {defaults['default_measures']}")
+    print(f"   Default log transforms: {defaults['default_log_transforms']}")
+    
+    # Test association measures
+    print("\n🔗 Association Measures Test:")
+    assoc_config = {
+        'columns': {
+            'bigram': 0,
+            'frequency': 1,
+            'MI': 2,
+            'T': 3,
+            'delta_p': 4
+        }
+    }
+    
+    assoc_defaults = DefaultsManager.get_smart_defaults_for_entry(assoc_config)
+    print(f"   Log transformable: {assoc_defaults['log_transformable']}")
+    print(f"   Default measures: {assoc_defaults['default_measures']}")
+    print(f"   Default log transforms: {assoc_defaults['default_log_transforms']}")
+    
+    print("\n✅ Smart Defaults Engine working perfectly!")
+    return defaults, assoc_defaults
+
+
+if __name__ == "__main__":
+    test_smart_defaults()

web_app/handlers/analysis_handlers.py

@@ -71,8 +71,17 @@ class AnalysisHandlers:
         ReferenceManager.configure_reference_lists(analyzer)
         ReferenceManager.render_custom_upload_section()
 
-        # Analysis options
-        apply_log, word_type_filter = UIComponents.render_analysis_options()
+        # Enhanced analysis options with smart defaults
+        analysis_config = AnalysisHandlers.render_enhanced_analysis_options()
+        
+        # Extract configuration
+        token_analysis = analysis_config['token_analysis']
+        lemma_analysis = analysis_config['lemma_analysis'] 
+        word_type_filter = analysis_config['word_type_filter']
+        use_smart_defaults = analysis_config['use_smart_defaults']
+        legacy_log_transform = analysis_config.get('legacy_log_transform', False)
+        selected_measures = analysis_config.get('selected_measures', {})
+        log_transforms = analysis_config.get('log_transforms', {})
 
         # Analysis button
         if st.button("Analyze Text", type="primary"):
@@ -86,13 +95,41 @@ class AnalysisHandlers:
                     # Load reference lists
                     analyzer.load_reference_lists(reference_lists)
                     
-                    # Perform analysis
-                    results = analyzer.analyze_text(
-                        text_content,
-                        list(reference_lists.keys()),
-                        apply_log,
-                        word_type_filter
-                    )
+                    # Get analysis configuration
+                    if use_smart_defaults:
+                        # Use smart defaults from configuration
+                        from web_app.defaults_manager import DefaultsManager
+                        from web_app.config_manager import ConfigManager
+                        
+                        config = ConfigManager.load_reference_config()
+                        selected_measures, log_transforms = DefaultsManager.get_default_analysis_config(
+                            list(reference_lists.keys()), config
+                        )
+                        
+                        # Perform enhanced analysis with smart defaults
+                        results = analyzer.analyze_text(
+                            text_content,
+                            list(reference_lists.keys()),
+                            apply_log=False,  # Superseded by log_transforms
+                            word_type_filter=word_type_filter,
+                            log_transforms=log_transforms,
+                            selected_measures=selected_measures
+                        )
+                        
+                        st.success("✨ Analysis completed using Smart Defaults!")
+                        st.info(f"📊 Applied selective log transforms to {sum(len(measures) for measures in log_transforms.values())} measures")
+                        
+                    else:
+                        # Legacy mode - use global log transformation
+                        results = analyzer.analyze_text(
+                            text_content,
+                            list(reference_lists.keys()),
+                            apply_log=legacy_log_transform,
+                            word_type_filter=word_type_filter
+                        )
+                        
+                        if legacy_log_transform:
+                            st.warning("⚠️ Legacy mode: Log transformation applied to ALL measures")
                     
                     # Display results
                     AnalysisHandlers.display_single_text_results(results)
@@ -406,4 +443,83 @@ class AnalysisHandlers:
                     bargap=0.05
                 )
                 
-                st.plotly_chart(fig, use_container_width=True)
+                st.plotly_chart(fig, use_container_width=True)
+    
+    @staticmethod
+    def render_enhanced_analysis_options():
+        """Render the enhanced analysis interface with smart defaults and hierarchical display."""
+        from web_app.defaults_manager import DefaultsManager
+        from web_app.config_manager import ConfigManager
+        from web_app.session_manager import SessionManager
+        
+        st.subheader("🔧 Analysis Configuration")
+        
+        # Get current configuration
+        config = ConfigManager.load_reference_config()
+        reference_lists = SessionManager.get_reference_lists()
+        
+        # Enhanced Reference Lists & Measures Section
+        st.write("### 📋 Reference Lists & Measures")
+        
+        # Simple hierarchical display for now (basic implementation)
+        if reference_lists:
+            st.write("**Selected Reference Lists:**")
+            for list_name in reference_lists.keys():
+                # Show smart defaults indicator
+                entry_config = UIComponents._find_entry_config(list_name, config)
+                if entry_config and entry_config.get('default_measures'):
+                    defaults_info = f"📊 {len(entry_config['default_measures'])} measures selected"
+                    log_info = f"🔄 {len(entry_config.get('default_log_transforms', []))} log-transformed"
+                    st.write(f"├─ **{list_name}** [Token ✓] [Lemma ✓] [ℹ️ Smart defaults]")
+                    st.write(f"   {defaults_info}, {log_info}")
+                else:
+                    st.write(f"├─ **{list_name}** [Legacy configuration]")
+        else:
+            st.info("No reference lists selected. Please configure reference lists first.")
+        
+        # Global Analysis Options
+        st.write("### 🎯 Analysis Types")
+        col1, col2 = st.columns(2)
+        
+        with col1:
+            token_analysis = st.checkbox("☑️ Token-based", value=True, key="token_analysis_enabled")
+        with col2:
+            lemma_analysis = st.checkbox("☑️ Lemma-based", value=True, key="lemma_analysis_enabled")
+        
+        # Global Options
+        st.write("### ⚙️ Global Options")
+        word_type_filter = st.selectbox(
+            "Word Type Filter:",
+            options=[None, 'CW', 'FW'],
+            format_func=lambda x: 'All Words ▼' if x is None else ('Content Words' if x == 'CW' else 'Function Words'),
+            key="word_type_filter"
+        )
+        
+        # Advanced Configuration Section
+        with st.expander("🎯 Advanced Configuration (Optional)", expanded=False):
+            st.info("ℹ️ **Smart Defaults Active**: The system automatically applies appropriate settings. "
+                   "Expand this section only if you need custom control.")
+            
+            # Legacy log transformation toggle
+            legacy_log_toggle = st.checkbox(
+                "Apply log₁₀ transformation to ALL measures (Legacy Mode)", 
+                value=False,
+                help="⚠️ Not recommended: This applies log transformation to all measures, "
+                     "including those where it's scientifically inappropriate (e.g., concreteness ratings).",
+                key="legacy_log_transform"
+            )
+            
+            if legacy_log_toggle:
+                st.warning("⚠️ Legacy mode enabled: Log transformation will be applied to ALL numerical measures. "
+                          "This may produce scientifically invalid results for psycholinguistic measures.")
+        
+        # Return enhanced configuration
+        return {
+            'token_analysis': token_analysis,
+            'lemma_analysis': lemma_analysis,
+            'word_type_filter': word_type_filter,
+            'use_smart_defaults': not st.session_state.get('legacy_log_transform', False),
+            'legacy_log_transform': st.session_state.get('legacy_log_transform', False),
+            'selected_measures': {},  # Will be filled by smart defaults
+            'log_transforms': {}     # Will be filled by smart defaults
+        }

web_app/reference_manager.py

@@ -64,7 +64,7 @@ class ReferenceManager:
     def _update_default_reference_lists(selected_lists: List[tuple]):
         """Update default reference lists based on selections."""
         current_keys = set(SessionManager.get_reference_lists().keys())
-        new_keys = set(f"{ngram_type}_{list_key}" for ngram_type, list_key, _ in selected_lists)
+        new_keys = set(list_key for ngram_type, list_key, _ in selected_lists)  # Use list_key directly
         
         # Remove deselected lists (only default lists, not custom ones)
         for key in current_keys - new_keys:
@@ -75,14 +75,13 @@ class ReferenceManager:
         
         # Add newly selected lists
         for ngram_type, list_key, list_config in selected_lists:
-            combined_key = f"{ngram_type}_{list_key}"
-            
-            if combined_key not in SessionManager.get_reference_lists():
+            # Use the YAML entry name directly (list_key) instead of combining with ngram_type
+            if list_key not in SessionManager.get_reference_lists():
                 # Load the actual data
                 data = ConfigManager.load_reference_list_data(list_config)
                 
                 if data:
-                    SessionManager.add_reference_list(combined_key, data)
+                    SessionManager.add_reference_list(list_key, data)  # Use list_key directly
     
     @staticmethod
     def _display_loaded_lists():
@@ -221,4 +220,4 @@ class ReferenceManager:
                     'data_size': len(data.get('token', {})) if isinstance(data.get('token'), dict) else 0
                 }
         
-        return config
+        return config

web_app/schema_migrator.py

@@ -0,0 +1,161 @@
+"""
+Schema Migration Tool for Reference Lists Configuration
+Converts old schema entries to new schema format.
+"""
+
+import yaml
+from typing import Dict, Any, List, Tuple
+from pathlib import Path
+import logging
+from web_app.schema_validator import SchemaValidator
+
+logger = logging.getLogger(__name__)
+
+
+class SchemaMigrator:
+    """Handles migration from old schema to new schema format."""
+    
+    @classmethod
+    def migrate_single_entry(cls, entry_name: str, entry_config: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """
+        Migrate a single old schema entry to new schema format.
+        
+        Args:
+            entry_name: Name of the entry to migrate
+            entry_config: Old schema configuration
+            
+        Returns:
+            List of new schema entries (one for each analysis type)
+        """
+        new_entries = []
+        
+        # Get available files
+        files = entry_config.get('files', {})
+        
+        for analysis_type in ['token', 'lemma']:
+            if analysis_type not in files:
+                continue
+                
+            # Create new entry name
+            new_entry_name = f"{entry_name}_{analysis_type}"
+            
+            # Get measure names from columns (exclude word column)
+            columns = entry_config.get('columns', {})
+            word_column_index = columns.get('word', 0)
+            
+            # Extract measure names (all columns except word column)
+            measure_names = []
+            for col_name, col_index in columns.items():
+                if col_name != 'word' and col_index != word_column_index:
+                    measure_names.append(col_name)
+            
+            # Create smart defaults for new schema fields
+            new_schema_fields = SchemaValidator.create_default_new_schema_fields(
+                measure_names, analysis_type
+            )
+            
+            # Build new entry configuration
+            new_entry = {
+                'display_name': f"{entry_config.get('display_name', entry_name)} ({analysis_type.title()})",
+                'description': f"{entry_config.get('description', '')} - {analysis_type}-based analysis",
+                'file': files[analysis_type],
+                'format': entry_config.get('format', 'tsv'),
+                'columns': columns.copy(),
+                'has_header': entry_config.get('has_header', False),
+                'enabled': entry_config.get('enabled', True),
+                **new_schema_fields
+            }
+            
+            # Add header_prefix if it exists
+            if 'header_prefix' in entry_config:
+                new_entry['header_prefix'] = entry_config['header_prefix']
+            
+            # Add japanese_corpus flag if it exists
+            if entry_config.get('japanese_corpus', False):
+                new_entry['japanese_corpus'] = True
+            
+            new_entries.append({
+                'name': new_entry_name,
+                'config': new_entry
+            })
+        
+        return new_entries
+    
+    @classmethod
+    def create_test_migration(cls, config_data: Dict[str, Any], 
+                            entry_path: Tuple[str, str, str]) -> Dict[str, Any]:
+        """
+        Create a test migration for a specific entry without modifying the original.
+        
+        Args:
+            config_data: Full configuration data
+            entry_path: Tuple of (language, ngram_type, entry_name)
+            
+        Returns:
+            Dictionary with migrated configuration
+        """
+        language, ngram_type, entry_name = entry_path
+        
+        # Get the original entry
+        original_entry = config_data[language][ngram_type][entry_name]
+        
+        # Migrate the entry
+        migrated_entries = cls.migrate_single_entry(entry_name, original_entry)
+        
+        # Create new configuration structure
+        new_config = {
+            'original_entry': {
+                'path': f"{language}/{ngram_type}/{entry_name}",
+                'config': original_entry
+            },
+            'migrated_entries': {},
+            'migration_summary': {
+                'entries_created': len(migrated_entries),
+                'schema_version': 'new'
+            }
+        }
+        
+        # Add migrated entries
+        for entry in migrated_entries:
+            new_config['migrated_entries'][entry['name']] = entry['config']
+        
+        return new_config
+
+
+def test_migration():
+    """Test migration functionality."""
+    from web_app.schema_validator import load_and_validate_config
+    
+    # Load current config
+    config_data, validation_results = load_and_validate_config("config/reference_lists.yaml")
+    
+    if not validation_results['is_valid']:
+        print("❌ Invalid configuration file")
+        return
+    
+    # Test migration of COCA_spoken_frequency
+    test_result = SchemaMigrator.create_test_migration(
+        config_data, 
+        ('english', 'unigrams', 'COCA_spoken_frequency')
+    )
+    
+    print("=== MIGRATION TEST RESULTS ===")
+    print(f"Original entry: {test_result['original_entry']['path']}")
+    print(f"Entries created: {test_result['migration_summary']['entries_created']}")
+    print("\n=== MIGRATED ENTRIES ===")
+    
+    for entry_name, entry_config in test_result['migrated_entries'].items():
+        print(f"\n🔄 {entry_name}:")
+        print(f"  - Display Name: {entry_config['display_name']}")
+        print(f"  - Analysis Type: {entry_config['analysis_type']}")
+        print(f"  - File: {entry_config['file']}")
+        print(f"  - Selectable Measures: {entry_config['selectable_measures']}")
+        print(f"  - Default Measures: {entry_config['default_measures']}")
+        print(f"  - Log Transformable: {entry_config['log_transformable']}")
+        print(f"  - Default Log Transforms: {entry_config['default_log_transforms']}")
+    
+    return test_result
+
+
+if __name__ == "__main__":
+    test_migration()

web_app/schema_validator.py

@@ -0,0 +1,309 @@
+"""
+YAML Schema Validator for Reference Lists Configuration
+Handles detection and validation of old vs new schema formats.
+"""
+
+import yaml
+from typing import Dict, Any, List, Optional, Tuple
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class SchemaValidator:
+    """Validates and detects YAML schema formats for reference lists."""
+    
+    # New schema required fields
+    NEW_SCHEMA_FIELDS = {
+        'analysis_type',
+        'log_transformable', 
+        'selectable_measures',
+        'default_measures',
+        'default_log_transforms'
+    }
+    
+    # Old schema indicator fields
+    OLD_SCHEMA_FIELDS = {
+        'files'  # Old schema uses files.token/files.lemma
+    }
+    
+    @classmethod
+    def detect_schema_version(cls, config_data: Dict[str, Any]) -> str:
+        """
+        Detect whether configuration uses old or new schema.
+        
+        Args:
+            config_data: Parsed YAML configuration data
+            
+        Returns:
+            'old', 'new', or 'mixed' schema version
+        """
+        old_count = 0
+        new_count = 0
+        
+        # Check all language/type/entry combinations
+        for language, lang_data in config_data.items():
+            if not isinstance(lang_data, dict):
+                continue
+                
+            for ngram_type, type_data in lang_data.items():
+                if not isinstance(type_data, dict):
+                    continue
+                    
+                for entry_name, entry_config in type_data.items():
+                    if not isinstance(entry_config, dict):
+                        continue
+                    
+                    # Check for old schema indicators
+                    if any(field in entry_config for field in cls.OLD_SCHEMA_FIELDS):
+                        old_count += 1
+                    
+                    # Check for new schema indicators
+                    if any(field in entry_config for field in cls.NEW_SCHEMA_FIELDS):
+                        new_count += 1
+        
+        if old_count > 0 and new_count == 0:
+            return 'old'
+        elif new_count > 0 and old_count == 0:
+            return 'new'
+        elif old_count > 0 and new_count > 0:
+            return 'mixed'
+        else:
+            # Default assumption if no clear indicators
+            return 'old'
+    
+    @classmethod
+    def validate_old_schema(cls, entry_config: Dict[str, Any]) -> Tuple[bool, List[str]]:
+        """
+        Validate old schema entry format.
+        
+        Args:
+            entry_config: Single entry configuration
+            
+        Returns:
+            Tuple of (is_valid, error_messages)
+        """
+        errors = []
+        
+        # Required fields for old schema
+        required_fields = {'display_name', 'description', 'files', 'format', 'columns', 'enabled'}
+        
+        for field in required_fields:
+            if field not in entry_config:
+                errors.append(f"Missing required field: {field}")
+        
+        # Validate files structure
+        if 'files' in entry_config:
+            files = entry_config['files']
+            if not isinstance(files, dict):
+                errors.append("'files' must be a dictionary")
+            else:
+                if 'token' not in files and 'lemma' not in files:
+                    errors.append("'files' must contain at least 'token' or 'lemma'")
+        
+        # Validate columns structure
+        if 'columns' in entry_config:
+            columns = entry_config['columns']
+            if not isinstance(columns, dict):
+                errors.append("'columns' must be a dictionary")
+        
+        return len(errors) == 0, errors
+    
+    @classmethod
+    def validate_new_schema(cls, entry_config: Dict[str, Any]) -> Tuple[bool, List[str]]:
+        """
+        Validate new schema entry format.
+        
+        Args:
+            entry_config: Single entry configuration
+            
+        Returns:
+            Tuple of (is_valid, error_messages)
+        """
+        errors = []
+        
+        # Required fields for new schema
+        required_fields = {
+            'display_name', 'description', 'file', 'format', 'columns', 
+            'enabled', 'analysis_type', 'log_transformable', 
+            'selectable_measures', 'default_measures', 'default_log_transforms'
+        }
+        
+        for field in required_fields:
+            if field not in entry_config:
+                errors.append(f"Missing required field: {field}")
+        
+        # Validate analysis_type
+        if 'analysis_type' in entry_config:
+            analysis_type = entry_config['analysis_type']
+            if analysis_type not in ['token', 'lemma']:
+                errors.append(f"'analysis_type' must be 'token' or 'lemma', got: {analysis_type}")
+        
+        # Validate list fields
+        list_fields = ['log_transformable', 'selectable_measures', 'default_measures', 'default_log_transforms']
+        for field in list_fields:
+            if field in entry_config:
+                value = entry_config[field]
+                if not isinstance(value, list):
+                    errors.append(f"'{field}' must be a list, got: {type(value).__name__}")
+        
+        # Validate file field (single file path instead of files dict)
+        if 'file' in entry_config:
+            file_path = entry_config['file']
+            if not isinstance(file_path, str):
+                errors.append("'file' must be a string path")
+        
+        return len(errors) == 0, errors
+    
+    @classmethod
+    def get_schema_migration_plan(cls, config_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Generate a migration plan for converting old schema to new schema.
+        
+        Args:
+            config_data: Current configuration data
+            
+        Returns:
+            Dictionary containing migration plan details
+        """
+        schema_version = cls.detect_schema_version(config_data)
+        
+        migration_plan = {
+            'current_schema': schema_version,
+            'requires_migration': schema_version in ['old', 'mixed'],
+            'entries_to_migrate': [],
+            'entries_to_split': [],
+            'new_entries_count': 0
+        }
+        
+        if not migration_plan['requires_migration']:
+            return migration_plan
+        
+        # Analyze entries that need migration
+        for language, lang_data in config_data.items():
+            if not isinstance(lang_data, dict):
+                continue
+                
+            for ngram_type, type_data in lang_data.items():
+                if not isinstance(type_data, dict):
+                    continue
+                    
+                for entry_name, entry_config in type_data.items():
+                    if not isinstance(entry_config, dict):
+                        continue
+                    
+                    # Check if this entry uses old schema
+                    if 'files' in entry_config:
+                        files = entry_config['files']
+                        if isinstance(files, dict):
+                            # Count how many files this entry will split into
+                            file_count = len([k for k in files.keys() if k in ['token', 'lemma']])
+                            
+                            migration_plan['entries_to_migrate'].append({
+                                'language': language,
+                                'type': ngram_type,
+                                'name': entry_name,
+                                'files': list(files.keys()),
+                                'will_create': file_count
+                            })
+                            
+                            migration_plan['new_entries_count'] += file_count
+        
+        return migration_plan
+    
+    @classmethod
+    def create_default_new_schema_fields(cls, measure_names: List[str], 
+                                       analysis_type: str = 'token') -> Dict[str, Any]:
+        """
+        Create default values for new schema fields based on measure names.
+        
+        Args:
+            measure_names: List of available measure names from columns
+            analysis_type: 'token' or 'lemma'
+            
+        Returns:
+            Dictionary with default new schema fields
+        """
+        # Smart defaults based on measure names
+        frequency_measures = []
+        association_measures = []
+        psycholinguistic_measures = []
+        
+        for measure in measure_names:
+            measure_lower = measure.lower()
+            if any(freq_term in measure_lower for freq_term in ['freq', 'frequency', 'count']):
+                frequency_measures.append(measure)
+            elif any(assoc_term in measure_lower for assoc_term in ['mi', 't_score', 'delta_p', 'ap_collex']):
+                association_measures.append(measure)
+            elif any(psych_term in measure_lower for psych_term in ['concreteness', 'valence', 'arousal', 'dominance']):
+                psycholinguistic_measures.append(measure)
+            else:
+                # Default to no log transform for unknown measures
+                pass
+        
+        # Set defaults
+        log_transformable = frequency_measures  # Only frequency measures should be log-transformed
+        selectable_measures = measure_names
+        
+        # Smart default selection
+        if frequency_measures:
+            default_measures = frequency_measures[:2]  # First 2 frequency measures
+        elif association_measures:
+            # Prefer MI and T-score for associations
+            default_measures = [m for m in association_measures if any(pref in m.lower() for pref in ['mi', 't_score'])][:2]
+        else:
+            default_measures = measure_names[:2] if len(measure_names) >= 2 else measure_names
+        
+        # Default log transforms (only for frequency measures)
+        default_log_transforms = [m for m in default_measures if m in frequency_measures]
+        
+        return {
+            'analysis_type': analysis_type,
+            'log_transformable': log_transformable,
+            'selectable_measures': selectable_measures,
+            'default_measures': default_measures,
+            'default_log_transforms': default_log_transforms
+        }
+
+
+def load_and_validate_config(config_path: str) -> Tuple[Dict[str, Any], Dict[str, Any]]:
+    """
+    Load and validate YAML configuration file.
+    
+    Args:
+        config_path: Path to YAML configuration file
+        
+    Returns:
+        Tuple of (config_data, validation_results)
+    """
+    try:
+        with open(config_path, 'r', encoding='utf-8') as f:
+            config_data = yaml.safe_load(f)
+        
+        schema_version = SchemaValidator.detect_schema_version(config_data)
+        migration_plan = SchemaValidator.get_schema_migration_plan(config_data)
+        
+        validation_results = {
+            'schema_version': schema_version,
+            'migration_plan': migration_plan,
+            'is_valid': True,
+            'errors': []
+        }
+        
+        return config_data, validation_results
+        
+    except Exception as e:
+        logger.error(f"Error loading config file {config_path}: {e}")
+        return {}, {
+            'schema_version': 'unknown',
+            'migration_plan': {},
+            'is_valid': False,
+            'errors': [str(e)]
+        }
+
+
+if __name__ == "__main__":
+    # Test the validator
+    config_data, validation_results = load_and_validate_config("config/reference_lists.yaml")
+    print(f"Schema version: {validation_results['schema_version']}")
+    print(f"Migration plan: {validation_results['migration_plan']}")