added span frequency feature ranking

utils/gram2vec_feat_utils.py

@@ -49,7 +49,17 @@ def get_shorthand(feature_str: str) -> str:
         return None
     if category not in FEATURE_HANDLERS:
         return None
-    code = load_code_map().get(human)
+    code_map = load_code_map()
+    code = code_map.get(human)
+    if code is None:
+        # Try normalizing terminology shown in UI
+        # Convert 'Preposition' phrasing back to 'Adposition' used in the code map
+        human_alt = (human
+                     .replace("Preposition", "Adposition")
+                     .replace("preposition", "adposition")
+                     .replace("Prepositional", "Adpositional")
+                     .replace("prepositional", "adpositional"))
+        code = code_map.get(human_alt)
     if code is None:
         # print(f"Warning: No code found for human-readable feature '{human}'")
         return None  # fallback to the human-readable name
@@ -78,6 +88,14 @@ def get_fullform(shorthand: str) -> str:
     if human is None:
         return None
 
+    # Normalize terminology for UI: prefer "Preposition" over "Adposition"
+    # Also handle potential "adpositional" variants if present
+    human = (human
+             .replace("Adposition", "Preposition")
+             .replace("adposition", "preposition")
+             .replace("Adpositional", "Prepositional")
+             .replace("adpositional", "prepositional"))
+
     return f"{category}:{human}"
 
 def highlight_both_spans(text, llm_spans, gram_spans):
@@ -169,8 +187,9 @@ def show_combined_spans_all(selected_feature_llm, selected_feature_g2v,
     if selected_feature_g2v and selected_feature_g2v != "None":
         # get gram2vec spans
         gram_spans_list = []
-        # clean the display string and get the feature name without the zscore
-        selected_feature_g2v = selected_feature_g2v.split(" | [Z=")[0].strip()
+        # In case any old label formatting with z-scores leaks through, strip it defensively
+        if "| [Z=" in selected_feature_g2v:
+            selected_feature_g2v = selected_feature_g2v.split(" | [Z=")[0].strip()
         print(f"Selected Gram2Vec feature: {selected_feature_g2v}")
         short = get_shorthand(selected_feature_g2v)
         print(f"short hand: {short}")

utils/interp_space_utils.py

@@ -17,6 +17,8 @@ from pydantic import BaseModel
 from pydantic import ValidationError
 import time
 from utils.llm_feat_utils import generate_feature_spans_cached
+from utils.gram2vec_feat_utils import get_shorthand, get_fullform
+from gram2vec.feature_locator import find_feature_spans
 from collections import Counter
 import numpy as np
 from sklearn.metrics.pairwise import cosine_similarity, euclidean_distances
@@ -633,6 +635,7 @@ def compute_clusters_g2v_representation(
     other_author_ids: List[Any],
     features_clm_name: str,
     top_n: int = 10,
+    max_candidates_for_span_sorting: int = 50,
 ) -> List[tuple]:  # Changed return type to List[tuple] to include scores
 
     # 1) Identify selected authors in the zoom region
@@ -666,8 +669,54 @@ def compute_clusters_g2v_representation(
     # 5) Rank features by mean z-score, keep positives only
     feature_scores = [(feat, float(score)) for feat, score in zip(all_features, selected_mean) if score > 0]
     feature_scores.sort(key=lambda x: x[1], reverse=True)
-
-    return feature_scores[:top_n]
+    
+    # 6) Extract top candidates for span-based sorting
+    candidate_features = feature_scores[:max_candidates_for_span_sorting]
+    
+    # 7) Extract spans for task authors to sort by frequency
+    task_author_names = {'Mystery author', 'Candidate Author 1', 'Candidate Author 2', 'Candidate Author 3'}
+    task_authors_in_selection = [aid for aid in author_ids if aid in task_author_names]
+    
+    if not task_authors_in_selection:
+        # If no task authors in selection, just return the z-score sorted features
+        print("[INFO] No task authors in selection, returning z-score sorted features")
+        return feature_scores[:top_n]
+    
+    # Get task author data
+    task_authors_df = background_corpus_df[background_corpus_df['authorID'].isin(task_authors_in_selection)]
+    
+    # Count spans for each feature across task authors
+    feature_span_counts = {}
+    for feat_shorthand, z_score in candidate_features:
+        span_count = 0
+        
+        # Convert shorthand to human-readable for display (if needed)
+        # Note: features in gram2vec dict are in shorthand format like "pos_unigrams:ADJ"
+        
+        for _, author_row in task_authors_df.iterrows():
+            author_text = author_row['fullText']
+            if isinstance(author_text, list):
+                author_text = '\n\n'.join(author_text)
+            
+            try:
+                # find_feature_spans expects shorthand format like "pos_unigrams:ADJ"
+                spans = find_feature_spans(author_text, feat_shorthand)
+                span_count += len(spans)
+            except Exception as e:
+                # If span extraction fails, continue with 0 spans for this author
+                pass
+        
+        feature_span_counts[feat_shorthand] = span_count
+    
+    # 8) Sort features by span frequency, then by z-score as tiebreaker
+    sorted_by_spans = sorted(
+        candidate_features,
+        key=lambda x: (-feature_span_counts.get(x[0], 0), -x[1])
+    )
+    
+    print(f"[INFO] Sorted gram2vec features by span frequency: {[(f, feature_span_counts.get(f, 0), z) for f, z in sorted_by_spans[:top_n]]}")
+    
+    return sorted_by_spans[:top_n]
 
 def generate_interpretable_space_representation(interp_space_path, styles_df_path, feat_clm, output_clm, num_feats=5):