Improve error handling, logging, and user feedback; add sidebar options for NPMI and top words

app.py

@@ -71,9 +71,9 @@ OUR_LABEL_FIELD = dataset_utils.OUR_LABEL_FIELD
 TOKENIZED_FIELD = dataset_utils.TOKENIZED_FIELD
 EMBEDDING_FIELD = dataset_utils.EMBEDDING_FIELD
 LENGTH_FIELD = dataset_utils.LENGTH_FIELD
-# TODO: Allow users to specify this.
-_MIN_VOCAB_COUNT = 10
-_SHOW_TOP_N_WORDS = 10
+# Allow users to specify these in the sidebar
+_MIN_VOCAB_COUNT_DEFAULT = 10
+_SHOW_TOP_N_WORDS_DEFAULT = 10
 
 
 @st.cache_resource(
@@ -100,28 +100,32 @@ def load_or_prepare(ds_args, show_embeddings, use_cache=False):
     if use_cache:
         logs.warning("Using cache")
     dstats = dataset_statistics.DatasetStatisticsCacheClass(CACHE_DIR, **ds_args, use_cache=use_cache)
-    logs.warning("Loading dataset")
-    dstats.load_or_prepare_dataset()
-    logs.warning("Loading labels")
-    dstats.load_or_prepare_labels()
-    logs.warning("Loading text lengths")
-    dstats.load_or_prepare_text_lengths()
-    logs.warning("Loading duplicates")
-    dstats.load_or_prepare_text_duplicates()
-    logs.warning("Loading vocabulary")
-    dstats.load_or_prepare_vocab()
-    logs.warning("Loading general statistics...")
-    dstats.load_or_prepare_general_stats()
-    if show_embeddings:
-        logs.warning("Loading Embeddings")
-        dstats.load_or_prepare_embeddings()
-    logs.warning("Loading nPMI")
     try:
-        dstats.load_or_prepare_npmi()
-    except:
-        logs.warning("Missing a cache for npmi")
-    logs.warning("Loading Zipf")
-    dstats.load_or_prepare_zipf()
+        logs.info("Loading dataset")
+        dstats.load_or_prepare_dataset()
+        logs.info("Loading labels")
+        dstats.load_or_prepare_labels()
+        logs.info("Loading text lengths")
+        dstats.load_or_prepare_text_lengths()
+        logs.info("Loading duplicates")
+        dstats.load_or_prepare_text_duplicates()
+        logs.info("Loading vocabulary")
+        dstats.load_or_prepare_vocab()
+        logs.info("Loading general statistics...")
+        dstats.load_or_prepare_general_stats()
+        if show_embeddings:
+            logs.info("Loading Embeddings")
+            dstats.load_or_prepare_embeddings()
+        logs.info("Loading nPMI")
+        try:
+            dstats.load_or_prepare_npmi()
+        except Exception as e:
+            logs.warning(f"Missing a cache for npmi: {e}")
+        logs.info("Loading Zipf")
+        dstats.load_or_prepare_zipf()
+    except Exception as e:
+        logs.error(f"Error during dataset preparation: {e}")
+        st.error(f"An error occurred during dataset preparation: {e}")
     return dstats
 
 @st.cache_resource(
@@ -136,9 +140,7 @@ def load_or_prepare_widgets(ds_args, show_embeddings, use_cache=False):
         ds_args:
         show_embeddings:
         use_cache:
-
     Returns:
-
     """
 
     if use_cache:
@@ -154,58 +156,56 @@ def load_or_prepare_widgets(ds_args, show_embeddings, use_cache=False):
     #except:
     #    logs.warning("We're screwed")
     if cache_dir_exists:
+        # Improved: catch and log specific exceptions, show user feedback
         try:
-            # We need to have the text_dset loaded for further load_or_prepare
             dstats.load_or_prepare_dataset()
-        except:
-            logs.warning("Missing a cache for load or prepare dataset")
+        except Exception as e:
+            logs.warning(f"Missing a cache for load or prepare dataset: {e}")
+            st.warning(f"Could not load or prepare dataset: {e}")
         try:
-            # Header widget
             dstats.load_or_prepare_dset_peek()
-        except:
-            logs.warning("Missing a cache for dset peek")
+        except Exception as e:
+            logs.warning(f"Missing a cache for dset peek: {e}")
         try:
-            # General stats widget
             dstats.load_or_prepare_general_stats()
-        except:
-            logs.warning("Missing a cache for general stats")
+        except Exception as e:
+            logs.warning(f"Missing a cache for general stats: {e}")
         try:
-            # Labels widget
             dstats.load_or_prepare_labels()
-        except:
-            logs.warning("Missing a cache for prepare labels")
+        except Exception as e:
+            logs.warning(f"Missing a cache for prepare labels: {e}")
         try:
-            # Text lengths widget
             dstats.load_or_prepare_text_lengths()
-        except:
-            logs.warning("Missing a cache for text lengths")
+        except Exception as e:
+            logs.warning(f"Missing a cache for text lengths: {e}")
         if show_embeddings:
             try:
-                # Embeddings widget
                 dstats.load_or_prepare_embeddings()
-            except:
-                logs.warning("Missing a cache for embeddings")
+            except Exception as e:
+                logs.warning(f"Missing a cache for embeddings: {e}")
         try:
             dstats.load_or_prepare_text_duplicates()
-        except:
-            logs.warning("Missing a cache for text duplicates")
+        except Exception as e:
+            logs.warning(f"Missing a cache for text duplicates: {e}")
         try:
             dstats.load_or_prepare_npmi()
-        except:
-            logs.warning("Missing a cache for npmi")
+        except Exception as e:
+            logs.warning(f"Missing a cache for npmi: {e}")
         try:
             dstats.load_or_prepare_zipf()
-        except:
-            logs.warning("Missing a cache for zipf")
+        except Exception as e:
+            logs.warning(f"Missing a cache for zipf: {e}")
     return dstats, cache_dir_exists
 
-def show_column(dstats, ds_name_to_dict, show_embeddings, column_id):
+def show_column(dstats, ds_name_to_dict, show_embeddings, column_id, min_vocab_count, show_top_n_words):
     """
     Function for displaying the elements in the right column of the streamlit app.
     Args:
         ds_name_to_dict (dict): the dataset name and options in dictionary form
         show_embeddings (Bool): whether embeddings should we loaded and displayed for this dataset
         column_id (str): what column of the dataset the analysis is done on
+        min_vocab_count (int): minimum vocabulary count for NPMI widget
+        show_top_n_words (int): number of top words to show
     Returns:
         The function displays the information using the functions defined in the st_utils class.
     """
@@ -222,7 +222,7 @@ def show_column(dstats, ds_name_to_dict, show_embeddings, column_id):
     st_utils.expander_text_duplicates(dstats, column_id)
     # Uses an interaction; handled a bit differently than other widgets.
     logs.info("showing npmi widget")
-    st_utils.npmi_widget(dstats.npmi_stats, _MIN_VOCAB_COUNT, column_id)
+    st_utils.npmi_widget(dstats.npmi_stats, min_vocab_count, column_id)
     logs.info("showing zipf")
     st_utils.expander_zipf(dstats.z, dstats.zipf_fig, column_id)
     if show_embeddings:
@@ -248,6 +248,23 @@ def main():
     # When not doing new development, use the cache.
     use_cache = True
     show_embeddings = st.sidebar.checkbox("Show text clusters")
+    # User-configurable options
+    min_vocab_count = st.sidebar.number_input(
+        "Minimum vocabulary count for NPMI widget",
+        min_value=1,
+        max_value=1000,
+        value=_MIN_VOCAB_COUNT_DEFAULT,
+        step=1,
+        help="Minimum number of occurrences for a word to be included in NPMI stats."
+    )
+    show_top_n_words = st.sidebar.number_input(
+        "Number of top words to show",
+        min_value=1,
+        max_value=100,
+        value=_SHOW_TOP_N_WORDS_DEFAULT,
+        step=1,
+        help="Number of top words to display in relevant widgets."
+    )
     # List of datasets for which embeddings are hard to compute:
 
     if compare_mode:
@@ -260,7 +277,7 @@ def main():
         )
         with left_col:
             if cache_exists_left:
-                show_column(dstats_left, ds_name_to_dict, show_embeddings, " A")
+                show_column(dstats_left, ds_name_to_dict, show_embeddings, " A", min_vocab_count, show_top_n_words)
             else:
                 st.markdown("### Missing pre-computed data measures!")
                 st.write(dataset_args_left)
@@ -269,7 +286,7 @@ def main():
         )
         with right_col:
             if cache_exists_right:
-                show_column(dstats_right, ds_name_to_dict, show_embeddings, " B")
+                show_column(dstats_right, ds_name_to_dict, show_embeddings, " B", min_vocab_count, show_top_n_words)
             else:
                 st.markdown("### Missing pre-computed data measures!")
                 st.write(dataset_args_right)
@@ -278,7 +295,7 @@ def main():
         dataset_args = st_utils.sidebar_selection(ds_name_to_dict, "")
         dstats, cache_exists = load_or_prepare_widgets(dataset_args, show_embeddings, use_cache=use_cache)
         if cache_exists:
-            show_column(dstats, ds_name_to_dict, show_embeddings, "")
+            show_column(dstats, ds_name_to_dict, show_embeddings, "", min_vocab_count, show_top_n_words)
         else:
             st.markdown("### Missing pre-computed data measures!")
             st.write(dataset_args)