json serial fixes

app.py

@@ -552,6 +552,8 @@ def make_json_serializable(obj, _stats=None):
     elif isinstance(obj, pd.Series):
         _stats['pandas_series'] += 1
         result = make_json_serializable(obj.tolist(), _stats)
+    elif isinstance(obj, pl.Path):
+        result = str(obj)
     elif isinstance(obj, pd.DataFrame):
         _stats['pandas_dataframe'] += 1
         result = make_json_serializable(obj.to_dict(orient='records'), _stats)
@@ -596,7 +598,7 @@ def call_subprocess(script_path, data):
                         f"{conversion_stats['fallback_none']} to None"
                     )
         
-        json_data_in = json.dumps(serializable_data)
+        json_data_in = json.dumps(serializable_data, cls=ut.NpEncoder)
 
         result = subprocess.run(["python", script_path], input=json_data_in, capture_output=True, text=True)
         ic(result.stderr)
@@ -2099,6 +2101,7 @@ def main():
             "trial_start_keyword_single_asc",
             "trial_end_keyword_single_asc",
             "close_gap_between_words_single_asc",
+            "close_gap_between_lines_single_asc",
             "paragraph_trials_only_single_asc",
             "discard_fixations_without_sfix_single_asc",
             "discard_far_out_of_text_fix_single_asc",
@@ -2123,7 +2126,7 @@ def main():
     if len(settings_to_save) > 0:
         single_file_tab_asc_tab.download_button(
             "⏬ Download all single .asc file settings as JSON",
-            json.dumps(settings_to_save),
+            json.dumps(settings_to_save, cls=ut.NpEncoder),
             "settings_to_save.json",
             "json",
             key="download_settings_to_save",
@@ -2268,7 +2271,7 @@ def main():
         for k, v in st.session_state["trials_dict_single_asc"].items():
             if not isinstance(v, dict):
                 trials_dict_for_showing[k] = v
-        single_file_tab_asc_tab.json(trials_dict_for_showing, expanded=False)
+        single_file_tab_asc_tab.json(ut.make_json_serializable(trials_dict_for_showing), expanded=False)
     if "trial_choices_single_asc" in st.session_state:
         single_file_tab_asc_tab.markdown("### Trial and algorithm selection")
         with single_file_tab_asc_tab.form(key="single_file_tab_asc_tab_trial_select_form"):
@@ -2314,7 +2317,7 @@ def main():
                     f'### Result dataframes for trial {st.session_state["trial_single_asc"]["trial_id"]}'
                 )
                 trial_expander_single = single_file_tab_asc_tab.expander("Show Trial Information", False)
-                trial_expander_single.json(filtered_trial, expanded=False)
+                trial_expander_single.json(ut.make_json_serializable(filtered_trial), expanded=False)
             events_df_expander_single = single_file_tab_asc_tab.expander("Show fixations and saccades before cleaning")
             events_df = st.session_state["events_df"].set_index("num").copy()
             events_df_expander_single.markdown("## Events before cleaning")
@@ -2390,7 +2393,7 @@ def main():
             filtered_trial = filter_trial_for_export(copy.deepcopy(trial))
             trial_expander_single = single_file_tab_asc_tab.expander("Show Trial Information", False)
             trial_expander_single.markdown(f'### Metadata for trial {trial["trial_id"]}')
-            trial_expander_single.json(filtered_trial, expanded=False)
+            trial_expander_single.json(ut.make_json_serializable(filtered_trial), expanded=False)
             if "saccade_df" not in st.session_state:
                 if st.session_state["dffix_single_asc"].shape[0] > 1:
                     saccade_df = get_saccade_df(
@@ -2464,7 +2467,7 @@ def main():
             )
             trial_expander_single.download_button(
                 "⏬ Download trial info as JSON",
-                json.dumps(filtered_trial),
+                json.dumps(filtered_trial, cls=ut.NpEncoder),
                 f'{filtered_trial["subject"]}_{filtered_trial["trial_id"]}.json',
                 "json",
                 key="download-trial_single_asc",
@@ -2791,7 +2794,7 @@ def main():
                                 "r",
                             ) as f:
                                 fixation_sequence_json = json.load(f)
-                            fixation_sequence_json_str = json.dumps(fixation_sequence_json)
+                            fixation_sequence_json_str = json.dumps(fixation_sequence_json, cls=ut.NpEncoder)
 
                             st.download_button(
                                 "⏬ Download fixations in eyekits format",
@@ -2804,7 +2807,7 @@ def main():
 
                             with open(f"results/textblock_eyekit_{subject}_{trial_id}.json", "r") as f:
                                 textblock_json = json.load(f)
-                            textblock_json_str = json.dumps(textblock_json)
+                            textblock_json_str = json.dumps(textblock_json, cls=ut.NpEncoder)
 
                             st.download_button(
                                 "⏬ Download stimulus in eyekits format",
@@ -2969,7 +2972,7 @@ def main():
                 )
             if in_st_nn("stimdf_single_csv"):
                 if ".json" in st.session_state["single_csv_file_stim"].name:
-                    st.json(st.session_state["stimdf_single_csv"], expanded=False)
+                    st.json(ut.make_json_serializable(st.session_state["stimdf_single_csv"]), expanded=False)
                 else:
                     st.dataframe(
                         st.session_state["stimdf_single_csv"],
@@ -3268,7 +3271,7 @@ def main():
 
                 with open(f'results/fixation_sequence_eyekit_{trial["trial_id"]}.json', "r") as f:
                     fixation_sequence_json = json.load(f)
-                fixation_sequence_json_str = json.dumps(fixation_sequence_json)
+                fixation_sequence_json_str = json.dumps(fixation_sequence_json, cls=ut.NpEncoder)
 
                 st.download_button(
                     "⏬ Download fixations in eyekits format",
@@ -3281,7 +3284,7 @@ def main():
 
                 with open(f'results/textblock_eyekit_{trial["trial_id"]}.json', "r") as f:
                     textblock_json = json.load(f)
-                textblock_json_str = json.dumps(textblock_json)
+                textblock_json_str = json.dumps(textblock_json, cls=ut.NpEncoder)
 
                 st.download_button(
                     "⏬ Download stimulus in eyekits format",
@@ -3397,7 +3400,7 @@ def main():
         if len(settings_to_save) > 0:
             st.download_button(
                 "⏬ Download all multi .asc file settings as JSON",
-                json.dumps(settings_to_save),
+                json.dumps(settings_to_save, cls=ut.NpEncoder),
                 "settings_to_save_multi_asc.json",
                 "json",
                 key="download_settings_to_save_multi_asc",
@@ -3756,7 +3759,7 @@ def main():
     if in_st_nn("all_fix_dfs_concat_multi_asc"):
         if "all_trials_by_subj" in st.session_state:
             multi_file_tab.markdown("### All meta data by subject and trial")
-            multi_file_tab.json(st.session_state["all_trials_by_subj"], expanded=False)
+            multi_file_tab.json(ut.make_json_serializable(st.session_state["all_trials_by_subj"]), expanded=False)
         multi_file_tab.markdown("### Item level stimulus overview")
         with multi_file_tab.popover("Column names definitions", help="Show column names and their definitions."):
             item_colnames_markdown = read_item_col_names()
@@ -4112,7 +4115,7 @@ def main():
     if in_st_nn("all_fix_dfs_concat_multi_csv"):
         if "all_trials_by_subj_csv" in st.session_state:
             multi_file_tab.markdown("### All meta data by subject and trial (CSV)")
-            multi_file_tab.json(st.session_state["all_trials_by_subj_csv"], expanded=False)
+            multi_file_tab.json(ut.make_json_serializable(st.session_state["all_trials_by_subj_csv"]), expanded=False)
         multi_file_tab.markdown("### Item level stimulus overview (CSV)")
         with multi_file_tab.popover("Column names definitions", help="Show column names and their definitions."):
             item_colnames_markdown = read_item_col_names()
@@ -4484,7 +4487,7 @@ def render_multi_trial_section(
         df_stim_expander.info("No stimulus information available for this trial.")
 
     filtered_trial = filter_trial_for_export(trial)
-    trial_info_expander.json(filtered_trial)
+    trial_info_expander.json(ut.make_json_serializable(filtered_trial))
 
     plot_expander = container.expander(f"Show corrected fixation plots{info_suffix}", True)
     show_plots_key = f"show_fix_sacc_plots_{suffix}"
@@ -4602,7 +4605,7 @@ def render_multi_trial_section(
 
             with open(f'results/fixation_sequence_eyekit_{trial["trial_id"]}.json', "r") as f:
                 fixation_sequence_json = json.load(f)
-            fixation_sequence_json_str = json.dumps(fixation_sequence_json)
+            fixation_sequence_json_str = json.dumps(fixation_sequence_json, cls=ut.NpEncoder)
 
             st.download_button(
                 "⏬ Download fixations in eyekits format",
@@ -4615,7 +4618,7 @@ def render_multi_trial_section(
 
             with open(f'results/textblock_eyekit_{trial["trial_id"]}.json', "r") as f:
                 textblock_json = json.load(f)
-            textblock_json_str = json.dumps(textblock_json)
+            textblock_json_str = json.dumps(textblock_json, cls=ut.NpEncoder)
 
             st.download_button(
                 "⏬ Download stimulus in eyekits format",
@@ -5141,7 +5144,7 @@ def show_cleaning_results(
                 st.pyplot(dffix_clean_fig)
             st.markdown("#### Fixations comparison before and after cleaning")
             if "Fixation Cleaning Stats" in trial:
-                st.json(trial["Fixation Cleaning Stats"])
+                st.json(ut.make_json_serializable(trial["Fixation Cleaning Stats"]))
         st.markdown("#### Cleaned fixations dataframe")
 
         st.dataframe(dffix_cleaned, height=200)

multi_proc_funcs.py

@@ -2093,11 +2093,17 @@ def _convert_to_json(obj):
     if isinstance(obj, (int, float, str, bool)):
         return obj
     elif isinstance(obj, dict):
-        return {k: _convert_to_json(v) for k, v in obj.items()}
-    elif isinstance(obj, list) or isinstance(obj, tuple):
+        return {str(k): _convert_to_json(v) for k, v in obj.items()}
+    elif isinstance(obj, (list, tuple)):
         return [_convert_to_json(item) for item in obj]
-    elif isinstance(obj, dict):
-        return {k: _convert_to_json(val) for k, val in obj.items()}
+    elif isinstance(obj, np.integer):
+        return int(obj)
+    elif isinstance(obj, np.floating):
+        return float(obj)
+    elif isinstance(obj, np.ndarray):
+        return _convert_to_json(obj.tolist())
+    elif isinstance(obj, (np.bool_, bool)):
+        return bool(obj)
     elif hasattr(obj, "to_dict"):
         return _convert_to_json(obj.to_dict())
     elif hasattr(obj, "tolist"):
@@ -2105,7 +2111,25 @@ def _convert_to_json(obj):
     elif obj is None:
         return None
     else:
-        raise TypeError(f"Object of type {type(obj)} is not JSON serializable")
+        try:
+            return str(obj)
+        except:
+            raise TypeError(f"Object of type {type(obj)} is not JSON serializable")
+
+
+class NpEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, np.integer):
+            return int(obj)
+        if isinstance(obj, np.floating):
+            return float(obj)
+        if isinstance(obj, np.ndarray):
+            return obj.tolist()
+        if isinstance(obj, (np.bool_, bool)):
+            return bool(obj)
+        if isinstance(obj, pl.Path):
+            return str(obj)
+        return super(NpEncoder, self).default(obj)
 
 
 def save_trial_to_json(trial, savename):
@@ -2117,7 +2141,7 @@ def save_trial_to_json(trial, savename):
             ic(f"Warning: Skipping non-serializable value for key '{key}' due to error: {e}")
 
     with open(savename, "w", encoding="utf-8") as f:
-        json.dump(filtered_trial, f, ensure_ascii=False, indent=4)
+        json.dump(filtered_trial, f, ensure_ascii=False, indent=4, cls=NpEncoder)
 
 
 def export_trial(trial: dict):

process_asc_files_in_multi_p.py

@@ -6,6 +6,7 @@ import os
 from multi_proc_funcs import DIST_MODELS_FOLDER, process_trial_choice, set_up_models
 import sys
 import pandas as pd
+import numpy as np
 
 
 logger = logging.getLogger('multiproc')
@@ -78,16 +79,37 @@ def process_asc_files_in_multi_proc(
 def make_json_compatible(obj):
     if isinstance(obj, dict):
         return {k: make_json_compatible(v) for k, v in obj.items()}
-    elif isinstance(obj, list):
+    elif isinstance(obj, (list, tuple)):
         return [make_json_compatible(v) for v in obj]
     elif isinstance(obj, pd.DataFrame):
         return obj.to_dict(orient="records")
     elif isinstance(obj, pd.Series):
         return obj.to_dict()
+    elif isinstance(obj, np.integer):
+        return int(obj)
+    elif isinstance(obj, np.floating):
+        return float(obj)
+    elif isinstance(obj, np.ndarray):
+        return make_json_compatible(obj.tolist())
+    elif isinstance(obj, (np.bool_, bool)):
+        return bool(obj)
     else:
         return obj
 
 
+class NpEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, np.integer):
+            return int(obj)
+        if isinstance(obj, np.floating):
+            return float(obj)
+        if isinstance(obj, np.ndarray):
+            return obj.tolist()
+        if isinstance(obj, (np.bool_, bool)):
+            return bool(obj)
+        return super(NpEncoder, self).default(obj)
+
+
 def main():
     try:
         input_data = sys.stdin.buffer.read()
@@ -149,13 +171,13 @@ def main():
             dffix = dffix.to_dict("records")
             trial = make_json_compatible(trial)
             out2.append((dffix, trial))
-        json_data_out = json.dumps(out2)
+        json_data_out = json.dumps(out2, cls=NpEncoder)
         logger.info("Finished appending")
         sys.stdout.flush()
         print(json_data_out)
     except Exception as e:
         logger.warning(e)
-        print(json.dumps({"error": str(e)}))
+        print(json.dumps({"error": str(e)}, cls=NpEncoder))
         with open("MULTI_ERROR.log",'w') as f:
             f.write(e)
 

utils.py

@@ -102,6 +102,56 @@ def download_url(url, target_filename):
                 return -1
 
 
+def make_json_serializable(obj):
+    """
+    Recursively convert non-JSON-serializable objects to serializable types.
+    """
+    if isinstance(obj, dict):
+        return {str(key): make_json_serializable(value) for key, value in obj.items()}
+    elif isinstance(obj, (list, tuple)):
+        return [make_json_serializable(item) for item in obj]
+    elif isinstance(obj, set):
+        return [make_json_serializable(item) for item in obj]
+    elif isinstance(obj, np.integer):
+        return int(obj)
+    elif isinstance(obj, np.floating):
+        return float(obj)
+    elif isinstance(obj, np.ndarray):
+        return make_json_serializable(obj.tolist())
+    elif isinstance(obj, (np.bool_, bool)):
+        return bool(obj)
+    elif isinstance(obj, pd.Series):
+        return make_json_serializable(obj.tolist())
+    elif isinstance(obj, pd.DataFrame):
+        return make_json_serializable(obj.to_dict(orient="records"))
+    elif isinstance(obj, bytes):
+        return obj.decode("utf-8", errors="replace")
+    elif isinstance(obj, pl.Path):
+        return str(obj)
+    elif isinstance(obj, (str, int, float, type(None))):
+        return obj
+    else:
+        try:
+            return str(obj)
+        except:
+            return None
+
+
+class NpEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, np.integer):
+            return int(obj)
+        if isinstance(obj, np.floating):
+            return float(obj)
+        if isinstance(obj, np.ndarray):
+            return obj.tolist()
+        if isinstance(obj, (np.bool_, bool)):
+            return bool(obj)
+        if isinstance(obj, pl.Path):
+            return str(obj)
+        return super(NpEncoder, self).default(obj)
+
+
 def asc_to_trial_ids(
     asc_file, close_gap_between_words,close_gap_between_lines, paragraph_trials_only, ias_files, trial_start_keyword, end_trial_at_keyword
 ):