Process nested JSON files

This code can normalize and flatten nested JSON files which can be useful for certain JSON files, especially logfiles. I also put in some code to help process large logfiles, although I may need to increase compute to process large files on this demo.

data_sources/upload_file.py

@@ -1,6 +1,20 @@
 import pandas as pd
 import sqlite3
 import csv
+import json
+import time
+
+def is_file_done_saving(file_path):
+    try:
+        with open(file_path, 'r') as f:
+            contents = f
+
+        if contents:
+            return True
+        else:
+            return False
+    except PermissionError:
+        return False
 
 def get_delimiter(file_path, bytes = 4096):
     sniffer = csv.Sniffer()
@@ -12,7 +26,14 @@ def read_file(file):
     if file.endswith(('.csv', '.tsv', '.txt')) :
         df = pd.read_csv(file, sep=get_delimiter(file))
     elif file.endswith('.json'):
-        df = pd.read_json(file)
+        with open(file, 'r') as f:
+            contents = json.load(f)
+        df = pd.json_normalize(contents)
+    elif file.endswith('.ndjson'):
+        with open(file, 'r') as f:
+            contents = f.read()
+        data = [json.loads(str(item)) for item in contents.strip().split('\n')]
+        df = pd.json_normalize(data)
     elif file.endswith('.xml'):
         df = pd.read_xml(file)
     elif file.endswith(('.xls','xlsx')):
@@ -22,6 +43,13 @@ def read_file(file):
     return df
 
 def process_data_upload(data_file, session_hash):
+    total_time = 0
+    while not is_file_done_saving(data_file):
+        total_time += .5
+        time.sleep(.5)
+        if total_time > 10:
+            break
+        
     df = read_file(data_file)
 
     # Read each sheet and store data in a DataFrame
@@ -34,6 +62,9 @@ def process_data_upload(data_file, session_hash):
     for column in df.columns:
         if "date" in column.lower() or "time" in column.lower():
             df[column] = pd.to_datetime(df[column])
+        if df[column].dtype == 'object' and isinstance(df[column].iloc[0], list):
+            df[column] = df[column].explode()
+
     
     connection = sqlite3.connect(f'data_source_{session_hash}.db')
     print("Opened database successfully");

functions/chat_functions.py

@@ -73,7 +73,7 @@ def example_display(input):
 
 css= ".file_marker .large{min-height:50px !important;} .example_btn{max-width:300px;}"
 
-with gr.Blocks(css=css) as demo:
+with gr.Blocks(css=css, delete_cache=(3600,3600)) as demo:
     title = gr.HTML("<h1 style='text-align:center;'>Virtual Data Analyst</h1>")
     description = gr.HTML("""<p style='text-align:center;'>Upload a data file and chat with our virtual data analyst 
                           to get insights on your data set. Currently accepts CSV, TSV, TXT, XLS, XLSX, XML, and JSON files. 
@@ -87,7 +87,7 @@ with gr.Blocks(css=css) as demo:
         example_btn_1 = gr.Button(value="Try Me: bank_marketing_campaign.csv", elem_classes="example_btn", size="md", variant="primary")
         example_btn_2 = gr.Button(value="Try Me: online_retail_data.csv", elem_classes="example_btn", size="md", variant="primary")
 
-    file_output = gr.File(label="Data File (CSV, TSV, TXT, XLS, XLSX, XML, JSON)", show_label=True, elem_classes="file_marker", file_types=['.csv','.xlsx','.txt','.json','.xml','.xls','.tsv'])
+    file_output = gr.File(label="Data File (CSV, TSV, TXT, XLS, XLSX, XML, JSON)", show_label=True, elem_classes="file_marker", file_types=['.csv','.xlsx','.txt','.json','.ndjson','.xml','.xls','.tsv'])
     example_btn_1.click(fn=run_example, inputs=example_file_1, outputs=file_output)
     example_btn_2.click(fn=run_example, inputs=example_file_2, outputs=file_output)
     file_output.change(fn=example_display, inputs=file_output, outputs=[example_btn_1, example_btn_2])

functions/sqlite_functions.py

@@ -1,6 +1,10 @@
 from typing import List
 from haystack import component
 import pandas as pd
+pd.set_option('display.max_rows', None)
+pd.set_option('display.max_columns', None)
+pd.set_option('display.width', None)
+pd.set_option('display.max_colwidth', None)
 import sqlite3
 
 @component