profiler

pages/profiler.py

@@ -1,6 +1,5 @@
-import streamlit as st
 from utils.menu import menu_with_redirect
-# from pages.solution import data_visualize
+from pages.solution import data_profiler
 
 menu_with_redirect()
-st.title("this is data profiler")
+data_profiler()

pages/solution.py

@@ -6,6 +6,8 @@ import boto3
 import time
 import pandas as pd
 import duckdb
+import ydata_profiling
+from streamlit_pandas_profiling import st_profile_report
 from pygwalker.api.streamlit import StreamlitRenderer
 import streamlit.components.v1 as components
 from openai import AzureOpenAI
@@ -1149,176 +1151,182 @@ def design_insight():
             st.session_state['data_prompt_value']= ''
             st.session_state['graph_prompt_value']= ''
 
-        col1, col2 = st.columns([1, 3])
-        with col1:
-            with st.container():
-                st.subheader('Dataset Columns')
-                s = selected_query[len("ID: "):]
-                end_index = s.find(",")
-                id = s[:end_index]
-                try:
-                    blob_content = getBlobContent(f"{query_lib}{st.session_state.userId}/{id}.json")
-                    content = json.loads(blob_content)
-                    st.session_state['query_file_content'] = content
-                    sql_query = content['sql']
-                    selected_db = content['database']
-                    df = execute_sql(sql_query, selected_db)
-                    df = drop_duplicate_columns(df)
-                    df_dict = get_column_types(df)
-                    df_dtypes = pd.DataFrame.from_dict(df_dict, orient='index', columns=['Dtype'])
-                    df_dtypes.reset_index(inplace=True)
-                    df_dtypes.rename(columns={'index': 'Column'}, inplace=True)
-
-                    int_cols = df_dtypes[df_dtypes['Dtype'] == 'int64']['Column'].reset_index(drop=True)
-                    float_cols = df_dtypes[df_dtypes['Dtype'] == 'float64']['Column'].reset_index(drop=True)
-                    string_cols = df_dtypes[df_dtypes['Dtype'] == 'string']['Column'].reset_index(drop=True)
-                    datetime_cols = df_dtypes[df_dtypes['Dtype'] == 'datetime']['Column'].reset_index(drop=True)
+        # col1, col2 = st.columns([1, 3])
+        # with col1:
+        with st.container():
+            st.subheader('Dataset Columns')
+            s = selected_query[len("ID: "):]
+            end_index = s.find(",")
+            id = s[:end_index]
+            try:
+                blob_content = getBlobContent(f"{query_lib}{st.session_state.userId}/{id}.json")
+                content = json.loads(blob_content)
+                st.session_state['query_file_content'] = content
+                sql_query = content['sql']
+                selected_db = content['database']
+                df = execute_sql(sql_query, selected_db)
+                df = drop_duplicate_columns(df)
+                df_dict = get_column_types(df)
+                df_dtypes = pd.DataFrame.from_dict(df_dict, orient='index', columns=['Dtype'])
+                df_dtypes.reset_index(inplace=True)
+                df_dtypes.rename(columns={'index': 'Column'}, inplace=True)
+
+                int_cols = df_dtypes[df_dtypes['Dtype'] == 'int64']['Column'].reset_index(drop=True)
+                float_cols = df_dtypes[df_dtypes['Dtype'] == 'float64']['Column'].reset_index(drop=True)
+                string_cols = df_dtypes[df_dtypes['Dtype'] == 'string']['Column'].reset_index(drop=True)
+                datetime_cols = df_dtypes[df_dtypes['Dtype'] == 'datetime']['Column'].reset_index(drop=True)
 
+                col1, col2, col3, col4 = st.columns(4)
+
+                with col1:
                     with st.expander("Integer Columns", icon=":material/looks_one:"):
                         st.write("\n\n".join(list(int_cols.values)))
 
-                    with st.expander("Decimal Number Columns", icon=":material/pin:"):
+                with col2:
+                    with st.expander("Decimal Columns", icon=":material/pin:"):
                         st.write("\n\n".join(list(float_cols.values)))
 
+                with col3:
                     with st.expander("String Columns", icon=":material/abc:"):
                         st.write("\n\n".join(list(string_cols.values)))
 
+                with col4:
                     with st.expander("Datetime Columns", icon=":material/calendar_month:"):
                         st.write("\n\n".join(list(datetime_cols.values)))
 
-                    st.session_state['explore_df'] = df
-                    st.session_state['explore_dtype'] = df_dtypes
+                st.session_state['explore_df'] = df
+                st.session_state['explore_dtype'] = df_dtypes
 
-                    logger.info("Dataset columns displayed using AG Grid.")
-                except Exception as e:
-                    st.error("Error while loading the dataset")
-                    logger.error("Error loading dataset: {}", e)
-
-        with col2:
-            with st.container():
-                st.subheader('Generate Insight')
-                # data_prompt_value = st.session_state.get('data_prompt', '')
-                data_prompt = st.text_area("What insight would you like to generate?")#, value=data_prompt_value)
-                if st.button('Generate Insight'):
-                    st.session_state['data_obj'] = None
-                    if data_prompt:
-                        st.session_state['data_prompt'] = data_prompt
-                        try:
-                            query, method_num = answer_guide_question(data_prompt, st.session_state['explore_df'], st.session_state['explore_dtype'], selected_db)
-                            if query:
-                                try:
-                                    mydf = df
-                                    st.session_state['query'] = query
-                                    result_df = duckdb.query(query).to_df()
-                                    st.session_state['data_obj'] = result_df
-                                    logger.info("Insight generated and displayed using AG Grid.")
-                                    # st.session_state['data_prompt'] = ''  # Clear the input field
-                                except Exception as e:
-                                    st.write('Error executing the query. Please try again.')
-                                    logger.error("Error executing the query: %s", e)
-                            else:
-                                st.write('Please retry again.')
-                            del st.session_state['code_execution_error']
-                        except Exception as e:
-                            st.write("Please try again with another prompt")
-                            logger.error("Error generating insight: %s", e)
-                if st.session_state['data_obj'] is not None:
-                    # st.text(st.session_state['data_prompt'])
-                    display_paginated_dataframe(st.session_state['data_obj'], "ag_grid_insight")
+                logger.info("Dataset columns displayed using AG Grid.")
+            except Exception as e:
+                st.error("Error while loading the dataset")
+                logger.error("Error loading dataset: {}", e)
+
+        # with col2:
+        with st.container():
+            st.subheader('Generate Insight')
+            # data_prompt_value = st.session_state.get('data_prompt', '')
+            data_prompt = st.text_area("What insight would you like to generate?")#, value=data_prompt_value)
+            if st.button('Generate Insight'):
+                st.session_state['data_obj'] = None
+                if data_prompt:
                     st.session_state['data_prompt'] = data_prompt
-                    
-            with st.container():
-                st.subheader('Generate Graph')
-                # graph_prompt_value = st.session_state.get('graph_prompt', '')
-                graph_prompt = st.text_area("What graph would you like to generate?")#, value=graph_prompt_value)
-                if st.button('Generate Graph'):
-                    graph_obj = None
-                    if graph_prompt:
-                        logger.debug("Graph prompt: %s | Previous graph prompt: %s", st.session_state.get('graph_prompt'), graph_prompt)
-                        if st.session_state['graph_prompt'] != graph_prompt:
+                    try:
+                        query, method_num = answer_guide_question(data_prompt, st.session_state['explore_df'], st.session_state['explore_dtype'], selected_db)
+                        if query:
                             try:
-                                graph_obj, st.session_state['graph_code'] = generate_graph(graph_prompt, st.session_state['explore_df'], st.session_state['explore_dtype'], selected_db)
-                                st.session_state['graph_obj'] = graph_obj
-
-                                if graph_obj is not None:
-                                    # st.text(st.session_state['graph_prompt'])
-                                    st.plotly_chart(graph_obj, use_container_width=True)
-                                    logger.info("Graph generated and displayed using Plotly.")
-                                else:
-                                    st.session_state['graph_obj'] = None
-                                    st.text('Error in generating graph, please try again.')
+                                mydf = df
+                                st.session_state['query'] = query
+                                result_df = duckdb.query(query).to_df()
+                                st.session_state['data_obj'] = result_df
+                                logger.info("Insight generated and displayed using AG Grid.")
+                                # st.session_state['data_prompt'] = ''  # Clear the input field
                             except Exception as e:
-                                logger.error("Error in generating graph: %s", e)
-                                st.write("Error in generating graph, please try again")
+                                st.write('Error executing the query. Please try again.')
+                                logger.error("Error executing the query: %s", e)
                         else:
-                            try:
-                                st.plotly_chart(st.session_state['graph_obj'], use_container_width=True)
-                            except Exception as e:
-                                st.write("Error in displaying graph, please try again")
-                    st.session_state['graph_prompt'] = graph_prompt
-                else:
-                    if st.session_state['graph_obj'] is not None:
+                            st.write('Please retry again.')
+                        del st.session_state['code_execution_error']
+                    except Exception as e:
+                        st.write("Please try again with another prompt")
+                        logger.error("Error generating insight: %s", e)
+            if st.session_state['data_obj'] is not None:
+                # st.text(st.session_state['data_prompt'])
+                display_paginated_dataframe(st.session_state['data_obj'], "ag_grid_insight")
+                st.session_state['data_prompt'] = data_prompt
+                
+        with st.container():
+            st.subheader('Generate Graph')
+            # graph_prompt_value = st.session_state.get('graph_prompt', '')
+            graph_prompt = st.text_area("What graph would you like to generate?")#, value=graph_prompt_value)
+            if st.button('Generate Graph'):
+                graph_obj = None
+                if graph_prompt:
+                    logger.debug("Graph prompt: %s | Previous graph prompt: %s", st.session_state.get('graph_prompt'), graph_prompt)
+                    if st.session_state['graph_prompt'] != graph_prompt:
+                        try:
+                            graph_obj, st.session_state['graph_code'] = generate_graph(graph_prompt, st.session_state['explore_df'], st.session_state['explore_dtype'], selected_db)
+                            st.session_state['graph_obj'] = graph_obj
+
+                            if graph_obj is not None:
+                                # st.text(st.session_state['graph_prompt'])
+                                st.plotly_chart(graph_obj, use_container_width=True)
+                                logger.info("Graph generated and displayed using Plotly.")
+                            else:
+                                st.session_state['graph_obj'] = None
+                                st.text('Error in generating graph, please try again.')
+                        except Exception as e:
+                            logger.error("Error in generating graph: %s", e)
+                            st.write("Error in generating graph, please try again")
+                    else:
                         try:
                             st.plotly_chart(st.session_state['graph_obj'], use_container_width=True)
                         except Exception as e:
                             st.write("Error in displaying graph, please try again")
-                            logger.error("Error in displaying graph: %s", e)                      
-            with st.container():
-                if 'graph_obj' in st.session_state or 'data_obj' in st.session_state:
-                    user_persona = st.selectbox('Select a persona to save the result of your exploration', persona_list)
-                    insight_desc = st.text_area(label='Describe the purpose of this insight for your reference later')
-                    if st.button('Save in Library'):
-                        base_prompt = st.session_state['query_file_content']['prompt']
-                        base_code = st.session_state['query_file_content']['sql']
+                st.session_state['graph_prompt'] = graph_prompt
+            else:
+                if st.session_state['graph_obj'] is not None:
+                    try:
+                        st.plotly_chart(st.session_state['graph_obj'], use_container_width=True)
+                    except Exception as e:
+                        st.write("Error in displaying graph, please try again")
+                        logger.error("Error in displaying graph: %s", e)                      
+        with st.container():
+            if 'graph_obj' in st.session_state or 'data_obj' in st.session_state:
+                user_persona = st.selectbox('Select a persona to save the result of your exploration', persona_list)
+                insight_desc = st.text_area(label='Describe the purpose of this insight for your reference later')
+                if st.button('Save in Library'):
+                    base_prompt = st.session_state['query_file_content']['prompt']
+                    base_code = st.session_state['query_file_content']['sql']
 
-                        insight_prompt = st.session_state.get('data_prompt', '')
-                        insight_code = st.session_state.get('query', '')
+                    insight_prompt = st.session_state.get('data_prompt', '')
+                    insight_code = st.session_state.get('query', '')
 
-                        chart_prompt = st.session_state.get('graph_prompt', '')
-                        chart_code = st.session_state.get('graph_code', '')
+                    chart_prompt = st.session_state.get('graph_prompt', '')
+                    chart_code = st.session_state.get('graph_code', '')
 
-                        try:
-                            result = get_existing_insight(base_code, user_persona)
-                            if result:
-                                existing_insight, file_number = result 
-                                if insight_prompt and insight_code is not None:
-                                    existing_insight['prompt'][f'prompt_{len(existing_insight["prompt"]) + 1}'] = {
-                                        'insight_prompt': insight_prompt,
-                                        'insight_code': insight_code
-                                    }
-                                if chart_prompt and chart_code is not None:
-                                    existing_insight['chart'][f'chart_{len(existing_insight["chart"]) + 1}'] = {
-                                        'chart_prompt': chart_prompt,
-                                        'chart_code': chart_code
-                                    }
-                                try:
-                                    update_insight(existing_insight, user_persona, file_number)
-                                    st.text('Insight updated with new Graph and/or Data.')
-                                    logger.info("Insight updated successfully.")
-                                except Exception as e:
-                                    st.write('Could not update the insight file. Please try again')
-                                    logger.error("Error while updating insight file: {}", e)
-                            else:
-                                # Create a new insight entry
-                                if not check_blob_exists(f"insight_library/{user_persona}/{st.session_state.userId}"):
-                                    blob_service_client = BlobServiceClient.from_connection_string(connection_string)
-                                    container_client = blob_service_client.get_container_client(container_name)
-                                    logger.info("Creating a new folder in the blob storage:", f"insight_library/{user_persona}/{st.session_state.userId}")
-                                    folder_path = f"insight_library/{user_persona}/{st.session_state.userId}/"
-                                    container_client.upload_blob(folder_path, data=b'')
-                                next_file_number = get_max_blob_num(f"insight_library/{user_persona}/{st.session_state.userId}/") + 1
-                                # logger.info(f"Next file number: {next_file_number}")
-
-                                try:
-                                    save_insight(next_file_number, user_persona, insight_desc, base_prompt, base_code,selected_db, insight_prompt, insight_code, chart_prompt, chart_code)
-                                    st.text(f'Insight #{next_file_number} with Graph and/or Data saved.')
-                                    # logger.info(f'Insight #{next_file_number} with Graph and/or Data saved.')
-                                except Exception as e:
-                                    st.write('Could not write the insight file.')
-                                    logger.error(f"Error while writing insight file: {e}")
-                        except Exception as e:
-                            st.write(f"Please try again")
-                            logger.error(f"Error checking existing insights: {e}")
+                    try:
+                        result = get_existing_insight(base_code, user_persona)
+                        if result:
+                            existing_insight, file_number = result 
+                            if insight_prompt and insight_code is not None:
+                                existing_insight['prompt'][f'prompt_{len(existing_insight["prompt"]) + 1}'] = {
+                                    'insight_prompt': insight_prompt,
+                                    'insight_code': insight_code
+                                }
+                            if chart_prompt and chart_code is not None:
+                                existing_insight['chart'][f'chart_{len(existing_insight["chart"]) + 1}'] = {
+                                    'chart_prompt': chart_prompt,
+                                    'chart_code': chart_code
+                                }
+                            try:
+                                update_insight(existing_insight, user_persona, file_number)
+                                st.text('Insight updated with new Graph and/or Data.')
+                                logger.info("Insight updated successfully.")
+                            except Exception as e:
+                                st.write('Could not update the insight file. Please try again')
+                                logger.error("Error while updating insight file: {}", e)
+                        else:
+                            # Create a new insight entry
+                            if not check_blob_exists(f"insight_library/{user_persona}/{st.session_state.userId}"):
+                                blob_service_client = BlobServiceClient.from_connection_string(connection_string)
+                                container_client = blob_service_client.get_container_client(container_name)
+                                logger.info("Creating a new folder in the blob storage:", f"insight_library/{user_persona}/{st.session_state.userId}")
+                                folder_path = f"insight_library/{user_persona}/{st.session_state.userId}/"
+                                container_client.upload_blob(folder_path, data=b'')
+                            next_file_number = get_max_blob_num(f"insight_library/{user_persona}/{st.session_state.userId}/") + 1
+                            # logger.info(f"Next file number: {next_file_number}")
+
+                            try:
+                                save_insight(next_file_number, user_persona, insight_desc, base_prompt, base_code,selected_db, insight_prompt, insight_code, chart_prompt, chart_code)
+                                st.text(f'Insight #{next_file_number} with Graph and/or Data saved.')
+                                # logger.info(f'Insight #{next_file_number} with Graph and/or Data saved.')
+                            except Exception as e:
+                                st.write('Could not write the insight file.')
+                                logger.error(f"Error while writing insight file: {e}")
+                    except Exception as e:
+                        st.write(f"Please try again")
+                        logger.error(f"Error checking existing insights: {e}")
 
 def get_insight_list(persona):
     try:
@@ -1427,7 +1435,7 @@ def data_visualize():
         st.image('logo.png')
     with col_bb:
         st.subheader("InsightLab - Data Visualize", divider='blue')
-        st.markdown('**Select a dataset that you generated and visualize the dataset.**')
+        st.markdown('**Select a dataset that you generated to visualize the dataset.**')
     with col_cc:
         st.markdown(APP_TITLE , unsafe_allow_html=True)
 
@@ -1445,15 +1453,55 @@ def data_visualize():
                         content = json.loads(blob_content)
                         sql_query = content['sql']
                         selected_db = content['database']
-                        df = execute_sql(sql_query, selected_db)
+                        st.session_state['visualize_df'] = execute_sql(sql_query, selected_db)
                         # Create a StreamlitRenderer instance
-                        pyg_app = StreamlitRenderer(df)
-
-                        # Display the interactive visualization
-                        pyg_app.explorer()
+                        if st.session_state.get('visualize_df') is not None:
+                            with st.expander(label = '**Raw Dataset**'):
+                                display_paginated_dataframe(st.session_state['visualize_df'], "base_dataset_for_visualization")
+                                # st.write(st.session_state['visualize_df'])
+                            if st.button('Perform Visualizing'):
+                                pyg_app = StreamlitRenderer(st.session_state['visualize_df'])
+                                # Display the interactive visualization
+                                pyg_app.explorer()
                         # pyg_html=pyg.walk(df).to_html()
                         # components.html(pyg_html, height=1000, scrolling=True)
                     except Exception as e:
                         st.error(f"Error loading dataset: {e}")
 
+def data_profiler():
+    col_aa, col_bb, col_cc = st.columns([1, 4, 1], gap="small", vertical_alignment="center")
+    with col_aa:
+        st.image('logo.png')
+    with col_bb:
+        st.subheader("InsightLab - Data Profiler", divider='blue')
+        st.markdown('**Select a dataset that you generated for detailed profiling report.**')
+    with col_cc:
+        st.markdown(APP_TITLE , unsafe_allow_html=True)
+
+    get_saved_query_blob_list()
+    selected_query = st.selectbox('Select a saved query', [""] + list(st.session_state['query_display_dict'].keys()))
+
+    if len(selected_query) > 0:
+        if 'selected_query' not in st.session_state or st.session_state['selected_query'] != selected_query:
+                with st.container():
+                    s = selected_query[len("ID: "):]
+                    end_index = s.find(",")
+                    id = s[:end_index]
+                    try:
+                        blob_content = getBlobContent(f"{query_lib}{st.session_state.userId}/{id}.json")
+                        content = json.loads(blob_content)
+                        sql_query = content['sql']
+                        selected_db = content['database']
+                        st.session_state['profile_df']  = execute_sql(sql_query, selected_db)
+
+                        if st.session_state.get('profile_df') is not None:
+                            with st.expander(label = '**Raw Dataset**'):
+                                display_paginated_dataframe(st.session_state['profile_df'], "base_dataset_for_profiling")
+                                # st.write(st.session_state['profile_df'])
+                            if st.button('Perform Profiling'):
+                                pr = st.session_state['profile_df'].profile_report()
+                                st_profile_report(pr)
+                    except Exception as e:
+                        st.error(f"Error loading dataset: {e}")
+
 

requirements.txt

@@ -9,6 +9,9 @@ altair_saver==0.5.0
 httpx==0.27.2
 duckdb
 pygwalker
+ydata_profiling
+pkg_resources
+streamlit-pandas-profiling
 plotly
 boto3
 pyodbc

utils/menu.py

@@ -90,8 +90,8 @@ def _authenticated_menu():
         
         st.sidebar.page_link("pages/composer.py", label="Dataset Composer")
         st.session_state.page = "composer"
-        # st.sidebar.page_link("pages/profiler.py",label="Data Profiler")
-        # st.session_state.page = "profiler"
+        st.sidebar.page_link("pages/profiler.py",label="Data Profiler")
+        st.session_state.page = "profiler"
         st.sidebar.page_link("pages/visualize.py", label="Data Visualizer")
         st.session_state.page = "visualize"
         st.sidebar.page_link("pages/designer.py", label="Insight Designer")