Upload 11 files

pages/1_Data_Validation.py

@@ -4,20 +4,18 @@ import plotly.express as px
 import plotly.graph_objects as go
 from Eda_functions import *
 import numpy as np
+import re
 import pickle
+from ydata_profiling import ProfileReport
 from streamlit_pandas_profiling import st_profile_report
 import streamlit as st
 import streamlit.components.v1 as components
 import sweetviz as sv
-from utilities import set_header,load_local_css
+from utilities import set_header,initialize_data,load_local_css
 from st_aggrid import GridOptionsBuilder,GridUpdateMode
 from st_aggrid import GridOptionsBuilder
 from st_aggrid import AgGrid
 import base64
-import os
-import tempfile
-#from ydata_profiling import ProfileReport
-import re
 
 st.set_page_config(
   page_title="Data Validation",
@@ -30,187 +28,155 @@ set_header()
 
 
 
+#preprocessing
+# with open('Categorised_data.pkl', 'rb') as file:
+#   Categorised_data = pickle.load(file)
+# with open("edited_dataframe.pkl", 'rb') as file:
+
+
+# df = pickle.load(file)
+# date=df.index
+# df.reset_index(inplace=True)
+# df['Date'] = pd.to_datetime(date)
+
 
+#prospects=pd.read_excel('EDA_Data.xlsx',sheet_name='Prospects')
+#spends=pd.read_excel('EDA_Data.xlsx',sheet_name='SPEND INPUT')
+#spends.columns=['Week','Streaming (Spends)','TV (Spends)','Search (Spends)','Digital (Spends)']
+#df=pd.concat([df,spends],axis=1)
+
+#df['Date'] =pd.to_datetime(df['Date']).dt.strftime('%m/%d/%Y')
+#df['Prospects']=prospects['Prospects']
+#df.drop(['Week'],axis=1,inplace=True)
 
-with open('data_import.pkl', 'rb') as f:
-    data = pickle.load(f)
- 
-st.session_state['cleaned_data']= data['final_df']
-st.session_state['category_dict'] = data['bin_dict']
 
 st.title('Data Validation and Insights')
 
+with open("Pickle_files/main_df",'rb') as f:
+   st.session_state['cleaned_data']= pickle.load(f)
+with open("Pickle_files/category_dict",'rb') as c:
+   st.session_state['category_dict']=pickle.load(c)
+
+# st.write(st.session_state['cleaned_data'])
+    
+target_variables=[st.session_state['category_dict'][key] for key in st.session_state['category_dict'].keys() if key =='Response_Metric']
 
-target_variables=[st.session_state['category_dict'][key] for key in st.session_state['category_dict'].keys() if key =='Response Metrics']
 
 target_column = st.selectbox('Select the Target Feature/Dependent Variable (will be used in all charts as reference)',list(*target_variables))
 st.session_state['target_column']=target_column
-panels=st.session_state['category_dict']['Panel Level 1'][0]
-selected_panels=st.multiselect('Please choose the panels you wish to analyze.If no panels are selected, insights will be derived from the overall data.',st.session_state['cleaned_data'][panels].unique())
-aggregation_dict = {item: 'sum' if key == 'Media' else 'mean' for key, value in st.session_state['category_dict'].items()  for item in value if item not in ['date','Panel_1']}
 
-with st.expander('**Reponse Metric Analysis**'):
-    
-    if len(selected_panels)>0:
-        st.session_state['Cleaned_data_panel']=st.session_state['cleaned_data'][st.session_state['cleaned_data']['Panel_1'].isin(selected_panels)]
- 
-        st.session_state['Cleaned_data_panel']=st.session_state['Cleaned_data_panel'].groupby(by='date').agg(aggregation_dict)
-        st.session_state['Cleaned_data_panel']=st.session_state['Cleaned_data_panel'].reset_index()
-    else:
-        st.session_state['Cleaned_data_panel']=st.session_state['cleaned_data'].groupby(by='date').agg(aggregation_dict)
-        st.session_state['Cleaned_data_panel']=st.session_state['Cleaned_data_panel'].reset_index()
 
+fig=line_plot_target(st.session_state['cleaned_data'], target=target_column, title=f'{target_column} Over Time')
+st.plotly_chart(fig, use_container_width=True)
 
-    fig=line_plot_target(st.session_state['Cleaned_data_panel'], target=target_column, title=f'{target_column} Over Time')
-    st.plotly_chart(fig, use_container_width=True)
 
+media_channel=list(*[st.session_state['category_dict'][key] for key in st.session_state['category_dict'].keys() if key =='Media'])
+# st.write(media_channel)
 
-    media_channel=list(*[st.session_state['category_dict'][key] for key in st.session_state['category_dict'].keys() if key =='Media'])
-    # st.write(media_channel)
+Non_media_channel=[col for col in st.session_state['cleaned_data'].columns if col not in media_channel]
 
-    Non_media_variables=list(*[st.session_state['category_dict'][key] for key in st.session_state['category_dict'].keys() if key =='Exogenous' or key=='Internal'])
 
+st.markdown('### Annual Data Summary')
+st.dataframe(summary(st.session_state['cleaned_data'], media_channel+[target_column], spends=None,Target=True), use_container_width=True)
 
-    st.markdown('### Annual Data Summary')
-    st.dataframe(summary(st.session_state['Cleaned_data_panel'], media_channel+[target_column], spends=None,Target=True), use_container_width=True)
-
-    if st.checkbox('Show raw data'):
-        st.write(pd.concat([pd.to_datetime(st.session_state['Cleaned_data_panel']['date']).dt.strftime('%m/%d/%Y'),st.session_state['Cleaned_data_panel'].select_dtypes(np.number).applymap(format_numbers)],axis=1))
+if st.checkbox('Show raw data'):
+    st.write(pd.concat([pd.to_datetime(st.session_state['cleaned_data']['Date']).dt.strftime('%m/%d/%Y'),st.session_state['cleaned_data'].select_dtypes(np.number).applymap(format_numbers)],axis=1))
 col1 = st.columns(1)
 
 if "selected_feature" not in st.session_state:
     st.session_state['selected_feature']=None
 
+st.header('1. Media Channels')
+
+if 'Validation' not in st.session_state:
+    st.session_state['Validation']=[]
+
+eda_columns=st.columns(2)
+with eda_columns[0]:
+    if st.button('Generate Profile Report'):
+        pr = st.session_state['cleaned_data'].profile_report()
 
-def generate_report_with_target(channel_data, target_feature):
-        report = sv.analyze([channel_data, "Dataset"], target_feat=target_feature)
-        temp_dir = tempfile.mkdtemp()
-        report_path = os.path.join(temp_dir, "report.html")
-        report.show_html(filepath=report_path, open_browser=False)  # Generate the report as an HTML file
-        return report_path
-
-def generate_profile_report(df):
-    pr = df.profile_report()
-    temp_dir = tempfile.mkdtemp()
-    report_path = os.path.join(temp_dir, "report.html")
-    pr.to_file(report_path)
-    return report_path
-
-
-#st.header()
-with st.expander('Univariate and Bivariate Report'):
-    eda_columns=st.columns(2)
-    with eda_columns[0]:
-        if st.button('Generate Profile Report',help='Univariate report which inlcudes all statistical analysis'):
-            with st.spinner('Generating Report'):     
-                report_file = generate_profile_report(st.session_state['Cleaned_data_panel'])
-
-                if os.path.exists(report_file):
-                    with open(report_file, 'rb') as f:
-                        st.success('Report Generated')
-                        st.download_button(
-                            label="Download EDA Report",
-                            data=f.read(),
-                            file_name="pandas_profiling_report.html",
-                            mime="text/html"
-                        )
-                else:
-                    st.warning("Report generation failed. Unable to find the report file.")
+        pr.to_file("Profile_Report.html")
+
+        with open("Profile_Report.html", "rb") as f:
+            profile_report_html = f.read()
+        b64 = base64.b64encode(profile_report_html).decode()
+        href = f'<a href="data:text/html;base64,{b64}" download="Profile_Report.html">Download Profile Report</a>'
+        st.markdown(href, unsafe_allow_html=True)
 
 with eda_columns[1]:
-    if st.button('Generate Sweetviz Report',help='Bivariate report for selected response metric'):
-       with st.spinner('Generating Report'):     
-            report_file = generate_report_with_target(st.session_state['Cleaned_data_panel'], target_column)
-
-            if os.path.exists(report_file):
-                with open(report_file, 'rb') as f:
-                    st.success('Report Generated')
-                    st.download_button(
-                        label="Download EDA Report",
-                        data=f.read(),
-                        file_name="report.html",
-                        mime="text/html"
-                    )
-            else:
-                st.warning("Report generation failed. Unable to find the report file.")
-            
-
-
-#st.warning('Work in Progress')
-with st.expander('Media Variables Analysis'):
-# Get the selected feature
-    st.session_state["selected_feature"]= st.selectbox('Select media', [col for col in media_channel if 'cost' not in col.lower() and 'spend' not in col.lower()])
-
-    # Filter spends features based on the selected feature
-    spends_features = [col for col in st.session_state['Cleaned_data_panel'].columns if any(keyword in col.lower() for keyword in ['cost', 'spend'])]
-    spends_feature = [col for col in spends_features if re.split(r'_cost|_spend', col.lower())[0] in st.session_state["selected_feature"]]
-
-    if 'validation' not in st.session_state:
-        st.session_state['validation']=[]
-
-
-    val_variables=[col for col in media_channel if col!='date']
-    if len(spends_feature)==0:  
-        st.warning('No spends varaible available for the selected metric in data') 
-        
-    else:
-        fig_row1 = line_plot(st.session_state['Cleaned_data_panel'], x_col='date', y1_cols=[st.session_state["selected_feature"]], y2_cols=[target_column], title=f'Analysis of {st.session_state["selected_feature"]} and {[target_column][0]} Over Time')
-        st.plotly_chart(fig_row1, use_container_width=True)
-        st.markdown('### Summary')
-        st.dataframe(summary(st.session_state['cleaned_data'],[st.session_state["selected_feature"]],spends=spends_feature[0]),use_container_width=True)
-
-        cols2=st.columns(2)
-        with cols2[0]:
-            if st.button('Validate'):
-                st.session_state['validation'].append(st.session_state["selected_feature"])
-        with cols2[1]:
-            if st.checkbox('Validate all'):
-                st.session_state['validation'].extend(val_variables)
-                st.success('All media variables are validated ✅')
-
-        if len(set(st.session_state['validation']).intersection(val_variables))!=len(val_variables):
-            validation_data=pd.DataFrame({'Validate':[True if col in st.session_state['validation'] else False for col in val_variables],
-                                        'Variables':val_variables
-                                        })
-            cols3=st.columns([1,30])
-            with cols3[1]:
-                validation_df=st.data_editor(validation_data,
-                                            # column_config={
-                                            # 'Validate':st.column_config.CheckboxColumn(wi)
-                                            
-                                            # },
-                                            column_config={
-                                                "Validate": st.column_config.CheckboxColumn(
-                                                    default=False,
-                                                    width=100,
-                                                ),
-                                                'Variables':st.column_config.TextColumn(
-                                                    width=1000
-
-                                                )
-                                                },hide_index=True)
-
-                selected_rows = validation_df[validation_df['Validate']==True]['Variables']
-
-                #st.write(selected_rows)
-
-                st.session_state['validation'].extend(selected_rows)
-
-                not_validated_variables = [col for col in val_variables if col not in st.session_state["validation"]]
-                if not_validated_variables: 
-                    not_validated_message = f'The following variables are not validated:\n{" , ".join(not_validated_variables)}'
-                    st.warning(not_validated_message)
-
-
-
-with st.expander('Non Media Variables Analysis'):
-    selected_columns_row4 = st.selectbox('Select Channel',Non_media_variables,index=1)
-    #     # Create the dual-axis line plot
-    fig_row4 = line_plot(st.session_state['Cleaned_data_panel'], x_col='date', y1_cols=[selected_columns_row4], y2_cols=[target_column], title=f'Analysis of {selected_columns_row4} and {target_column} Over Time')
+    if st.button('Generate Sweetviz Report'):
+    
+        def generate_report_with_target(df, target_feature):
+            report = sv.analyze([df, "Dataset"], target_feat=target_feature)
+            return report
+
+        report = generate_report_with_target(st.session_state['cleaned_data'], target_feature=target_column)
+        report.show_html()
+
+
+selected_media = st.selectbox('Select media', np.unique([Categorised_data[col]['VB'] for col in media_channel]))
+# selected_feature=st.multiselect('Select Metric', df.columns[df.columns.str.contains(selected_media,case=False)])
+st.session_state["selected_feature"]=st.selectbox('Select Metric',[col for col in  media_channel  if    Categorised_data[col]['VB'] in selected_media ] )
+spends_features=[col for col in df.columns if 'spends' in col.lower() or 'cost' in col.lower()]
+spends_feature=[col for col in spends_features if col.split('_')[0] in st.session_state["selected_feature"].split('_')[0]]
+#st.write(spends_features)
+#st.write(spends_feature)
+#st.write(selected_feature)
+
+
+val_variables=[col for col in media_channel if col!='Date']
+if len(spends_feature)==0:  
+    st.warning('No spends varaible available for the selected metric in data') 
+       
+else:
+    st.write(f'Selected spends variable {spends_feature[0]} if wrong please name the varaibles properly')
+    # Create the dual-axis line plot
+    fig_row1 = line_plot(df, x_col='Date', y1_cols=[st.session_state["selected_feature"]], y2_cols=[target_column], title=f'Analysis of {st.session_state["selected_feature"]} and {[target_column][0]} Over Time')
+    st.plotly_chart(fig_row1, use_container_width=True)
+    st.markdown('### Annual Data Summary')
+    st.dataframe(summary(df,[st.session_state["selected_feature"]],spends=spends_feature[0]),use_container_width=True)
+    if st.button('Validate'):
+        st.session_state['Validation'].append(st.session_state["selected_feature"])
+
+    if st.checkbox('Validate all'):
+        st.session_state['Validation'].extend(val_variables)
+        st.success('All media variables are validated ✅')
+    if len(set(st.session_state['Validation']).intersection(val_variables))!=len(val_variables):
+        #st.write(st.session_state['Validation'])
+        validation_data=pd.DataFrame({'Variables':val_variables,
+                                    'Validated':[1 if col in st.session_state['Validation'] else 0 for col in val_variables],
+                                    'Bucket':[Categorised_data[col]['VB'] for col in val_variables]})
+        gd=GridOptionsBuilder.from_dataframe(validation_data)
+        gd.configure_pagination(enabled=True)
+        gd.configure_selection(use_checkbox=True,selection_mode='multiple')
+        #gd.configure_selection_toggle_all(None, show_toggle_all=True)
+        #gd.configure_columns_auto_size_mode(GridOptionsBuilder.configure_columns)
+        gridoptions=gd.build()
+        #st.text(st.session_state['Validation'])
+        table = AgGrid(validation_data,gridOptions=gridoptions,update_mode=GridUpdateMode.SELECTION_CHANGED,fit_columns_on_grid_load=True)
+        #st.table(table)
+        selected_rows = table["selected_rows"]
+        st.session_state['Validation'].extend([col['Variables'] for col in selected_rows])
+        not_validated_variables = [col for col in val_variables if col not in st.session_state["Validation"]]
+        if not_validated_variables:
+            not_validated_message = f'The following variables are not validated:\n{" , ".join(not_validated_variables)}'
+            st.warning(not_validated_message)
+
+
+
+st.header('2. Non Media Variables')
+selected_columns_row = [col for col in df.columns if ("imp" not in col.lower()) and ('cli' not in col.lower() ) and ('spend' not in col.lower()) and col!='Date']
+selected_columns_row4 = st.selectbox('Select Channel',selected_columns_row )
+if not selected_columns_row4: 
+    st.warning('Please select at least one.')
+else:
+    # Create the dual-axis line plot
+    fig_row4 = line_plot(df, x_col='Date', y1_cols=[selected_columns_row4], y2_cols=[target_column], title=f'Analysis of {selected_columns_row4} and {target_column} Over Time')
     st.plotly_chart(fig_row4, use_container_width=True)
     selected_non_media=selected_columns_row4
-    sum_df = st.session_state['Cleaned_data_panel'][['date', selected_non_media,target_column]]
-    sum_df['Year']=pd.to_datetime(st.session_state['Cleaned_data_panel']['date']).dt.year
+    sum_df = df[['Date', selected_non_media,target_column]]
+    sum_df['Year']=pd.to_datetime(df['Date']).dt.year
     #st.dataframe(df)
     #st.dataframe(sum_df.head(2))
     sum_df=sum_df.groupby('Year').agg('sum')
@@ -218,34 +184,58 @@ with st.expander('Non Media Variables Analysis'):
     sum_df=sum_df.applymap(format_numbers) 
     sum_df.fillna('-',inplace=True)
     sum_df=sum_df.replace({"0.0":'-','nan':'-'})
-    st.markdown('### Summary')    
+    st.markdown('### Annual Data Summary')    
     st.dataframe(sum_df,use_container_width=True)
 
-
-with st.expander('Correlation Analysis'):
-    options = list(st.session_state['Cleaned_data_panel'].select_dtypes(np.number).columns)
-
-    # selected_options = []
-    # num_columns = 4
-    # num_rows = -(-len(options) // num_columns)  # Ceiling division to calculate rows
-
-    # # Create a grid of checkboxes
-    # st.header('Select Features for Correlation Plot')
-    # tick=False
-    # if st.checkbox('Select all'):
-    #     tick=True
-    # selected_options = []
-    # for row in range(num_rows):
-    #     cols = st.columns(num_columns)
-    #     for col in cols:
-    #         if options:
-    #             option = options.pop(0) 
-    #             selected = col.checkbox(option,value=tick)
-    #             if selected:
-    #                 selected_options.append(option)
-    # # Display selected options
-
-    selected_options=st.multiselect('Select Variables For correlation plot',[var for var in options if var!= target_column],default=options[3])
-
-    st.pyplot(correlation_plot(st.session_state['Cleaned_data_panel'],selected_options,target_column))
+    # if st.checkbox('Validate',key='2'):
+    #     st.session_state['Validation'].append(selected_columns_row4)
+# val_variables=[col for col in media_channel if col!='Date']
+# if st.checkbox('Validate all'):
+#     st.session_state['Validation'].extend(val_variables)
+# validation_data=pd.DataFrame({'Variables':val_variables,
+#                             'Validated':[1 if col in st.session_state['Validation'] else 0 for col in val_variables],
+#                             'Bucket':[Categorised_data[col]['VB'] for col in val_variables]})
+# gd=GridOptionsBuilder.from_dataframe(validation_data)
+# gd.configure_pagination(enabled=True)
+# gd.configure_selection(use_checkbox=True,selection_mode='multiple')
+# #gd.configure_selection_toggle_all(None, show_toggle_all=True)
+# #gd.configure_columns_auto_size_mode(GridOptionsBuilder.configure_columns)
+# gridoptions=gd.build()
+# #st.text(st.session_state['Validation'])
+# table = AgGrid(validation_data,gridOptions=gridoptions,update_mode=GridUpdateMode.SELECTION_CHANGED,fit_columns_on_grid_load=True)
+# #st.table(table)
+# selected_rows = table["selected_rows"]
+# st.session_state['Validation'].extend([col['Variables'] for col in selected_rows])
+# not_validated_variables = [col for col in val_variables if col not in st.session_state["Validation"]]
+# if not_validated_variables:
+#     not_validated_message = f'The following variables are not validated:\n{" , ".join(not_validated_variables)}'
+#     st.warning(not_validated_message)
+
+options = list(df.select_dtypes(np.number).columns)
+st.markdown(' ')
+st.markdown(' ')
+st.markdown('# Exploratory Data Analysis')
+st.markdown(' ')
+
+selected_options = []
+num_columns = 4
+num_rows = -(-len(options) // num_columns)  # Ceiling division to calculate rows
+
+# Create a grid of checkboxes
+st.header('Select Features for Correlation Plot')
+tick=False
+if st.checkbox('Select all'):
+    tick=True
+selected_options = []
+for row in range(num_rows):
+    cols = st.columns(num_columns)
+    for col in cols:
+        if options:
+            option = options.pop(0) 
+            selected = col.checkbox(option,value=tick)
+            if selected:
+                selected_options.append(option)
+# Display selected options
+#st.write('You selected:', selected_options)
+st.pyplot(correlation_plot(df,selected_options,target_column))
 

pages/2_Transformations_with_panel.py

@@ -0,0 +1,612 @@
+'''
+MMO Build Sprint 3
+date :
+additions : adding more variables to session state for saved model : random effect, predicted train & test
+'''
+
+import streamlit as st
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+from Eda_functions import format_numbers
+import numpy as np
+import pickle
+from st_aggrid import AgGrid
+from st_aggrid import GridOptionsBuilder,GridUpdateMode
+from utilities import set_header,load_local_css
+from st_aggrid import GridOptionsBuilder
+import time
+import itertools
+import statsmodels.api as sm
+import numpy as npc
+import re
+import itertools
+from sklearn.metrics import mean_absolute_error, r2_score,mean_absolute_percentage_error
+from sklearn.preprocessing import MinMaxScaler
+import os
+import matplotlib.pyplot as plt
+from statsmodels.stats.outliers_influence import variance_inflation_factor
+st.set_option('deprecation.showPyplotGlobalUse', False)
+import statsmodels.api as sm
+import statsmodels.formula.api as smf
+
+from datetime import datetime
+import seaborn as sns
+from Data_prep_functions import *
+
+
+def get_random_effects(media_data, panel_col, mdf):
+    random_eff_df = pd.DataFrame(columns=[panel_col, "random_effect"])
+
+    for i, market in enumerate(media_data[panel_col].unique()):
+        print(i, end='\r')
+        intercept = mdf.random_effects[market].values[0]
+        random_eff_df.loc[i, 'random_effect'] = intercept
+        random_eff_df.loc[i, panel_col] = market
+
+    return random_eff_df
+
+
+def mdf_predict(X_df, mdf, random_eff_df) :
+    X=X_df.copy()
+    X['fixed_effect'] = mdf.predict(X)
+    X=pd.merge(X, random_eff_df, on=panel_col, how='left')
+    X['pred'] = X['fixed_effect'] + X['random_effect']
+    # X.to_csv('Test/megred_df.csv',index=False)
+    X.drop(columns=['fixed_effect', 'random_effect'], inplace=True)
+    return X['pred']
+
+st.set_page_config(
+  page_title="Model Build",
+  page_icon=":shark:",
+  layout="wide",
+  initial_sidebar_state='collapsed'
+)
+
+load_local_css('styles.css')
+set_header()
+
+
+st.title('1. Build Your Model')
+
+# set the panel column
+date_col = 'date'
+
+
+media_data=pd.read_csv(r'upf_data_converted.csv')
+# with open("Pickle_files/main_df",'rb') as f:
+#    media_data= pickle.load(f)
+   
+
+media_data.columns=[i.lower().strip().replace(' ','_').replace('-','').replace(':','').replace("__", "_") for i in media_data.columns]
+#st.write(media_data.columns)
+#media_data.drop(['indicacao_impressions','infleux_impressions','influencer_impressions'],axis=1,inplace=True)
+target_col = 'total_approved_accounts_revenue'
+# st.write(media_data.columns)
+media_data.sort_values(date_col, inplace=True)
+media_data.reset_index(drop=True,inplace=True)
+
+date=media_data[date_col]
+st.session_state['date']=date
+revenue=media_data[target_col]
+media_data.drop([target_col],axis=1,inplace=True)
+media_data.drop([date_col],axis=1,inplace=True)
+media_data.reset_index(drop=True,inplace=True)
+
+
+if st.toggle('Apply Transformations on DMA/Panel Level'):
+  dma=st.selectbox('Select the Level of data ',[ col for col in media_data.columns if col.lower() in ['dma','panel', 'markets']])
+  panel_col= dma
+
+else:
+  #""" code to aggregate data on date """
+
+
+  dma=None
+
+# dma_dict={ dm:media_data[media_data[dma]==dm] for dm in media_data[dma].unique()}
+# st.write(dma_dict)
+
+st.markdown('## Select the Range of Transformations')
+columns = st.columns(2)
+old_shape=media_data.shape
+
+
+if "old_shape" not in st.session_state:
+   st.session_state['old_shape']=old_shape
+
+
+with columns[0]:
+  slider_value_adstock  = st.slider('Select Adstock Range (only applied to media)', 0.0, 1.0, (0.2, 0.4), step=0.1, format="%.2f")
+with columns[1]:
+  slider_value_lag = st.slider('Select Lag Range (applied to media, seasonal, macroeconomic variables)', 1, 7, (1, 3), step=1)
+
+# with columns[2]:
+#    slider_value_power=st.slider('Select Power range (only applied to media )',0,4,(1,2),step=1)
+
+# with columns[1]:
+#    st.number_input('Select the range of half saturation point ',min_value=1,max_value=5)
+#    st.number_input('Select the range of  ')
+
+# Section 1 - Transformations Functions
+def lag(data,features,lags,dma=None):
+    if dma:
+
+        transformed_data=pd.concat([data.groupby([dma])[features].shift(lag).add_suffix(f'_lag_{lag}') for lag in lags],axis=1)
+        transformed_data=transformed_data.fillna(method='bfill')
+        return pd.concat([transformed_data,data],axis=1)
+
+    else:
+
+        #''' data should be aggregated on date'''
+
+        transformed_data=pd.concat([data[features].shift(lag).add_suffix(f'_lag_{lag}') for lag in lags],axis=1)
+        transformed_data=transformed_data.fillna(method='bfill')
+
+        return pd.concat([transformed_data,data],axis=1)
+
+#adstock
+def adstock(df, alphas, cutoff, features,dma=None):
+    # st.write(features)
+
+    if dma:
+        transformed_data=pd.DataFrame()
+        for d in df[dma].unique():
+            dma_sub_df = df[df[dma] == d]
+            n = len(dma_sub_df)
+
+
+            weights = np.array([[[alpha**(i-j) if i >= j and j >= i-cutoff else 0. for j in range(n)] for i in range(n)] for alpha in alphas])
+            X = dma_sub_df[features].to_numpy()
+
+            res = pd.DataFrame(np.hstack(weights @ X),
+                               columns=[f'{col}_adstock_{alpha}' for alpha in alphas for col in features])
+
+            transformed_data=pd.concat([transformed_data,res],axis=0)
+            transformed_data.reset_index(drop=True,inplace=True)
+        return pd.concat([transformed_data,df],axis=1)
+
+    else:
+
+        n = len(df)
+
+
+        weights = np.array([[[alpha**(i-j) if i >= j and j >= i-cutoff else 0. for j in range(n)] for i in range(n)] for alpha in alphas])
+
+        X = df[features].to_numpy()
+        res = pd.DataFrame(np.hstack(weights @ X),
+                           columns=[f'{col}_adstock_{alpha}' for alpha in alphas for col in features])
+        return  pd.concat([res,df],axis=1)
+
+
+
+
+# Section 2 - Begin Transformations
+
+if 'media_data' not in st.session_state:
+
+  st.session_state['media_data']=pd.DataFrame()
+
+# Sprint3 additions
+if 'random_effects' not in st.session_state:
+  st.session_state['random_effects']=pd.DataFrame()
+if 'pred_train' not in st.session_state:
+  st.session_state['pred_train'] = []
+if 'pred_test' not in st.session_state:
+  st.session_state['pred_test'] = []
+# end of Sprint3 additions
+
+# variables_to_be_transformed=[col for col in media_data.columns if col.lower() not in ['dma','panel'] ] # change for buckets
+variables_to_be_transformed=[col for col in media_data.columns if '_clicks' in col.lower() or '_impress' in col.lower()] # srishti - change
+# st.write(variables_to_be_transformed)
+# st.write(media_data[variables_to_be_transformed].dtypes)
+
+with columns[0]:
+  if st.button('Apply Transformations'):
+    with st.spinner('Applying Transformations'):
+      transformed_data_lag=lag(media_data,features=variables_to_be_transformed,lags=np.arange(slider_value_lag[0],slider_value_lag[1]+1,1),dma=dma)
+
+      # variables_to_be_transformed=[col for col in list(transformed_data_lag.columns) if col not in ['Date','DMA','Panel']] #change for buckets
+      variables_to_be_transformed = [col for col in media_data.columns if
+                                    '_clicks' in col.lower() or '_impress' in col.lower()]  # srishti - change
+
+      transformed_data_adstock=adstock(df=transformed_data_lag, alphas=np.arange(slider_value_adstock[0],slider_value_adstock[1],0.1), cutoff=8, features=variables_to_be_transformed,dma=dma)
+
+      # st.success('Done')
+      st.success("Transformations complete!")
+
+      st.write(f'old shape {old_shape}, new shape {transformed_data_adstock.shape}')
+      # st.write(media_data.head(10))
+      # st.write(transformed_data_adstock.head(10))
+
+      transformed_data_adstock.columns = [c.replace(".","_") for c in transformed_data_adstock.columns] # srishti
+      # st.write(transformed_data_adstock.columns)
+      st.session_state['media_data']=transformed_data_adstock # srishti
+
+    # with st.spinner('Applying Transformations'):
+    #   time.sleep(2)
+    #   st.success("Transformations complete!")
+
+# if st.session_state['media_data'].shape[1]>old_shape[1]:
+  # with columns[0]:
+    # st.write(f'Total no.of variables before transformation: {old_shape[1]}, Total no.of variables after transformation: {st.session_state["media_data"].shape[1]}')
+  #st.write(f'Total no.of variables after transformation: {st.session_state["media_data"].shape[1]}')
+
+# Section 3 - Create combinations
+
+# bucket=['paid_search', 'kwai','indicacao','infleux', 'influencer','FB: Level Achieved - Tier 1 Impressions',
+#       ' FB: Level Achieved - Tier 2 Impressions','paid_social_others',
+#         ' GA App: Will And Cid Pequena Baixo Risco Clicks',
+#       'digital_tactic_others',"programmatic"
+#       ]
+
+# srishti - bucket names changed
+bucket=['paid_search', 'kwai','indicacao','infleux', 'influencer','fb_level_achieved_tier_2',
+      'fb_level_achieved_tier_1','paid_social_others',
+        'ga_app',
+      'digital_tactic_others',"programmatic"
+      ]
+
+with columns[1]:
+  if st.button('Create Combinations of Variables'):
+
+    top_3_correlated_features=[]
+    # for col in st.session_state['media_data'].columns[:19]:
+    original_cols = [c for c in st.session_state['media_data'].columns if "_clicks" in c.lower() or "_impressions" in c.lower()]
+    original_cols = [c for c in original_cols if "_lag" not in c.lower() and "_adstock" not in c.lower()]
+    # st.write(original_cols)
+
+    # for col in st.session_state['media_data'].columns[:19]:
+    for col in original_cols: # srishti - new
+        corr_df=pd.concat([st.session_state['media_data'].filter(regex=col),
+                  revenue],axis=1).corr()[target_col].iloc[:-1]
+        top_3_correlated_features.append(list(corr_df.sort_values(ascending=False).head(2).index))
+        # st.write(col, top_3_correlated_features)
+    flattened_list = [item for sublist in top_3_correlated_features for item in sublist]
+    # all_features_set={var:[col for col in flattened_list if var in col] for var in bucket}
+    all_features_set={var:[col for col in flattened_list if var in col] for var in bucket if len([col for col in flattened_list if var in col])>0} # srishti
+
+    channels_all=[values for values in all_features_set.values()]
+    # st.write(channels_all)
+    st.session_state['combinations'] = list(itertools.product(*channels_all))
+  # if 'combinations' not in st.session_state:
+  #   st.session_state['combinations']=combinations_all
+
+    st.session_state['final_selection']=st.session_state['combinations']
+    st.success('Done')
+    # st.write(f"{len(st.session_state['combinations'])} combinations created")
+
+
+    revenue.reset_index(drop=True,inplace=True)
+  if 'Model_results' not in st.session_state:
+        st.session_state['Model_results']={'Model_object':[],
+      'Model_iteration':[],
+      'Feature_set':[],
+      'MAPE':[],
+      'R2':[],
+      'ADJR2':[]
+      }
+
+  def reset_model_result_dct():
+      st.session_state['Model_results']={'Model_object':[],
+      'Model_iteration':[],
+      'Feature_set':[],
+      'MAPE':[],
+      'R2':[],
+      'ADJR2':[]
+      }
+
+      # if st.button('Build Model'):
+  if 'iterations' not in st.session_state:
+    st.session_state['iterations']=0
+      # st.write("1",st.session_state["final_selection"])
+
+  if 'final_selection' not in st.session_state:
+      st.session_state['final_selection']=False
+
+save_path = r"Model/"
+with columns[1]:
+  if  st.session_state['final_selection']:
+    st.write(f'Total combinations created {format_numbers(len(st.session_state["final_selection"]))}')
+
+    
+if st.checkbox('Build all iterations'):
+   iterations=len(st.session_state['final_selection'])
+else:
+   iterations = st.number_input('Select the number of iterations to perform', min_value=0, step=100, value=st.session_state['iterations'],on_change=reset_model_result_dct)
+  #  st.write("iterations=", iterations)
+
+if st.button('Build Model',on_click=reset_model_result_dct):
+  st.session_state['iterations']=iterations 
+  # st.write("2",st.session_state["final_selection"])
+
+  # Section 4 - Model
+
+  st.session_state['media_data']=st.session_state['media_data'].fillna(method='ffill')
+  st.markdown(
+      'Data Split -- Training Period: May 9th, 2023 - October 5th,2023 , Testing Period: October 6th, 2023 - November 7th, 2023 ')
+  progress_bar = st.progress(0)  # Initialize the progress bar
+  # time_remaining_text = st.empty()  # Create an empty space for time remaining text
+  start_time = time.time()  # Record the start time
+  progress_text = st.empty()
+  # time_elapsed_text = st.empty()
+  # for i, selected_features in enumerate(st.session_state["final_selection"][40000:40000 + int(iterations)]):
+  # st.write(st.session_state["final_selection"])
+  # for i, selected_features in enumerate(st.session_state["final_selection"]):
+  for i, selected_features in enumerate(st.session_state["final_selection"][0:int(iterations)]): # srishti
+      df = st.session_state['media_data']
+
+      fet = [var for var in selected_features if len(var) > 0]
+      inp_vars_str = " + ".join(fet)  # new
+      
+
+      X = df[fet]
+      y = revenue
+      ss = MinMaxScaler()
+      X = pd.DataFrame(ss.fit_transform(X), columns=X.columns)
+      # X = sm.add_constant(X)
+
+      X['total_approved_accounts_revenue'] = revenue  # Sprint2
+      X[panel_col] = df[panel_col] # Sprint2
+
+
+
+      X_train=X.iloc[:8000]
+      X_test=X.iloc[8000:]
+      y_train=y.iloc[:8000]
+      y_test=y.iloc[8000:]
+
+
+
+      md = smf.mixedlm("total_approved_accounts_revenue ~ {}".format(inp_vars_str),
+                      data=X_train[['total_approved_accounts_revenue'] + fet],
+                      groups=X_train[panel_col])
+      mdf = md.fit()
+      predicted_values = mdf.fittedvalues
+
+      # st.write(fet)
+      # positive_coeff=fet
+      # negetive_coeff=[]
+
+      coefficients = mdf.fe_params.to_dict()
+      model_possitive = [col for col in coefficients.keys() if coefficients[col] > 0]
+      # st.write(positive_coeff)
+      # st.write(model_possitive)
+      pvalues = [var for var in list(mdf.pvalues) if var <= 0.06]
+
+      # if (len(model_possitive) / len(selected_features)) > 0.9 and (len(pvalues) / len(selected_features)) >= 0.8:
+      if (len(model_possitive) / len(selected_features)) > 0 and (len(pvalues) / len(selected_features)) >= 0: # srishti - changed just for testing, revert later
+          # predicted_values = model.predict(X_train)
+          mape = mean_absolute_percentage_error(y_train, predicted_values)
+          r2 = r2_score(y_train, predicted_values)
+          adjr2 = 1 - (1 - r2) * (len(y_train) - 1) / (len(y_train) - len(selected_features) - 1)
+
+          filename = os.path.join(save_path, f"model_{i}.pkl")
+          with open(filename, "wb") as f:
+              pickle.dump(mdf, f)
+          # with open(r"C:\Users\ManojP\Documents\MMM\simopt\Model\model.pkl", 'rb') as file:
+          #   model = pickle.load(file)
+
+          st.session_state['Model_results']['Model_object'].append(filename)
+          st.session_state['Model_results']['Model_iteration'].append(i)
+          st.session_state['Model_results']['Feature_set'].append(fet)
+          st.session_state['Model_results']['MAPE'].append(mape)
+          st.session_state['Model_results']['R2'].append(r2)
+          st.session_state['Model_results']['ADJR2'].append(adjr2)
+
+      current_time = time.time()
+      time_taken = current_time - start_time
+      time_elapsed_minutes = time_taken / 60
+      completed_iterations_text = f"{i + 1}/{iterations}"
+      progress_bar.progress((i + 1) / int(iterations))
+      progress_text.text(f'Completed iterations: {completed_iterations_text},Time Elapsed (min): {time_elapsed_minutes:.2f}')
+
+  st.write(f'Out of {st.session_state["iterations"]} iterations : {len(st.session_state["Model_results"]["Model_object"])} valid models')
+  pd.DataFrame(st.session_state['Model_results']).to_csv('model_output.csv')
+
+  def to_percentage(value):
+    return f'{value * 100:.1f}%'
+
+## Section 5 - Select Model
+st.title('2. Select Models')
+if 'tick' not in st.session_state:
+   st.session_state['tick']=False
+if st.checkbox('Show results of top 10 models (based on MAPE and Adj. R2)',value=st.session_state['tick']):
+  st.session_state['tick']=True
+  st.write('Select one model iteration to generate performance metrics for it:')
+  data=pd.DataFrame(st.session_state['Model_results'])
+  data.sort_values(by=['MAPE'],ascending=False,inplace=True)
+  data.drop_duplicates(subset='Model_iteration',inplace=True)
+  top_10=data.head(10)
+  top_10['Rank']=np.arange(1,len(top_10)+1,1)
+  top_10[['MAPE','R2','ADJR2']]=np.round(top_10[['MAPE','R2','ADJR2']],4).applymap(to_percentage)
+  top_10_table = top_10[['Rank','Model_iteration','MAPE','ADJR2','R2']]
+  #top_10_table.columns=[['Rank','Model Iteration Index','MAPE','Adjusted R2','R2']]
+  gd=GridOptionsBuilder.from_dataframe(top_10_table)
+  gd.configure_pagination(enabled=True)
+  gd.configure_selection(use_checkbox=True)
+
+
+  gridoptions=gd.build()
+
+  table = AgGrid(top_10,gridOptions=gridoptions,update_mode=GridUpdateMode.SELECTION_CHANGED)
+
+  selected_rows=table.selected_rows
+  # if st.session_state["selected_rows"] != selected_rows:
+  #   st.session_state["build_rc_cb"] = False
+  st.session_state["selected_rows"] = selected_rows
+  if 'Model' not in st.session_state:
+    st.session_state['Model']={}
+
+# Section 6 - Display Results
+
+  if len(selected_rows)>0:
+    st.header('2.1 Results Summary')
+
+    model_object=data[data['Model_iteration']==selected_rows[0]['Model_iteration']]['Model_object']
+    features_set=data[data['Model_iteration']==selected_rows[0]['Model_iteration']]['Feature_set']
+
+    with open(str(model_object.values[0]), 'rb') as file:
+        # print(file)
+        model = pickle.load(file)
+    st.write(model.summary())
+    st.header('2.2 Actual vs. Predicted Plot')
+
+    df=st.session_state['media_data']
+    X=df[features_set.values[0]]
+    # X = sm.add_constant(X)
+    y=revenue
+
+    ss = MinMaxScaler()
+    X = pd.DataFrame(ss.fit_transform(X), columns=X.columns)
+
+    # Sprint2 changes
+    X['total_approved_accounts_revenue'] = revenue  # new
+    X[panel_col] = df[panel_col]
+    X[date_col]=date
+
+
+
+    X_train=X.iloc[:8000]
+    X_test=X.iloc[8000:].reset_index(drop=True)
+    y_train=y.iloc[:8000]
+    y_test=y.iloc[8000:].reset_index(drop=True)
+
+
+    random_eff_df = get_random_effects(media_data, panel_col, model)
+    train_pred = model.fittedvalues
+    test_pred = mdf_predict(X_test, model, random_eff_df)
+    print("__"*20, test_pred.isna().sum())
+
+    # save x test to test - srishti
+    x_test_to_save = X_test.copy()
+    x_test_to_save['Actuals'] = y_test
+    x_test_to_save['Predictions'] = test_pred
+
+    x_train_to_save=X_train.copy()
+    x_train_to_save['Actuals'] = y_train
+    x_train_to_save['Predictions'] = train_pred
+
+    x_train_to_save.to_csv('Test/x_train_to_save.csv',index=False)
+    x_test_to_save.to_csv('Test/x_test_to_save.csv',index=False)
+
+    st.session_state['X']=X_train
+    st.session_state['features_set']=features_set.values[0]
+    print("**"*20,"selected model features : ",features_set.values[0])
+    metrics_table,line,actual_vs_predicted_plot=plot_actual_vs_predicted(X_train[date_col], y_train, train_pred, model,target_column='Revenue',is_panel=True) # Sprint2
+
+    st.plotly_chart(actual_vs_predicted_plot,use_container_width=True)
+
+
+
+    st.markdown('## 2.3 Residual Analysis')
+    columns=st.columns(2)
+    with columns[0]:
+      fig=plot_residual_predicted(y_train,train_pred,X_train) # Sprint2
+      st.plotly_chart(fig)
+
+    with columns[1]:
+      st.empty()
+      fig = qqplot(y_train,train_pred) # Sprint2
+      st.plotly_chart(fig)
+
+    with columns[0]:
+      fig=residual_distribution(y_train,train_pred) # Sprint2
+      st.pyplot(fig)
+
+
+
+    vif_data = pd.DataFrame()
+    # X=X.drop('const',axis=1)
+    X_train_with_panels = X_train.copy() # Sprint2 -- creating a copy of xtrain. Later deleting panel, target & date from xtrain
+    X_train.drop(columns=[target_col, panel_col, date_col], inplace=True) # Sprint2
+    vif_data["Variable"] = X_train.columns
+    vif_data["VIF"] = [variance_inflation_factor(X_train.values, i) for i in range(X_train.shape[1])]
+    vif_data.sort_values(by=['VIF'],ascending=False,inplace=True)
+    vif_data=np.round(vif_data)
+    vif_data['VIF']=vif_data['VIF'].astype(float)
+    st.header('2.4 Variance Inflation Factor (VIF)')
+    #st.dataframe(vif_data)
+    color_mapping = {
+    'darkgreen': (vif_data['VIF'] < 3),
+    'orange': (vif_data['VIF'] >= 3) & (vif_data['VIF'] <= 10),
+    'darkred': (vif_data['VIF'] > 10)
+    }
+
+# Create a horizontal bar plot
+    fig, ax = plt.subplots()
+    fig.set_figwidth(10)  # Adjust the width of the figure as needed
+
+    # Sort the bars by descending VIF values
+    vif_data = vif_data.sort_values(by='VIF', ascending=False)
+
+    # Iterate through the color mapping and plot bars with corresponding colors
+    for color, condition in color_mapping.items():
+        subset = vif_data[condition]
+        bars = ax.barh(subset["Variable"], subset["VIF"], color=color, label=color)
+
+        # Add text annotations on top of the bars
+        for bar in bars:
+            width = bar.get_width()
+            ax.annotate(f'{width:}', xy=(width, bar.get_y() + bar.get_height() / 2), xytext=(5, 0),
+                        textcoords='offset points', va='center')
+
+    # Customize the plot
+    ax.set_xlabel('VIF Values')
+    #ax.set_title('2.4 Variance Inflation Factor (VIF)')
+    #ax.legend(loc='upper right')
+
+    # Display the plot in Streamlit
+    st.pyplot(fig)
+
+
+
+    with st.expander('Results Summary Test data'):
+      # ss = MinMaxScaler()
+      # X_test = pd.DataFrame(ss.fit_transform(X_test), columns=X_test.columns)
+      st.header('2.2 Actual vs. Predicted Plot')
+
+      metrics_table,line,actual_vs_predicted_plot=plot_actual_vs_predicted(X_test[date_col], y_test, test_pred, model,target_column='Revenue',is_panel=True) # Sprint2
+
+      st.plotly_chart(actual_vs_predicted_plot,use_container_width=True)
+
+      st.markdown('## 2.3 Residual Analysis')
+      columns=st.columns(2)
+      with columns[0]:
+        fig=plot_residual_predicted(revenue,test_pred,X_test) # Sprint2
+        st.plotly_chart(fig)
+
+      with columns[1]:
+        st.empty()
+        fig = qqplot(revenue,test_pred) # Sprint2
+        st.plotly_chart(fig)
+
+      with columns[0]:
+        fig=residual_distribution(revenue,test_pred) # Sprint2
+        st.pyplot(fig)
+
+    value=False
+    if st.checkbox('Save this model to tune',key='build_rc_cb'):
+      mod_name=st.text_input('Enter model name')
+      if len(mod_name)>0:
+        st.session_state['Model'][mod_name]={"Model_object":model,'feature_set':st.session_state['features_set'],'X_train':X_train_with_panels}
+        st.session_state['X_train']=X_train_with_panels
+        st.session_state['X_test']=X_test
+        st.session_state['y_train']=y_train
+        st.session_state['y_test']=y_test
+
+        # Sprint3 additions
+        random_eff_df= get_random_effects(media_data, panel_col, model)
+        st.session_state['random_effects']=random_eff_df
+
+        st.session_state['pred_train']=model.fittedvalues
+        st.session_state['pred_test']=mdf_predict(X_test, model, random_eff_df)
+        # End of Sprint3 additions
+
+        with open("best_models.pkl", "wb") as f:
+          pickle.dump(st.session_state['Model'], f)
+          st.success('Model saved! Proceed to the next page to tune the model')
+        value=False
+
+        # st.write(st.session_state['Model'][mod_name]['X_train'].columns)
+        # st.write(st.session_state['X_test'].columns)

pages/3_Model_Tuning_with_panel.py

@@ -0,0 +1,437 @@
+'''
+MMO Build Sprint 3
+date :
+changes : capability to tune MixedLM as well as simple LR in the same page
+'''
+
+import streamlit as st
+import pandas as pd
+from Eda_functions import format_numbers
+import pickle
+from utilities import set_header,load_local_css
+import statsmodels.api as sm
+import re
+from sklearn.preprocessing import MinMaxScaler
+import matplotlib.pyplot as plt
+from statsmodels.stats.outliers_influence import variance_inflation_factor
+st.set_option('deprecation.showPyplotGlobalUse', False)
+import statsmodels.formula.api as smf
+from Data_prep_functions import *
+
+for i in ["model_tuned", "X_train_tuned", "X_test_tuned", "tuned_model_features"] :
+    if i not in st.session_state :
+        st.session_state[i] = None
+
+st.set_page_config(
+  page_title="Model Tuning",
+  page_icon=":shark:",
+  layout="wide",
+  initial_sidebar_state='collapsed'
+)
+load_local_css('styles.css')
+set_header()
+
+# Sprint3
+is_panel= True
+panel_col= 'dma' # set the panel column
+date_col = 'date'
+target_col = 'total_approved_accounts_revenue'
+
+st.title('1. Model Tuning')
+
+
+if "X_train" not in st.session_state:
+   st.error(
+"Oops! It seems there are no saved models available. Please build and save a model from the previous page to proceed.")
+   st.stop()
+X_train=st.session_state['X_train']
+X_test=st.session_state['X_test']
+y_train=st.session_state['y_train']
+y_test=st.session_state['y_test']
+df=st.session_state['media_data']
+
+# st.write(X_train.columns)
+# st.write(X_test.columns)
+
+with open("best_models.pkl", 'rb') as file:
+  model_dict= pickle.load(file)
+
+if 'selected_model' not in st.session_state:
+   st.session_state['selected_model']=0
+
+# st.write(model_dict[st.session_state["selected_model"]]['X_train'].columns)
+
+st.markdown('### 1.1 Event Flags')
+st.markdown('Helps in quantifying the impact of specific occurrences of events')
+with st.expander('Apply Event Flags'):
+  st.session_state["selected_model"]=st.selectbox('Select Model to apply flags',model_dict.keys())
+  model =model_dict[st.session_state["selected_model"]]['Model_object']
+  date=st.session_state['date']
+  date=pd.to_datetime(date)
+  X_train =model_dict[st.session_state["selected_model"]]['X_train']
+
+  features_set= model_dict[st.session_state["selected_model"]]['feature_set']
+
+  col=st.columns(3)
+  min_date=min(date)
+  max_date=max(date)
+  with col[0]:
+    start_date=st.date_input('Select Start Date',min_date,min_value=min_date,max_value=max_date)
+  with col[1]:
+    end_date=st.date_input('Select End Date',max_date,min_value=min_date,max_value=max_date)
+  with col[2]:
+    repeat=st.selectbox('Repeat Annually',['Yes','No'],index=1)
+  if repeat =='Yes':
+      repeat=True
+  else:
+      repeat=False
+  # X_train=sm.add_constant(X_train)
+
+  if 'Flags' not in st.session_state:
+    st.session_state['Flags']={}
+  # print("**"*50)
+  # print(y_train)
+  # print("**"*50)
+  # print(model.fittedvalues)
+  if is_panel : # Sprint3
+      met, line_values, fig_flag = plot_actual_vs_predicted(X_train[date_col], y_train,
+                                                            model.fittedvalues, model,
+                                                            target_column='Revenue',
+                                                            flag=(start_date, end_date),
+                                                            repeat_all_years=repeat, is_panel=True)
+      st.plotly_chart(fig_flag, use_container_width=True)
+
+      # create flag on test
+      met, test_line_values, fig_flag = plot_actual_vs_predicted(X_test[date_col], y_test,
+                                                            st.session_state['pred_test'], model,
+                                                            target_column='Revenue',
+                                                            flag=(start_date, end_date),
+                                                            repeat_all_years=repeat, is_panel=True)
+
+  else :
+      met,line_values,fig_flag=plot_actual_vs_predicted(date[:150], y_train, model.predict(X_train), model,flag=(start_date,end_date),repeat_all_years=repeat)
+      st.plotly_chart(fig_flag,use_container_width=True)
+
+      met,test_line_values,fig_flag=plot_actual_vs_predicted(date[150:], y_test, model.predict(X_test), model,flag=(start_date,end_date),repeat_all_years=repeat)
+
+
+  flag_name='f1'
+  flag_name=st.text_input('Enter Flag Name')
+  if st.button('Update flag'):
+    st.session_state['Flags'][flag_name]= {}
+    st.session_state['Flags'][flag_name]['train']=line_values
+    st.session_state['Flags'][flag_name]['test']=test_line_values
+    # st.write(st.session_state['Flags'][flag_name])
+    st.success(f'{flag_name} stored')
+
+  options=list(st.session_state['Flags'].keys())
+  selected_options = []
+  num_columns = 4
+  num_rows = -(-len(options) // num_columns)
+
+
+tick=False
+if st.checkbox('Select all'):
+    tick=True
+selected_options = []
+for row in range(num_rows):
+    cols = st.columns(num_columns)
+    for col in cols:
+        if options:
+            option = options.pop(0)
+            selected = col.checkbox(option,value=tick)
+            if selected:
+                selected_options.append(option)
+
+st.markdown('### 1.2 Select Parameters to Apply')
+parameters=st.columns(3)
+with parameters[0]:
+   Trend=st.checkbox("**Trend**")
+   st.markdown('Helps account for long-term trends or seasonality that could influence advertising effectiveness')
+with parameters[1]:
+   week_number=st.checkbox('**Week_number**')
+   st.markdown('Assists in detecting and incorporating weekly patterns or seasonality')
+with parameters[2]:
+   sine_cosine=st.checkbox('**Sine and Cosine Waves**')
+   st.markdown('Helps in capturing cyclical patterns or seasonality in the data')
+
+if st.button('Build model with Selected Parameters and Flags'):
+  st.header('2.1 Results Summary')
+  # date=list(df.index)
+  # df = df.reset_index(drop=True)
+  # st.write(df.head(2))
+  # X_train=df[features_set]
+  ss = MinMaxScaler()
+  if is_panel == True :
+    X = X_train[features_set]
+    X_train_tuned = pd.DataFrame(ss.fit_transform(X), columns=X.columns)
+    X_train_tuned[target_col] = X_train[target_col]
+    X_train_tuned[date_col] = X_train[date_col]
+    X_train_tuned[panel_col] = X_train[panel_col]
+
+    X = X_test[features_set]
+    X_test_tuned = pd.DataFrame(ss.transform(X), columns=X.columns)
+    X_test_tuned[target_col] = X_test[target_col]
+    X_test_tuned[date_col] = X_test[date_col]
+    X_test_tuned[panel_col] = X_test[panel_col]
+
+  else :
+    X_train_tuned = pd.DataFrame(ss.fit_transform(X_train), columns=X_train.columns)
+    X_train_tuned = sm.add_constant(X_train_tuned)
+
+    X_test_tuned = pd.DataFrame(ss.transform(X_test), columns=X_test.columns)
+    X_test_tuned = sm.add_constant(X_test_tuned)
+
+  for flag in selected_options:
+    X_train_tuned[flag]=st.session_state['Flags'][flag]['train']
+    X_test_tuned[flag]=st.session_state['Flags'][flag]['test']
+
+    #test
+    # X_train_tuned.to_csv("Test/X_train_tuned_flag.csv",index=False)
+    # X_test_tuned.to_csv("Test/X_test_tuned_flag.csv",index=False)
+
+  new_features = features_set
+    # print("()()"*20,flag, len(st.session_state['Flags'][flag]))
+  if Trend:
+     # Sprint3 - group by panel, calculate trend of each panel spearately. Add trend to new feature set
+     if is_panel :
+         newdata = pd.DataFrame()
+         panel_wise_end_point_train = {}
+         for panel, groupdf in X_train_tuned.groupby(panel_col):
+             groupdf.sort_values(date_col, inplace=True)
+             groupdf['Trend'] = np.arange(1, len(groupdf) + 1, 1)
+             newdata = pd.concat([newdata, groupdf])
+             panel_wise_end_point_train[panel] = len(groupdf)
+         X_train_tuned = newdata.copy()
+
+         test_newdata=pd.DataFrame()
+         for panel, test_groupdf in X_test_tuned.groupby(panel_col):
+             test_groupdf.sort_values(date_col, inplace=True)
+             start = panel_wise_end_point_train[panel]+1
+             end = start + len(test_groupdf)
+             # print("??"*20, panel, len(test_groupdf), len(np.arange(start, end, 1)), start)
+             test_groupdf['Trend'] = np.arange(start, end, 1)
+             test_newdata = pd.concat([test_newdata, test_groupdf])
+         X_test_tuned = test_newdata.copy()
+
+         new_features = new_features + ['Trend']
+
+         # test
+         X_test_tuned.to_csv("Test/X_test_tuned_trend.csv", index=False)
+         X_train_tuned.to_csv("Test/X_train_tuned_trend.csv", index=False)
+         pd.concat([X_train_tuned,X_test_tuned]).sort_values([panel_col, date_col]).to_csv("Test/X_train_test_tuned_trend.csv", index=False)
+
+     else :
+         X_train_tuned['Trend']=np.arange(1,len(X_train_tuned)+1,1)
+         X_test_tuned['Trend'] = np.arange(len(X_train_tuned)+1, len(X_train_tuned)+len(X_test_tuned), 1)
+
+  if week_number :
+     # Sprint3 - create weeknumber from date column in xtrain tuned. add week num to new feature set
+     if is_panel :
+        X_train_tuned[date_col] = pd.to_datetime(X_train_tuned[date_col])
+        X_train_tuned['Week_number'] = X_train_tuned[date_col].dt.day_of_week
+        if X_train_tuned['Week_number'].nunique() == 1 :
+            st.write("All dates in the data are of the same week day. Hence Week number can't be used.")
+        else :
+            X_test_tuned[date_col] = pd.to_datetime(X_test_tuned[date_col])
+            X_test_tuned['Week_number'] = X_test_tuned[date_col].dt.day_of_week
+            new_features = new_features + ['Week_number']
+
+     else :
+        date = pd.to_datetime(date.values)
+        X_train_tuned['Week_number'] = date.dt.day_of_week[:150]
+        X_test_tuned['Week_number'] = date.dt.day_of_week[150:]
+
+  if sine_cosine :
+      # Sprint3 - create panel wise sine cosine waves in xtrain tuned. add to new feature set
+      if is_panel :
+        new_features = new_features + ['sine_wave', 'cosine_wave']
+        newdata = pd.DataFrame()
+        groups = X_train_tuned.groupby(panel_col)
+        frequency = 2 * np.pi / 365  # Adjust the frequency as needed
+
+        train_panel_wise_end_point = {}
+        for panel, groupdf in groups:
+            num_samples = len(groupdf)
+            train_panel_wise_end_point[panel] = num_samples
+            days_since_start = np.arange(num_samples)
+            sine_wave = np.sin(frequency * days_since_start)
+            cosine_wave = np.cos(frequency * days_since_start)
+            sine_cosine_df = pd.DataFrame({'sine_wave': sine_wave, 'cosine_wave': cosine_wave})
+            assert len(sine_cosine_df) == len(groupdf)
+            # groupdf = pd.concat([groupdf, sine_cosine_df], axis=1)
+            groupdf['sine_wave'] = sine_wave
+            groupdf['cosine_wave'] = cosine_wave
+            newdata = pd.concat([newdata, groupdf])
+
+        test_groups = X_test_tuned.groupby(panel_col)
+        for panel, test_groupdf in test_groups:
+            num_samples = len(test_groupdf)
+            start = train_panel_wise_end_point[panel]
+            days_since_start = np.arange(start, start+num_samples, 1)
+            # print("##", panel, num_samples, start, len(np.arange(start, start+num_samples, 1)))
+            sine_wave = np.sin(frequency * days_since_start)
+            cosine_wave = np.cos(frequency * days_since_start)
+            sine_cosine_df = pd.DataFrame({'sine_wave': sine_wave, 'cosine_wave': cosine_wave})
+            assert len(sine_cosine_df) == len(test_groupdf)
+            # groupdf = pd.concat([groupdf, sine_cosine_df], axis=1)
+            test_groupdf['sine_wave'] = sine_wave
+            test_groupdf['cosine_wave'] = cosine_wave
+            newdata = pd.concat([newdata, test_groupdf])
+
+        X_train_tuned = newdata.copy()
+
+
+      else :
+        num_samples = len(X_train_tuned)
+        frequency = 2 * np.pi / 365  # Adjust the frequency as needed
+        days_since_start = np.arange(num_samples)
+        sine_wave = np.sin(frequency * days_since_start)
+        cosine_wave = np.cos(frequency * days_since_start)
+        sine_cosine_df = pd.DataFrame({'sine_wave': sine_wave, 'cosine_wave': cosine_wave})
+        # Concatenate the sine and cosine waves with the scaled X DataFrame
+        X_train_tuned = pd.concat([X_train_tuned, sine_cosine_df], axis=1)
+
+        test_num_samples = len(X_test_tuned)
+        start = num_samples
+        days_since_start = np.arange(start, start+test_num_samples, 1)
+        sine_wave = np.sin(frequency * days_since_start)
+        cosine_wave = np.cos(frequency * days_since_start)
+        sine_cosine_df = pd.DataFrame({'sine_wave': sine_wave, 'cosine_wave': cosine_wave})
+        # Concatenate the sine and cosine waves with the scaled X DataFrame
+        X_test_tuned = pd.concat([X_test_tuned, sine_cosine_df], axis=1)
+
+  # model
+
+  if is_panel :
+      if selected_options :
+        new_features =  new_features + selected_options
+
+      inp_vars_str = " + ".join(new_features)
+
+      # X_train_tuned.to_csv("Test/X_train_tuned.csv",index=False)
+      # st.write(X_train_tuned[['total_approved_accounts_revenue'] + new_features].dtypes)
+      # st.write(X_train_tuned[['total_approved_accounts_revenue', panel_col] + new_features].isna().sum())
+
+      md_tuned = smf.mixedlm("total_approved_accounts_revenue ~ {}".format(inp_vars_str),
+                             data=X_train_tuned[['total_approved_accounts_revenue'] + new_features],
+                             groups=X_train_tuned[panel_col])
+      model_tuned = md_tuned.fit()
+
+
+
+      # plot act v pred for original model and tuned model
+      metrics_table, line, actual_vs_predicted_plot = plot_actual_vs_predicted(X_train[date_col], y_train,
+                                                                               model.fittedvalues, model,
+                                                                               target_column='Revenue',
+                                                                               is_panel=True)
+      metrics_table_tuned, line, actual_vs_predicted_plot_tuned = plot_actual_vs_predicted(X_train_tuned[date_col],
+                                                                                           X_train_tuned[target_col],
+                                                                                           model_tuned.fittedvalues,
+                                                                                           model_tuned,
+                                                                                           target_column='Revenue',
+                                                                                           is_panel=True)
+
+  else :
+      model_tuned = sm.OLS(y_train, X_train_tuned).fit()
+
+      metrics_table, line, actual_vs_predicted_plot = plot_actual_vs_predicted(date[:150], y_train,
+                                                                               model.predict(X_train), model,
+                                                                               target_column='Revenue')
+      metrics_table_tuned, line, actual_vs_predicted_plot_tuned = plot_actual_vs_predicted(date[:150], y_train,
+                                                                                           model_tuned.predict(
+                                                                                               X_train_tuned),
+                                                                                           model_tuned,
+                                                                                           target_column='Revenue')
+
+  # st.write(metrics_table_tuned)
+  mape=np.round(metrics_table.iloc[0,1],2)
+  r2=np.round(metrics_table.iloc[1,1],2)
+  adjr2=np.round(metrics_table.iloc[2,1],2)
+
+  mape_tuned=np.round(metrics_table_tuned.iloc[0,1],2)
+  r2_tuned=np.round(metrics_table_tuned.iloc[1,1],2)
+  adjr2_tuned=np.round(metrics_table_tuned.iloc[2,1],2)
+
+  parameters_=st.columns(3)
+  with parameters_[0]:
+     st.metric('R2',r2_tuned,np.round(r2_tuned-r2,2))
+  with parameters_[1]:
+     st.metric('Adjusted R2',adjr2_tuned,np.round(adjr2_tuned-adjr2,2))
+  with parameters_[2]:
+     st.metric('MAPE',mape_tuned,np.round(mape_tuned-mape,2),'inverse')
+
+  st.header('2.2 Actual vs. Predicted Plot')
+  # if is_panel:
+  #   metrics_table, line, actual_vs_predicted_plot = plot_actual_vs_predicted(date, y_train, model.predict(X_train),
+  #                                                                              model, target_column='Revenue',is_panel=True)
+  # else:
+  #   metrics_table,line,actual_vs_predicted_plot=plot_actual_vs_predicted(date, y_train, model.predict(X_train), model,target_column='Revenue')
+
+  metrics_table,line,actual_vs_predicted_plot=plot_actual_vs_predicted(X_train_tuned[date_col], X_train_tuned[target_col],
+                                                                                  model_tuned.fittedvalues, model_tuned,
+                                                                                  target_column='Revenue',
+                                                                                  is_panel=True)
+      # plot_actual_vs_predicted(X_train[date_col], y_train,
+      #                                                                             model.fittedvalues, model,
+      #                                                                             target_column='Revenue',
+      #                                                                             is_panel=is_panel)
+
+  st.plotly_chart(actual_vs_predicted_plot,use_container_width=True)
+
+
+
+  st.markdown('## 2.3 Residual Analysis')
+  columns=st.columns(2)
+  with columns[0]:
+    fig=plot_residual_predicted(y_train,model.predict(X_train),X_train)
+    st.plotly_chart(fig)
+
+  with columns[1]:
+    st.empty()
+    fig = qqplot(y_train,model.predict(X_train))
+    st.plotly_chart(fig)
+
+  with columns[0]:
+    fig=residual_distribution(y_train,model.predict(X_train))
+    st.pyplot(fig)
+
+  if st.checkbox('Use this model to build response curves',key='123'):
+    st.session_state["tuned_model"] = model_tuned
+    st.session_state["X_train_tuned"] = X_train_tuned
+    st.session_state["X_test_tuned"] = X_test_tuned
+    st.session_state["X_train_tuned"] = X_train_tuned
+    st.session_state["X_test_tuned"] = X_test_tuned
+    if is_panel :
+        st.session_state["tuned_model_features"] = new_features
+    with open("tuned_model.pkl", "wb") as f:
+        pickle.dump(st.session_state['tuned_model'], f)
+        st.success('Model saved!')
+
+#   raw_data=df[features_set]
+#   columns_raw=[re.split(r"(_lag|_adst)",col)[0] for col in raw_data.columns]
+#   raw_data.columns=columns_raw
+#   columns_media=[col for col in columns_raw if Categorised_data[col]['BB']=='Media']
+#   raw_data=raw_data[columns_media]
+
+#   raw_data['Date']=list(df.index)
+
+#   spends_var=[col for col in df.columns if "spends" in col.lower() and 'adst' not in col.lower() and 'lag' not in col.lower()]
+#   spends_df=df[spends_var]
+#   spends_df['Week']=list(df.index)
+
+
+#   j=0
+#   X1=X.copy()
+#   col=X1.columns
+#   for i in model.params.values:
+#       X1[col[j]]=X1.iloc[:,j]*i
+#       j+=1
+#   contribution_df=X1
+#   contribution_df['Date']=list(df.index)
+#   excel_file='Overview_data.xlsx'
+
+#   with pd.ExcelWriter(excel_file,engine='xlsxwriter') as writer:
+#      raw_data.to_excel(writer,sheet_name='RAW DATA MMM',index=False)
+#      spends_df.to_excel(writer,sheet_name='SPEND INPUT',index=False)
+#      contribution_df.to_excel(writer,sheet_name='CONTRIBUTION MMM')

pages/4_Saved_Model_Results.py

@@ -7,14 +7,16 @@ import statsmodels.api as sm
 from sklearn.metrics import mean_absolute_percentage_error
 import sys
 import os
-from utilities import set_header, load_local_css, load_authenticator
+from utilities import (set_header, 
+                       load_local_css,
+                       load_authenticator)
 import seaborn as sns
 import matplotlib.pyplot as plt
 import sweetviz as sv
 import tempfile
 from sklearn.preprocessing import MinMaxScaler
 from st_aggrid import AgGrid
-from st_aggrid import GridOptionsBuilder, GridUpdateMode
+from st_aggrid import GridOptionsBuilder,GridUpdateMode
 from st_aggrid import GridOptionsBuilder
 import sys
 import re
@@ -22,586 +24,390 @@ import re
 sys.setrecursionlimit(10**6)
 
 original_stdout = sys.stdout
-sys.stdout = open("temp_stdout.txt", "w")
+sys.stdout = open('temp_stdout.txt', 'w')
 sys.stdout.close()
 sys.stdout = original_stdout
 
-st.set_page_config(layout="wide")
-load_local_css("styles.css")
+st.set_page_config(layout='wide')
+load_local_css('styles.css')
 set_header()
 
 for k, v in st.session_state.items():
-    if k not in ["logout", "login", "config"] and not k.startswith("FormSubmitter"):
+    if k not in ['logout', 'login','config'] and not k.startswith('FormSubmitter'):
         st.session_state[k] = v
 
-authenticator = st.session_state.get("authenticator")
+authenticator = st.session_state.get('authenticator')
 if authenticator is None:
     authenticator = load_authenticator()
 
-name, authentication_status, username = authenticator.login("Login", "main")
-auth_status = st.session_state.get("authentication_status")
+name, authentication_status, username = authenticator.login('Login', 'main')
+auth_status = st.session_state.get('authentication_status')
 
 if auth_status == True:
-    is_state_initiaized = st.session_state.get("initialized", False)
+    is_state_initiaized = st.session_state.get('initialized',False)
     if not is_state_initiaized:
-        a = 1
+        a=1
+      
 
     def plot_residual_predicted(actual, predicted, df_):
-        df_["Residuals"] = actual - pd.Series(predicted)
-        df_["StdResidual"] = (df_["Residuals"] - df_["Residuals"].mean()) / df_[
-            "Residuals"
-        ].std()
-
-        # Create a Plotly scatter plot
-        fig = px.scatter(
-            df_,
-            x=predicted,
-            y="StdResidual",
-            opacity=0.5,
-            color_discrete_sequence=["#11B6BD"],
-        )
-
-        # Add horizontal lines
-        fig.add_hline(y=0, line_dash="dash", line_color="darkorange")
-        fig.add_hline(y=2, line_color="red")
-        fig.add_hline(y=-2, line_color="red")
-
-        fig.update_xaxes(title="Predicted")
-        fig.update_yaxes(title="Standardized Residuals (Actual - Predicted)")
-
-        # Set the same width and height for both figures
-        fig.update_layout(
-            title="Residuals over Predicted Values",
-            autosize=False,
-            width=600,
-            height=400,
-        )
-
-        return fig
+            df_['Residuals'] = actual - pd.Series(predicted)
+            df_['StdResidual'] = (df_['Residuals'] - df_['Residuals'].mean()) / df_['Residuals'].std()
+            
+            # Create a Plotly scatter plot
+            fig = px.scatter(df_, x=predicted, y='StdResidual', opacity=0.5,color_discrete_sequence=["#11B6BD"])
+            
+            # Add horizontal lines
+            fig.add_hline(y=0, line_dash="dash", line_color="darkorange")
+            fig.add_hline(y=2, line_color="red")
+            fig.add_hline(y=-2, line_color="red")
+            
+            fig.update_xaxes(title='Predicted')
+            fig.update_yaxes(title='Standardized Residuals (Actual - Predicted)')
+            
+            # Set the same width and height for both figures
+            fig.update_layout(title='Residuals over Predicted Values', autosize=False, width=600, height=400)
+            
+            return fig
 
     def residual_distribution(actual, predicted):
-        Residuals = actual - pd.Series(predicted)
-
-        # Create a Seaborn distribution plot
-        sns.set(style="whitegrid")
-        plt.figure(figsize=(6, 4))
-        sns.histplot(Residuals, kde=True, color="#11B6BD")
-
-        plt.title(" Distribution of Residuals")
-        plt.xlabel("Residuals")
-        plt.ylabel("Probability Density")
-
-        return plt
-
+            Residuals = actual - pd.Series(predicted)
+            
+            # Create a Seaborn distribution plot
+            sns.set(style="whitegrid")
+            plt.figure(figsize=(6, 4))
+            sns.histplot(Residuals, kde=True, color="#11B6BD")
+            
+            plt.title(' Distribution of Residuals')
+            plt.xlabel('Residuals')
+            plt.ylabel('Probability Density')
+            
+            return plt
+
+    
     def qqplot(actual, predicted):
-        Residuals = actual - pd.Series(predicted)
-        Residuals = pd.Series(Residuals)
-        Resud_std = (Residuals - Residuals.mean()) / Residuals.std()
-
-        # Create a QQ plot using Plotly with custom colors
-        fig = go.Figure()
-        fig.add_trace(
-            go.Scatter(
-                x=sm.ProbPlot(Resud_std).theoretical_quantiles,
-                y=sm.ProbPlot(Resud_std).sample_quantiles,
-                mode="markers",
-                marker=dict(size=5, color="#11B6BD"),
-                name="QQ Plot",
+            Residuals = actual - pd.Series(predicted)
+            Residuals = pd.Series(Residuals)
+            Resud_std = (Residuals - Residuals.mean()) / Residuals.std()
+            
+            # Create a QQ plot using Plotly with custom colors
+            fig = go.Figure()
+            fig.add_trace(go.Scatter(x=sm.ProbPlot(Resud_std).theoretical_quantiles,
+                                    y=sm.ProbPlot(Resud_std).sample_quantiles,
+                                    mode='markers',
+                                    marker=dict(size=5, color="#11B6BD"),
+                                    name='QQ Plot'))
+            
+            # Add the 45-degree reference line
+            diagonal_line = go.Scatter(
+                x=[-2, 2],  # Adjust the x values as needed to fit the range of your data
+                y=[-2, 2],  # Adjust the y values accordingly
+                mode='lines',
+                line=dict(color='red'),  # Customize the line color and style
+                name=' '
             )
-        )
-
-        # Add the 45-degree reference line
-        diagonal_line = go.Scatter(
-            x=[-2, 2],  # Adjust the x values as needed to fit the range of your data
-            y=[-2, 2],  # Adjust the y values accordingly
-            mode="lines",
-            line=dict(color="red"),  # Customize the line color and style
-            name=" ",
-        )
-        fig.add_trace(diagonal_line)
-
-        # Customize the layout
-        fig.update_layout(
-            title="QQ Plot of Residuals",
-            title_x=0.5,
-            autosize=False,
-            width=600,
-            height=400,
-            xaxis_title="Theoretical Quantiles",
-            yaxis_title="Sample Quantiles",
-        )
+            fig.add_trace(diagonal_line)
+            
+            # Customize the layout
+            fig.update_layout(title='QQ Plot of Residuals',title_x=0.5, autosize=False, width=600, height=400,
+                            xaxis_title='Theoretical Quantiles', yaxis_title='Sample Quantiles')
+            
+            return fig
 
-        return fig
 
     def plot_actual_vs_predicted(date, y, predicted_values, model):
 
         fig = go.Figure()
 
-        fig.add_trace(
-            go.Scatter(
-                x=date, y=y, mode="lines", name="Actual", line=dict(color="blue")
-            )
-        )
-        fig.add_trace(
-            go.Scatter(
-                x=date,
-                y=predicted_values,
-                mode="lines",
-                name="Predicted",
-                line=dict(color="orange"),
-            )
-        )
-
+        fig.add_trace(go.Scatter(x=date, y=y, mode='lines', name='Actual', line=dict(color='blue')))
+        fig.add_trace(go.Scatter(x=date, y=predicted_values, mode='lines', name='Predicted', line=dict(color='orange')))
+        
         # Calculate MAPE
-        mape = mean_absolute_percentage_error(y, predicted_values) * 100
-
+        mape = mean_absolute_percentage_error(y, predicted_values)*100
+        
         # Calculate R-squared
         rss = np.sum((y - predicted_values) ** 2)
         tss = np.sum((y - np.mean(y)) ** 2)
         r_squared = 1 - (rss / tss)
-
+        
         # Get the number of predictors
         num_predictors = model.df_model
-
+        
         # Get the number of samples
         num_samples = len(y)
-
+        
         # Calculate Adjusted R-squared
-        adj_r_squared = 1 - (
-            (1 - r_squared) * ((num_samples - 1) / (num_samples - num_predictors - 1))
-        )
-        metrics_table = pd.DataFrame(
-            {
-                "Metric": ["MAPE", "R-squared", "AdjR-squared"],
-                "Value": [mape, r_squared, adj_r_squared],
-            }
-        )
+        adj_r_squared = 1 - ((1 - r_squared) * ((num_samples - 1) / (num_samples - num_predictors - 1)))
+        metrics_table = pd.DataFrame({
+        'Metric': ['MAPE', 'R-squared', 'AdjR-squared'],
+        'Value': [mape, r_squared, adj_r_squared]})
         fig.update_layout(
-            xaxis=dict(title="Date"),
-            yaxis=dict(title="Value"),
-            title=f"MAPE : {mape:.2f}%, AdjR2: {adj_r_squared:.2f}",
-            xaxis_tickangle=-30,
+            xaxis=dict(title='Date'),
+            yaxis=dict(title='Value'),
+            title=f'MAPE : {mape:.2f}%, AdjR2: {adj_r_squared:.2f}',
+            xaxis_tickangle=-30
         )
 
-        return metrics_table, fig
-
+        return metrics_table,fig
     def contributions(X, model):
         X1 = X.copy()
         for j, col in enumerate(X1.columns):
             X1[col] = X1[col] * model.params.values[j]
 
-        return np.round(
-            (X1.sum() / sum(X1.sum()) * 100).sort_values(ascending=False), 2
-        )
+        return np.round((X1.sum() / sum(X1.sum()) * 100).sort_values(ascending=False), 2)
 
-    transformed_data = pd.read_csv("transformed_data.csv")
+    transformed_data=pd.read_csv('transformed_data.csv')
 
     # hard coded for now, need to get features set from model
 
-    feature_set_dct = {
-        "app_installs_-_appsflyer": [
-            "paid_search_clicks",
-            "fb:_level_achieved_-_tier_1_impressions_lag2",
-            "fb:_level_achieved_-_tier_2_clicks_lag2",
-            "paid_social_others_impressions_adst.1",
-            "ga_app:_will_and_cid_pequena_baixo_risco_clicks_lag2",
-            "digital_tactic_others_clicks",
-            "kwai_clicks_adst.3",
-            "programmaticclicks",
-            "indicacao_clicks_adst.1",
-            "infleux_clicks_adst.4",
-            "influencer_clicks",
-        ],
-        "account_requests_-_appsflyer": [
-            "paid_search_impressions",
-            "fb:_level_achieved_-_tier_1_clicks_adst.1",
-            "fb:_level_achieved_-_tier_2_clicks_adst.1",
-            "paid_social_others_clicks_lag2",
-            "ga_app:_will_and_cid_pequena_baixo_risco_clicks_lag5_adst.1",
-            "digital_tactic_others_clicks_adst.1",
-            "kwai_clicks_adst.2",
-            "programmaticimpressions_lag4_adst.1",
-            "indicacao_clicks",
-            "infleux_clicks_adst.2",
-            "influencer_clicks",
-        ],
-        "total_approved_accounts_-_appsflyer": [
-            "paid_search_clicks",
-            "fb:_level_achieved_-_tier_1_impressions_lag2_adst.1",
-            "fb:_level_achieved_-_tier_2_impressions_lag2",
-            "paid_social_others_clicks_lag2_adst.2",
-            "ga_app:_will_and_cid_pequena_baixo_risco_impressions_lag4",
-            "digital_tactic_others_clicks",
-            "kwai_impressions_adst.2",
-            "programmaticclicks_adst.5",
-            "indicacao_clicks_adst.1",
-            "infleux_clicks_adst.3",
-            "influencer_clicks",
-        ],
-        "total_approved_accounts_-_revenue": [
-            "paid_search_impressions_adst.5",
-            "kwai_impressions_lag2_adst.3",
-            "indicacao_clicks_adst.3",
-            "infleux_clicks_adst.3",
-            "programmaticclicks_adst.4",
-            "influencer_clicks_adst.3",
-            "fb:_level_achieved_-_tier_1_impressions_adst.2",
-            "fb:_level_achieved_-_tier_2_impressions_lag3_adst.5",
-            "paid_social_others_impressions_adst.3",
-            "ga_app:_will_and_cid_pequena_baixo_risco_clicks_lag3_adst.5",
-            "digital_tactic_others_clicks_adst.2",
-        ],
-    }
-
-    # """ the above part should be modified so that we are fetching features set from the saved model"""
-
-    def contributions(X, model, target):
+    feature_set_dct={'app_installs_-_appsflyer':['paid_search_clicks',
+                                            'fb:_level_achieved_-_tier_1_impressions_lag2',
+                                            'fb:_level_achieved_-_tier_2_clicks_lag2',
+                                            'paid_social_others_impressions_adst.1',
+                                            'ga_app:_will_and_cid_pequena_baixo_risco_clicks_lag2',
+                                            'digital_tactic_others_clicks',
+                                            'kwai_clicks_adst.3',
+                                            'programmaticclicks',
+                                            'indicacao_clicks_adst.1',
+                                            'infleux_clicks_adst.4',
+                                            'influencer_clicks'],
+                                            
+                'account_requests_-_appsflyer':['paid_search_impressions',
+                                                'fb:_level_achieved_-_tier_1_clicks_adst.1',
+                                                'fb:_level_achieved_-_tier_2_clicks_adst.1',
+                                                'paid_social_others_clicks_lag2',
+                                                'ga_app:_will_and_cid_pequena_baixo_risco_clicks_lag5_adst.1',
+                                                'digital_tactic_others_clicks_adst.1',
+                                                'kwai_clicks_adst.2',
+                                                'programmaticimpressions_lag4_adst.1',
+                                                'indicacao_clicks',
+                                                'infleux_clicks_adst.2',
+                                                'influencer_clicks'],
+
+                'total_approved_accounts_-_appsflyer':['paid_search_clicks',
+                                                        'fb:_level_achieved_-_tier_1_impressions_lag2_adst.1',
+                                                        'fb:_level_achieved_-_tier_2_impressions_lag2',
+                                                        'paid_social_others_clicks_lag2_adst.2',
+                                                        'ga_app:_will_and_cid_pequena_baixo_risco_impressions_lag4',
+                                                        'digital_tactic_others_clicks',
+                                                        'kwai_impressions_adst.2',
+                                                        'programmaticclicks_adst.5',
+                                                        'indicacao_clicks_adst.1',
+                                                        'infleux_clicks_adst.3',
+                                                        'influencer_clicks'],
+
+                'total_approved_accounts_-_revenue':['paid_search_impressions_adst.5',
+                                                    'kwai_impressions_lag2_adst.3',
+                                                    'indicacao_clicks_adst.3',
+                                                    'infleux_clicks_adst.3',
+                                                    'programmaticclicks_adst.4',
+                                                    'influencer_clicks_adst.3',
+                                                    'fb:_level_achieved_-_tier_1_impressions_adst.2',
+                                                    'fb:_level_achieved_-_tier_2_impressions_lag3_adst.5',
+                                                    'paid_social_others_impressions_adst.3',
+                                                    'ga_app:_will_and_cid_pequena_baixo_risco_clicks_lag3_adst.5',
+                                                    'digital_tactic_others_clicks_adst.2']                       
+                                            
+                }
+
+    #""" the above part should be modified so that we are fetching features set from the saved model"""
+
+
+
+    def contributions(X, model,target):
         X1 = X.copy()
         for j, col in enumerate(X1.columns):
             X1[col] = X1[col] * model.params.values[j]
-
-        contributions = np.round(
-            (X1.sum() / sum(X1.sum()) * 100).sort_values(ascending=False), 2
-        )
-        contributions = (
-            pd.DataFrame(contributions, columns=target)
-            .reset_index()
-            .rename(columns={"index": "Channel"})
-        )
-        contributions["Channel"] = [
-            re.split(r"_imp|_cli", col)[0] for col in contributions["Channel"]
-        ]
-
+        
+        contributions= np.round((X1.sum() / sum(X1.sum()) * 100).sort_values(ascending=False), 2)
+        contributions=pd.DataFrame(contributions,columns=target).reset_index().rename(columns={'index':'Channel'})
+        contributions['Channel']=[ re.split(r'_imp|_cli', col)[0] for col in contributions['Channel']]
+        
         return contributions
+    
 
-    def model_fit(features_set, target):
+    def model_fit(features_set,target):
         X = transformed_data[features_set]
-        y = transformed_data[target]
+        y=  transformed_data[target]
         ss = MinMaxScaler()
         X = pd.DataFrame(ss.fit_transform(X), columns=X.columns)
         X = sm.add_constant(X)
-        X_train = X.iloc[:150]
-        X_test = X.iloc[150:]
-        y_train = y.iloc[:150]
-        y_test = y.iloc[150:]
+        X_train=X.iloc[:150]
+        X_test=X.iloc[150:]
+        y_train=y.iloc[:150]
+        y_test=y.iloc[150:]
         model = sm.OLS(y_train, X_train).fit()
         predicted_values_train = model.predict(X_train)
         r2 = model.rsquared
         adjr2 = model.rsquared_adj
         train_mape = mean_absolute_percentage_error(y_train, predicted_values_train)
-        test_mape = mean_absolute_percentage_error(y_test, model.predict(X_test))
-        summary = model.summary()
-        train_contributions = contributions(X_train, model, [target])
-        return (
-            pd.DataFrame(
-                {
-                    "Model": target,
-                    "R2": np.round(r2, 2),
-                    "ADJr2": np.round(adjr2, 2),
-                    "Train Mape": np.round(train_mape, 2),
-                    "Test Mape": np.round(test_mape, 2),
-                    "Summary": summary,
-                    "Model_object": model,
-                },
-                index=[0],
-            ),
-            train_contributions,
-        )
+        test_mape=mean_absolute_percentage_error(y_test, model.predict(X_test))
+        summary=model.summary()
+        train_contributions=contributions(X_train,model,[target])
+        return pd.DataFrame({'Model':target,'R2':np.round(r2,2),'ADJr2':np.round(adjr2,2),'Train Mape':np.round(train_mape,2),
+                             'Test Mape':np.round(test_mape,2),'Summary':summary,'Model_object':model
+                             },index=[0]), train_contributions
 
-    metrics_table = pd.DataFrame()
+    metrics_table=pd.DataFrame()
 
-    if "contribution_df" not in st.session_state:
-        st.session_state["contribution_df"] = pd.DataFrame()
+    if 'contribution_df' not in st.session_state:
+        st.session_state["contribution_df"]=pd.DataFrame()
 
-    for target, feature_set in feature_set_dct.items():
-        metrics_table = pd.concat(
-            [metrics_table, model_fit(features_set=feature_set, target=target)[0]]
-        )
-        if st.session_state["contribution_df"].empty:
-            st.session_state["contribution_df"] = model_fit(
-                features_set=feature_set, target=target
-            )[1]
-        else:
-            st.session_state["contribution_df"] = pd.merge(
-                st.session_state["contribution_df"],
-                model_fit(features_set=feature_set, target=target)[1],
-            )
+    for target,feature_set in feature_set_dct.items():
+       metrics_table= pd.concat([metrics_table,model_fit(features_set=feature_set,target=target)[0]])
+       if st.session_state["contribution_df"].empty:
+           st.session_state["contribution_df"]= model_fit(features_set=feature_set,target=target)[1]
+       else:
+        st.session_state["contribution_df"]=pd.merge(st.session_state["contribution_df"],model_fit(features_set=feature_set,target=target)[1])
 
     # st.write(st.session_state["contribution_df"])
+        
+
+    metrics_table.reset_index(drop=True,inplace=True)
+
 
-    metrics_table.reset_index(drop=True, inplace=True)
 
-    eda_columns = st.columns(2)
+    
+    
+
+
+
+    eda_columns=st.columns(2)
     with eda_columns[1]:
-        eda = st.button(
-            "Generate EDA Report",
-            help="Click to generate a bivariate report for the selected response metric from the table below.",
-        )
+        eda=st.button('Generate EDA Report',help="Click to generate a bivariate report for the selected response metric from the table below.")
+    
+
 
     # st.markdown('Model Metrics')
+        
+    st.title('Contribution Overview')
 
-    st.title("Contribution Overview")
-
-    contribution_selections = st.multiselect(
-        "Select the models to compare contributions",
-        [
-            col
-            for col in st.session_state["contribution_df"].columns
-            if col.lower() != "channel"
-        ],
-        default=[
-            col
-            for col in st.session_state["contribution_df"].columns
-            if col.lower() != "channel"
-        ][-1],
-    )
-    trace_data = []
+    contribution_selections=st.multiselect('Select the models to compare contributions',[col for col in st.session_state['contribution_df'].columns if col.lower() != 'channel'   ],default=[col for col in st.session_state['contribution_df'].columns if col.lower() != 'channel'   ][-1])
+    trace_data=[]
 
     for selection in contribution_selections:
 
-        trace = go.Bar(
-            x=st.session_state["contribution_df"]["Channel"],
-            y=st.session_state["contribution_df"][selection],
-            name=selection,
-            text=np.round(st.session_state["contribution_df"][selection], 0)
-            .astype(int)
-            .astype(str)
-            + "%",
-            textposition="outside",
-        )
+        trace=go.Bar(x=st.session_state['contribution_df']['Channel'], y=st.session_state['contribution_df'][selection],name=selection,text=np.round(st.session_state['contribution_df'][selection],0).astype(int).astype(str)+'%',textposition='outside')
         trace_data.append(trace)
 
     layout = go.Layout(
-        title="Metrics Contribution by Channel",
-        xaxis=dict(title="Channel Name"),
-        yaxis=dict(title="Metrics Contribution"),
-        barmode="group",
-    )
+    title='Metrics Contribution by Channel',
+    xaxis=dict(title='Channel Name'),
+    yaxis=dict(title='Metrics Contribution'),
+    barmode='group'
+                )
     fig = go.Figure(data=trace_data, layout=layout)
-    st.plotly_chart(fig, use_container_width=True)
-
-    ############################################ Waterfall Chart ############################################
-    # import plotly.graph_objects as go
-
-    # # Initialize a Plotly figure
-    # fig = go.Figure()
-
-    # for selection in contribution_selections:
-    #     # Ensure y_values are numeric
-    #     y_values = st.session_state["contribution_df"][selection].values.astype(float)
-
-    #     # Generating text labels for each bar, ensuring operations are compatible with string formats
-    #     text_values = [f"{val}%" for val in np.round(y_values, 0).astype(int)]
-
-    #     fig.add_trace(
-    #         go.Waterfall(
-    #             name=selection,
-    #             orientation="v",
-    #             measure=["relative"]
-    #             * len(y_values),  # Adjust if you have absolute values at certain points
-    #             x=st.session_state["contribution_df"]["Channel"].tolist(),
-    #             text=text_values,
-    #             textposition="outside",
-    #             y=y_values,
-    #             increasing={"marker": {"color": "green"}},
-    #             decreasing={"marker": {"color": "red"}},
-    #             totals={"marker": {"color": "blue"}},
-    #         )
-    #     )
-
-    # fig.update_layout(
-    #     title="Metrics Contribution by Channel",
-    #     xaxis={"title": "Channel Name"},
-    #     yaxis={"title": "Metrics Contribution"},
-    #     height=600,
-    # )
-
-    # # Displaying the waterfall chart in Streamlit
-    # st.plotly_chart(fig, use_container_width=True)
-
-    import plotly.graph_objects as go
-
-    # Initialize a Plotly figure
-    fig = go.Figure()
+    st.plotly_chart(fig,use_container_width=True)
 
-    for selection in contribution_selections:
-        # Ensure contributions are numeric
-        contributions = (
-            st.session_state["contribution_df"][selection].values.astype(float).tolist()
-        )
-        channel_names = st.session_state["contribution_df"]["Channel"].tolist()
+    st.title('Analysis of Models Result')
+    #st.markdown()
+    gd_table=metrics_table.iloc[:,:-2]
+    gd=GridOptionsBuilder.from_dataframe(gd_table)
+    #gd.configure_pagination(enabled=True)
+    gd.configure_selection(use_checkbox=True)
 
-        display_name, display_contribution, base_contribution = [], [], 0
-        for channel_name, contribution in zip(channel_names, contributions):
-            if channel_name != "const":
-                display_name.append(channel_name)
-                display_contribution.append(contribution)
-            else:
-                base_contribution = contribution
-
-        display_name = ["Base Sales"] + display_name
-        display_contribution = [base_contribution] + display_contribution
-
-        # Generating text labels for each bar, ensuring operations are compatible with string formats
-        text_values = [
-            f"{val}%" for val in np.round(display_contribution, 0).astype(int)
-        ]
-
-        fig.add_trace(
-            go.Waterfall(
-                orientation="v",
-                measure=["relative"]
-                * len(
-                    display_contribution
-                ),  # Adjust if you have absolute values at certain points
-                x=display_name,
-                text=text_values,
-                textposition="outside",
-                y=display_contribution,
-                increasing={"marker": {"color": "green"}},
-                decreasing={"marker": {"color": "red"}},
-                totals={"marker": {"color": "blue"}},
-            )
-        )
 
-    fig.update_layout(
-        title="Metrics Contribution by Channel",
-        xaxis={"title": "Channel Name"},
-        yaxis={"title": "Metrics Contribution"},
-        height=600,
-    )
-
-    # Displaying the waterfall chart in Streamlit
-    st.plotly_chart(fig, use_container_width=True)
-
-    ############################################ Waterfall Chart ############################################
-
-    st.title("Analysis of Models Result")
-    # st.markdown()
-    gd_table = metrics_table.iloc[:, :-2]
-
-    gd = GridOptionsBuilder.from_dataframe(gd_table)
-    # gd.configure_pagination(enabled=True)
-    gd.configure_selection(
-        use_checkbox=True,
-        selection_mode="single",
-        pre_select_all_rows=False,
-        pre_selected_rows=[1],
-    )
-
-    gridoptions = gd.build()
-    table = AgGrid(
-        gd_table, gridOptions=gridoptions, fit_columns_on_grid_load=True, height=200
-    )
+    gridoptions=gd.build()
+    table = AgGrid(gd_table,gridOptions=gridoptions,fit_columns_on_grid_load=True,height=200)
     # table=metrics_table.iloc[:,:-2]
     # table.insert(0, "Select", False)
     # selection_table=st.data_editor(table,column_config={"Select": st.column_config.CheckboxColumn(required=True)})
+    
 
-    if len(table.selected_rows) == 0:
-        st.warning(
-            "Click on the checkbox to view comprehensive results of the selected model."
-        )
+
+    if len(table.selected_rows)==0:
+        st.warning("Click on the checkbox to view comprehensive results of the selected model.")
         st.stop()
-    else:
-        target_column = table.selected_rows[0]["Model"]
-        feature_set = feature_set_dct[target_column]
+    else:    
+        target_column=table.selected_rows[0]['Model']
+        feature_set=feature_set_dct[target_column]
 
     with eda_columns[1]:
         if eda:
-
             def generate_report_with_target(channel_data, target_feature):
-                report = sv.analyze(
-                    [channel_data, "Dataset"], target_feat=target_feature, verbose=False
-                )
+                report = sv.analyze([channel_data, "Dataset"], target_feat=target_feature,verbose=False)
                 temp_dir = tempfile.mkdtemp()
                 report_path = os.path.join(temp_dir, "report.html")
-                report.show_html(
-                    filepath=report_path, open_browser=False
-                )  # Generate the report as an HTML file
+                report.show_html(filepath=report_path, open_browser=False)  # Generate the report as an HTML file
                 return report_path
-
-            report_data = transformed_data[feature_set]
-            report_data[target_column] = transformed_data[target_column]
+            
+            report_data=transformed_data[feature_set]
+            report_data[target_column]=transformed_data[target_column]
             report_file = generate_report_with_target(report_data, target_column)
-
+        
             if os.path.exists(report_file):
-                with open(report_file, "rb") as f:
+                with open(report_file, 'rb') as f:
                     st.download_button(
                         label="Download EDA Report",
                         data=f.read(),
                         file_name="report.html",
-                        mime="text/html",
+                        mime="text/html"
                     )
             else:
                 st.warning("Report generation failed. Unable to find the report file.")
 
-    model = metrics_table[metrics_table["Model"] == target_column]["Model_object"].iloc[
-        0
-    ]
-    st.header("Model Summary")
+
+
+    model=metrics_table[metrics_table['Model']==target_column]['Model_object'].iloc[0]
+    st.header('Model Summary')
     st.write(model.summary())
-    X = transformed_data[feature_set]
-    ss = MinMaxScaler()
-    X = pd.DataFrame(ss.fit_transform(X), columns=X.columns)
-    X = sm.add_constant(X)
-    y = transformed_data[target_column]
-    X_train = X.iloc[:150]
-    X_test = X.iloc[150:]
-    y_train = y.iloc[:150]
-    y_test = y.iloc[150:]
-    X.index = transformed_data["date"]
-    y.index = transformed_data["date"]
-
-    metrics_table_train, fig_train = plot_actual_vs_predicted(
-        X_train.index, y_train, model.predict(X_train), model
-    )
-    metrics_table_test, fig_test = plot_actual_vs_predicted(
-        X_test.index, y_test, model.predict(X_test), model
-    )
-
-    metrics_table_train = metrics_table_train.set_index("Metric").transpose()
-    metrics_table_train.index = ["Train"]
-    metrics_table_test = metrics_table_test.set_index("Metric").transpose()
-    metrics_table_test.index = ["test"]
-    metrics_table = np.round(pd.concat([metrics_table_train, metrics_table_test]), 2)
-
-    st.markdown("Result Overview")
-    st.dataframe(np.round(metrics_table, 2), use_container_width=True)
-
-    st.subheader("Actual vs Predicted Plot Train")
-
-    st.plotly_chart(fig_train, use_container_width=True)
-    st.subheader("Actual vs Predicted Plot Test")
-    st.plotly_chart(fig_test, use_container_width=True)
-
-    st.markdown("## Residual Analysis")
-    columns = st.columns(2)
-
-    Xtrain1 = X_train.copy()
+    X=transformed_data[feature_set]
+    ss=MinMaxScaler()
+    X=pd.DataFrame(ss.fit_transform(X),columns=X.columns)
+    X=sm.add_constant(X)
+    y=transformed_data[target_column]
+    X_train=X.iloc[:150]
+    X_test=X.iloc[150:]
+    y_train=y.iloc[:150]
+    y_test=y.iloc[150:]
+    X.index=transformed_data['date']
+    y.index=transformed_data['date']
+
+    metrics_table_train,fig_train= plot_actual_vs_predicted(X_train.index, y_train, model.predict(X_train), model)
+    metrics_table_test,fig_test= plot_actual_vs_predicted(X_test.index, y_test, model.predict(X_test), model)
+
+    metrics_table_train=metrics_table_train.set_index('Metric').transpose()
+    metrics_table_train.index=['Train']
+    metrics_table_test=metrics_table_test.set_index('Metric').transpose()
+    metrics_table_test.index=['test']
+    metrics_table=np.round(pd.concat([metrics_table_train,metrics_table_test]),2) 
+
+    st.markdown('Result Overview')
+    st.dataframe(np.round(metrics_table,2),use_container_width=True)
+
+    st.subheader('Actual vs Predicted Plot Train')
+
+    st.plotly_chart(fig_train,use_container_width=True)
+    st.subheader('Actual vs Predicted Plot Test')
+    st.plotly_chart(fig_test,use_container_width=True)
+
+    st.markdown('## Residual Analysis')
+    columns=st.columns(2)
+
+
+    Xtrain1=X_train.copy()
     with columns[0]:
-        fig = plot_residual_predicted(y_train, model.predict(Xtrain1), Xtrain1)
+        fig=plot_residual_predicted(y_train,model.predict(Xtrain1),Xtrain1)
         st.plotly_chart(fig)
 
     with columns[1]:
         st.empty()
-        fig = qqplot(y_train, model.predict(X_train))
+        fig = qqplot(y_train,model.predict(X_train))
         st.plotly_chart(fig)
 
     with columns[0]:
-        fig = residual_distribution(y_train, model.predict(X_train))
+        fig=residual_distribution(y_train,model.predict(X_train))
         st.pyplot(fig)
 
 
+
 elif auth_status == False:
-    st.error("Username/Password is incorrect")
+    st.error('Username/Password is incorrect')
     try:
-        username_forgot_pw, email_forgot_password, random_password = (
-            authenticator.forgot_password("Forgot password")
-        )
+        username_forgot_pw, email_forgot_password, random_password = authenticator.forgot_password('Forgot password')
         if username_forgot_pw:
-            st.success("New password sent securely")
+            st.success('New password sent securely')
             # Random password to be transferred to the user securely
         elif username_forgot_pw == False:
-            st.error("Username not found")
+            st.error('Username not found')
     except Exception as e:
         st.error(e)

pages/5_Model_Result_Overview.py

@@ -0,0 +1,103 @@
+import streamlit as st
+from utilities import (set_header,
+                       initialize_data,
+                       load_local_css,
+                       create_channel_summary,
+                       create_contribution_pie,
+                       create_contribuion_stacked_plot,
+                       create_channel_spends_sales_plot,
+                       format_numbers,
+                       channel_name_formating,
+                       load_authenticator)
+import plotly.graph_objects as go
+import streamlit_authenticator as stauth
+import yaml
+from yaml import SafeLoader
+import time
+
+st.set_page_config(layout='wide')
+load_local_css('styles.css')
+set_header()
+
+target='Revenue'
+# for k, v in st.session_state.items():
+
+#     if k not in ['logout', 'login','config'] and not k.startswith('FormSubmitter'):
+#         st.session_state[k] = v
+
+# authenticator = st.session_state.get('authenticator')
+
+# if authenticator is None:
+#     authenticator = load_authenticator()
+    
+# name, authentication_status, username = authenticator.login('Login', 'main')
+# auth_status = st.session_state['authentication_status']
+
+# if auth_status:
+#     authenticator.logout('Logout', 'main')
+    
+#     is_state_initiaized = st.session_state.get('initialized',False)
+#     if not is_state_initiaized:
+initialize_data()
+scenario = st.session_state['scenario']
+raw_df = st.session_state['raw_df']
+st.header('Overview of previous spends')
+
+
+columns = st.columns((1,1,3))
+
+with columns[0]:
+    st.metric(label = 'Spends', value=format_numbers(float(scenario.actual_total_spends)))
+###print(f"##################### {scenario.actual_total_sales} ##################")
+with columns[1]:
+    st.metric(label = target, value=format_numbers(float(scenario.actual_total_sales),include_indicator=False))
+
+
+actual_summary_df = create_channel_summary(scenario)
+actual_summary_df['Channel'] = actual_summary_df['Channel'].apply(channel_name_formating) 
+
+columns = st.columns((2,1))
+with columns[0]:
+    with st.expander('Channel wise overview'):
+        st.markdown(actual_summary_df.style.set_table_styles(
+        [{
+            'selector': 'th',
+            'props': [('background-color', '#11B6BD')]
+        },
+            {
+            'selector' : 'tr:nth-child(even)',
+            'props' : [('background-color', '#11B6BD')]
+            }]).to_html(), unsafe_allow_html=True)
+        
+st.markdown("<hr>",unsafe_allow_html=True)
+##############################
+
+st.plotly_chart(create_contribution_pie(),use_container_width=True)
+st.markdown("<hr>",unsafe_allow_html=True)
+
+
+################################3
+st.plotly_chart(create_contribuion_stacked_plot(scenario),use_container_width=True)
+st.markdown("<hr>",unsafe_allow_html=True)
+#######################################
+
+selected_channel_name = st.selectbox('Channel', st.session_state['channels_list'] + ['non media'], format_func=channel_name_formating)
+selected_channel = scenario.channels.get(selected_channel_name,None)
+
+st.plotly_chart(create_channel_spends_sales_plot(selected_channel), use_container_width=True)
+
+st.markdown("<hr>",unsafe_allow_html=True)
+
+# elif auth_status == False:
+#     st.error('Username/Password is incorrect')
+    
+# if auth_status != True:
+#     try:
+#         username_forgot_pw, email_forgot_password, random_password = authenticator.forgot_password('Forgot password')
+#         if username_forgot_pw:
+#             st.success('New password sent securely')
+#             # Random password to be transferred to user securely
+#         elif username_forgot_pw == False:
+#             st.error('Username not found')
+#     except Exception as e:
+#         st.error(e)

pages/6_Build_Response_Curves.py

@@ -0,0 +1,168 @@
+import streamlit as st
+import plotly.express as px
+import numpy as np
+import plotly.graph_objects as go
+from utilities import channel_name_formating, load_authenticator, initialize_data
+from sklearn.metrics import r2_score
+from collections import OrderedDict
+from classes import class_from_dict,class_to_dict
+import pickle
+import json
+
+for k, v in st.session_state.items():
+    if k not in ['logout', 'login','config'] and not k.startswith('FormSubmitter'):
+        st.session_state[k] = v
+
+def s_curve(x,K,b,a,x0):
+    return K / (1 + b*np.exp(-a*(x-x0)))
+
+def save_scenario(scenario_name):
+    """
+    Save the current scenario with the mentioned name in the session state
+    
+    Parameters
+    ----------
+    scenario_name
+        Name of the scenario to be saved
+    """
+    if 'saved_scenarios' not in st.session_state:
+        st.session_state = OrderedDict()
+        
+    #st.session_state['saved_scenarios'][scenario_name] = st.session_state['scenario'].save()
+    st.session_state['saved_scenarios'][scenario_name] = class_to_dict(st.session_state['scenario'])
+    st.session_state['scenario_input'] = ""
+    print(type(st.session_state['saved_scenarios']))
+    with open('../saved_scenarios.pkl', 'wb') as f:
+        pickle.dump(st.session_state['saved_scenarios'],f)
+
+
+def reset_curve_parameters():
+    del st.session_state['K']
+    del st.session_state['b']
+    del st.session_state['a']
+    del st.session_state['x0']
+    
+def update_response_curve():
+    # st.session_state['rcs'][selected_channel_name]['K'] = st.session_state['K']
+    # st.session_state['rcs'][selected_channel_name]['b'] = st.session_state['b']
+    # st.session_state['rcs'][selected_channel_name]['a'] = st.session_state['a']
+    # st.session_state['rcs'][selected_channel_name]['x0'] = st.session_state['x0']
+    # rcs = st.session_state['rcs']
+    _channel_class = st.session_state['scenario'].channels[selected_channel_name]
+    _channel_class.update_response_curves({
+                           'K'  : st.session_state['K'], 
+                           'b'  : st.session_state['b'], 
+                           'a'  : st.session_state['a'],
+                           'x0' : st.session_state['x0']})
+    
+
+# authenticator = st.session_state.get('authenticator')
+# if authenticator is None:
+#     authenticator = load_authenticator()
+
+# name, authentication_status, username = authenticator.login('Login', 'main')
+# auth_status = st.session_state.get('authentication_status')
+
+# if auth_status == True:
+#     is_state_initiaized = st.session_state.get('initialized',False)
+#     if not is_state_initiaized:
+#         print("Scenario page state reloaded")
+    
+initialize_data()
+
+st.subheader("Build response curves")
+    
+channels_list = st.session_state['channels_list']
+selected_channel_name = st.selectbox('Channel', st.session_state['channels_list'] + ['Others'], format_func=channel_name_formating,on_change=reset_curve_parameters)
+
+rcs = {}
+for channel_name in channels_list:
+    rcs[channel_name] = st.session_state['scenario'].channels[channel_name].response_curve_params
+# rcs = st.session_state['rcs']
+
+
+if 'K' not in st.session_state:
+    st.session_state['K'] = rcs[selected_channel_name]['K']
+if 'b' not in st.session_state:
+    st.session_state['b'] = rcs[selected_channel_name]['b']
+if 'a' not in st.session_state:
+    st.session_state['a'] = rcs[selected_channel_name]['a']
+if 'x0' not in st.session_state:
+    st.session_state['x0'] = rcs[selected_channel_name]['x0']
+    
+x = st.session_state['actual_input_df'][selected_channel_name].values
+y = st.session_state['actual_contribution_df'][selected_channel_name].values
+
+power = (np.ceil(np.log(x.max()) / np.log(10) )- 3)
+
+# fig = px.scatter(x, s_curve(x/10**power,
+#                             st.session_state['K'],
+#                             st.session_state['b'],
+#                             st.session_state['a'],
+#                             st.session_state['x0']))
+
+fig = px.scatter(x=x, y=y)
+fig.add_trace(go.Scatter(x=sorted(x), y=s_curve(sorted(x)/10**power,st.session_state['K'],
+                                    st.session_state['b'],
+                                    st.session_state['a'],
+                                    st.session_state['x0']),
+                        line=dict(color='red')))
+
+fig.update_layout(title_text="Response Curve",showlegend=False)
+fig.update_annotations(font_size=10)
+fig.update_xaxes(title='Spends')
+fig.update_yaxes(title='Revenue')
+
+st.plotly_chart(fig,use_container_width=True)
+
+r2 = r2_score(y, s_curve(x / 10**power, 
+                        st.session_state['K'],
+                        st.session_state['b'],
+                        st.session_state['a'],
+                        st.session_state['x0']))
+
+st.metric('R2',round(r2,2))
+columns = st.columns(4)
+
+with columns[0]:
+    st.number_input('K',key='K',format="%0.5f")
+with columns[1]:
+    st.number_input('b',key='b',format="%0.5f")
+with columns[2]:
+    st.number_input('a',key='a',step=0.0001,format="%0.5f")
+with columns[3]:
+    st.number_input('x0',key='x0',format="%0.5f")
+    
+
+st.button('Update parameters',on_click=update_response_curve)
+st.button('Reset parameters',on_click=reset_curve_parameters)
+scenario_name = st.text_input('Scenario name', key='scenario_input',placeholder='Scenario name',label_visibility='collapsed')
+st.button('Save', on_click=lambda  : save_scenario(scenario_name),disabled=len(st.session_state['scenario_input']) == 0)
+
+file_name = st.text_input('rcs download file name', key='file_name_input',placeholder='file name',label_visibility='collapsed')
+st.download_button(
+                    label="Download response curves",
+                    data=json.dumps(rcs),
+                    file_name=f"{file_name}.json",
+                    mime="application/json",
+                    disabled= len(file_name) == 0,
+                )
+
+
+def s_curve_derivative(x, K, b, a, x0):
+    # Derivative of the S-curve function
+    return a * b * K * np.exp(-a * (x - x0)) / ((1 + b * np.exp(-a * (x - x0))) ** 2)
+
+# Parameters of the S-curve
+K = st.session_state['K']
+b = st.session_state['b']
+a = st.session_state['a']
+x0 = st.session_state['x0']
+
+# Optimized spend value obtained from the tool
+optimized_spend = st.number_input('value of x')  # Replace this with your optimized spend value
+
+# Calculate the slope at the optimized spend value
+slope_at_optimized_spend = s_curve_derivative(optimized_spend, K, b, a, x0)
+
+st.write("Slope ", slope_at_optimized_spend)

pages/8_Scenario_Planner.py

@@ -23,38 +23,43 @@ import re
 import pandas as pd
 import plotly.express as px
 
-
+target = "Revenue"
 st.set_page_config(layout="wide")
 load_local_css("styles.css")
 set_header()
 
 for k, v in st.session_state.items():
-    if k not in ["logout", "login", "config"] and not k.startswith("FormSubmitter"):
+    if k not in ["logout", "login", "config"] and not k.startswith(
+        "FormSubmitter"
+    ):
         st.session_state[k] = v
 # ======================================================== #
 # ======================= Functions ====================== #
 # ======================================================== #
 
 
-def optimize(key, status_placeholder):
+def optimize(key):
     """
     Optimize the spends for the sales
     """
 
     channel_list = [
-        key for key, value in st.session_state["optimization_channels"].items() if value
+        key
+        for key, value in st.session_state["optimization_channels"].items()
+        if value
     ]
-
+    # print('channel_list')
+    # print(channel_list)
+    # print('@@@@@@@@')
     if len(channel_list) > 0:
         scenario = st.session_state["scenario"]
-        if key.lower() == "media spends":
+        if key.lower() == "spends":
             with status_placeholder:
                 with st.spinner("Optimizing"):
                     result = st.session_state["scenario"].optimize(
                         st.session_state["total_spends_change"], channel_list
                     )
-        # elif key.lower() == "revenue":
-        else:
+        elif key.lower() == "sales":
             with status_placeholder:
                 with st.spinner("Optimizing"):
 
@@ -64,11 +69,14 @@ def optimize(key, status_placeholder):
         for channel_name, modified_spends in result:
 
             st.session_state[channel_name] = numerize(
-                modified_spends * scenario.channels[channel_name].conversion_rate,
+                modified_spends
+                * scenario.channels[channel_name].conversion_rate,
                 1,
             )
             prev_spends = (
-                st.session_state["scenario"].channels[channel_name].actual_total_spends
+                st.session_state["scenario"]
+                .channels[channel_name]
+                .actual_total_spends
             )
             st.session_state[f"{channel_name}_change"] = round(
                 100 * (modified_spends - prev_spends) / prev_spends, 2
@@ -97,46 +105,15 @@ def save_scenario(scenario_name):
         pickle.dump(st.session_state["saved_scenarios"], f)
 
 
-if "allow_spends_update" not in st.session_state:
-    st.session_state["allow_spends_update"] = True
-
-if "allow_sales_update" not in st.session_state:
-    st.session_state["allow_sales_update"] = True
-
-
-def update_sales_abs_slider():
-    actual_sales = _scenario.actual_total_sales
-    if validate_input(st.session_state["total_sales_change_abs_slider"]):
-        modified_sales = extract_number_for_string(
-            st.session_state["total_sales_change_abs_slider"]
-        )
-        st.session_state["total_sales_change"] = round(
-            ((modified_sales / actual_sales) - 1) * 100
-        )
-        st.session_state["total_sales_change_abs"] = numerize(modified_sales, 1)
-
-
 def update_sales_abs():
-    if (
-        st.session_state["total_sales_change_abs"]
-        in st.session_state["total_sales_change_abs_slider_options"]
-    ):
-        st.session_state["allow_sales_update"] = True
-    else:
-        st.session_state["allow_sales_update"] = False
-
     actual_sales = _scenario.actual_total_sales
-    if (
-        validate_input(st.session_state["total_sales_change_abs"])
-        and st.session_state["allow_sales_update"]
-    ):
+    if validate_input(st.session_state["total_sales_change_abs"]):
         modified_sales = extract_number_for_string(
             st.session_state["total_sales_change_abs"]
         )
         st.session_state["total_sales_change"] = round(
             ((modified_sales / actual_sales) - 1) * 100
         )
-        st.session_state["total_sales_change_abs_slider"] = numerize(modified_sales, 1)
 
 
 def update_sales():
@@ -145,95 +122,32 @@ def update_sales():
         * _scenario.actual_total_sales,
         1,
     )
-    st.session_state["total_sales_change_abs_slider"] = numerize(
-        (1 + st.session_state["total_sales_change"] / 100)
-        * _scenario.actual_total_sales,
-        1,
-    )
-
-
-def update_all_spends_abs_slider():
-    actual_spends = _scenario.actual_total_spends
-    if validate_input(st.session_state["total_spends_change_abs_slider"]):
-        modified_spends = extract_number_for_string(
-            st.session_state["total_spends_change_abs_slider"]
-        )
-        st.session_state["total_spends_change"] = round(
-            ((modified_spends / actual_spends) - 1) * 100
-        )
-        st.session_state["total_spends_change_abs"] = numerize(modified_spends, 1)
-
-        update_all_spends()
-
-
-# def update_all_spends_abs_slider():
-#     actual_spends = _scenario.actual_total_spends
-#     if validate_input(st.session_state["total_spends_change_abs_slider"]):
-#         print("#" * 100)
-#         print(st.session_state["total_spends_change_abs_slider"])
-#         print("#" * 100)
-
-#         modified_spends = extract_number_for_string(
-#             st.session_state["total_spends_change_abs_slider"]
-#         )
-#         st.session_state["total_spends_change"] = (
-#             (modified_spends / actual_spends) - 1
-#         ) * 100
-#         st.session_state["total_spends_change_abs"] = st.session_state[
-#             "total_spends_change_abs_slider"
-#         ]
-
-#         update_all_spends()
 
 
 def update_all_spends_abs():
-    if (
-        st.session_state["total_spends_change_abs"]
-        in st.session_state["total_spends_change_abs_slider_options"]
-    ):
-        st.session_state["allow_spends_update"] = True
-    else:
-        st.session_state["allow_spends_update"] = False
-
     actual_spends = _scenario.actual_total_spends
-    if (
-        validate_input(st.session_state["total_spends_change_abs"])
-        and st.session_state["allow_spends_update"]
-    ):
+    if validate_input(st.session_state["total_spends_change_abs"]):
         modified_spends = extract_number_for_string(
             st.session_state["total_spends_change_abs"]
         )
+        print(modified_spends)
+        print(actual_spends)
+
         st.session_state["total_spends_change"] = (
             (modified_spends / actual_spends) - 1
         ) * 100
-        st.session_state["total_spends_change_abs_slider"] = st.session_state[
-            "total_spends_change_abs"
-        ]
 
         update_all_spends()
 
 
-def update_spends():
-    st.session_state["total_spends_change_abs"] = numerize(
-        (1 + st.session_state["total_spends_change"] / 100)
-        * _scenario.actual_total_spends,
-        1,
-    )
-    st.session_state["total_spends_change_abs_slider"] = numerize(
-        (1 + st.session_state["total_spends_change"] / 100)
-        * _scenario.actual_total_spends,
-        1,
-    )
-
-    update_all_spends()
-
-
 def update_all_spends():
     """
     Updates spends for all the channels with the given overall spends change
     """
     percent_change = st.session_state["total_spends_change"]
-
+    st.session_state["total_spends_change_abs"] = numerize(
+        (1 + percent_change / 100) * _scenario.actual_total_spends, 1
+    )
     for channel_name in st.session_state["channels_list"]:
         channel = st.session_state["scenario"].channels[channel_name]
         current_spends = channel.actual_total_spends
@@ -285,10 +199,16 @@ def update_data(channel_name):
     """
 
     if validate_input(st.session_state[channel_name]):
-        modified_spends = extract_number_for_string(st.session_state[channel_name])
+        modified_spends = extract_number_for_string(
+            st.session_state[channel_name]
+        )
         prev_spends = (
-            st.session_state["scenario"].channels[channel_name].actual_total_spends
-            * st.session_state["scenario"].channels[channel_name].conversion_rate
+            st.session_state["scenario"]
+            .channels[channel_name]
+            .actual_total_spends
+            * st.session_state["scenario"]
+            .channels[channel_name]
+            .conversion_rate
         )
         st.session_state[f"{channel_name}_change"] = round(
             100 * (modified_spends - prev_spends) / prev_spends, 2
@@ -296,7 +216,9 @@ def update_data(channel_name):
         st.session_state["scenario"].update(
             channel_name,
             modified_spends
-            / st.session_state["scenario"].channels[channel_name].conversion_rate,
+            / st.session_state["scenario"]
+            .channels[channel_name]
+            .conversion_rate,
         )
     # st.session_state['scenario'].update(channel_name, modified_spends)
     # else:
@@ -327,55 +249,31 @@ def select_all_channels_for_optimization():
         st.session_state[f"{channel_name}_selected"] = st.session_state[
             "optimze_all_channels"
         ]
-        st.session_state["optimization_channels"][channel_name] = st.session_state[
-            "optimze_all_channels"
-        ]
+        st.session_state["optimization_channels"][channel_name] = (
+            st.session_state["optimze_all_channels"]
+        )
 
 
 def update_penalty():
     """
     Updates the penalty flag for sales calculation
     """
-    st.session_state["scenario"].update_penalty(st.session_state["apply_penalty"])
+    st.session_state["scenario"].update_penalty(
+        st.session_state["apply_penalty"]
+    )
 
 
-def reset_scenario(panel_selected, file_selected, updated_rcs):
+def reset_scenario():
     # #print(st.session_state['default_scenario_dict'])
     # st.session_state['scenario']  = class_from_dict(st.session_state['default_scenario_dict'])
     # for channel in st.session_state['scenario'].channels.values():
     #     st.session_state[channel.name] = float(channel.actual_total_spends * channel.conversion_rate)
-    # initialize_data()
-
-    if panel_selected == "Aggregated":
-        initialize_data(
-            panel=panel_selected,
-            target_file=file_selected,
-            updated_rcs=updated_rcs,
-            metrics=metrics_selected,
-        )
-        panel = None
-    else:
-        initialize_data(
-            panel=panel_selected,
-            target_file=file_selected,
-            updated_rcs=updated_rcs,
-            metrics=metrics_selected,
-        )
-
+    initialize_data()
     for channel_name in st.session_state["channels_list"]:
         st.session_state[f"{channel_name}_selected"] = False
         st.session_state[f"{channel_name}_change"] = 0
     st.session_state["optimze_all_channels"] = False
 
-    st.session_state["total_sales_change"] = 0
-
-    update_spends()
-    update_sales()
-
-    reset_inputs()
-
-    # st.rerun()
-
 
 def format_number(num):
     if num >= 1_000_000:
@@ -407,7 +305,9 @@ def summary_plot(data, x, y, title, text_column):
         hovertemplate="%{x:.2s}",
     )
 
-    fig.update_layout(xaxis_title=x, yaxis_title="Channel Name", showlegend=False)
+    fig.update_layout(
+        xaxis_title=x, yaxis_title="Channel Name", showlegend=False
+    )
     return fig
 
 
@@ -442,21 +342,27 @@ def calculate_rgba(
         relative_position = (current_channel_spends - start_value) / (
             left_value - start_value
         )
-        alpha = 0.8 - (0.6 * relative_position)  # Alpha decreases from start to end
+        alpha = 0.8 - (
+            0.6 * relative_position
+        )  # Alpha decreases from start to end
 
     elif left_value < current_channel_spends <= right_value:
         color = "green"
         relative_position = (current_channel_spends - left_value) / (
             right_value - left_value
         )
-        alpha = 0.8 - (0.6 * relative_position)  # Alpha decreases from start to end
+        alpha = 0.8 - (
+            0.6 * relative_position
+        )  # Alpha decreases from start to end
 
     elif right_value < current_channel_spends <= end_value:
         color = "red"
         relative_position = (current_channel_spends - right_value) / (
             end_value - right_value
         )
-        alpha = 0.2 + (0.6 * relative_position)  # Alpha increases from start to end
+        alpha = 0.2 + (
+            0.6 * relative_position
+        )  # Alpha increases from start to end
 
     else:
         # Default case, if the spends are outside the defined ranges
@@ -526,7 +432,9 @@ def plot_response_curves():
 
         for index in range(len(x_plot)):
             marginal_roi.append(
-                a * y[index] * (1 - y[index] / np.maximum(K, np.finfo(float).eps))
+                a
+                * y[index]
+                * (1 - y[index] / np.maximum(K, np.finfo(float).eps))
             )
 
         x = (
@@ -558,7 +466,9 @@ def plot_response_curves():
             st.session_state["scenario"].channels[col].modified_total_spends
             * st.session_state["scenario"].channels[col].conversion_rate
         )
-        y_optimal = st.session_state["scenario"].channels[col].modified_total_sales
+        y_optimal = (
+            st.session_state["scenario"].channels[col].modified_total_sales
+        )
 
         # if col == "Paid_social_others":
         #     debug_temp(x_optimal * x_actual / x_actual.sum(), power, K, b, a, x0)
@@ -666,7 +576,7 @@ def plot_response_curves():
     fig.update_layout(
         # height=1000,
         # width=1000,
-        title_text=f"Response Curves (X: Spends Vs Y: {target})",
+        title_text="Response Curves (X: Spends Vs Y: Revenue)",
         showlegend=False,
         shapes=shapes,
     )
@@ -808,144 +718,12 @@ authenticator = stauth.Authenticate(
 st.session_state["authenticator"] = authenticator
 name, authentication_status, username = authenticator.login("Login", "main")
 auth_status = st.session_state.get("authentication_status")
-
-import os
-import glob
-
-
-def get_excel_names(directory):
-    # Create a list to hold the final parts of the filenames
-    last_portions = []
-
-    # Patterns to match Excel files (.xlsx and .xls) that contain @#
-    patterns = [
-        os.path.join(directory, "*@#*.xlsx"),
-        os.path.join(directory, "*@#*.xls"),
-    ]
-
-    # Process each pattern
-    for pattern in patterns:
-        files = glob.glob(pattern)
-
-        # Extracting the last portion after @# for each file
-        for file in files:
-            base_name = os.path.basename(file)
-            last_portion = base_name.split("@#")[-1]
-            last_portion = last_portion.replace(".xlsx", "").replace(
-                ".xls", ""
-            )  # Removing extensions
-            last_portions.append(last_portion)
-
-    return last_portions
-
-
-def name_formating(channel_name):
-    # Replace underscores with spaces
-    name_mod = channel_name.replace("_", " ")
-
-    # Capitalize the first letter of each word
-    name_mod = name_mod.title()
-
-    return name_mod
-
-
-@st.cache_resource(show_spinner=False)
-def panel_fetch(file_selected):
-    raw_data_mmm_df = pd.read_excel(file_selected, sheet_name="RAW DATA MMM")
-
-    if "Panel" in raw_data_mmm_df.columns:
-        panel = list(set(raw_data_mmm_df["Panel"]))
-    else:
-        raw_data_mmm_df = None
-        panel = None
-
-    return panel
-
-
-def reset_inputs():
-    if "total_spends_change_abs" in st.session_state:
-        del st.session_state.total_spends_change_abs
-    if "total_spends_change" in st.session_state:
-        del st.session_state.total_spends_change
-    if "total_spends_change_abs_slider" in st.session_state:
-        del st.session_state.total_spends_change_abs_slider
-
-    if "total_sales_change_abs" in st.session_state:
-        del st.session_state.total_sales_change_abs
-    if "total_sales_change" in st.session_state:
-        del st.session_state.total_sales_change
-    if "total_sales_change_abs_slider" in st.session_state:
-        del st.session_state.total_sales_change_abs_slider
-
-    st.session_state["initialized"] = False
-
-
 if auth_status == True:
     authenticator.logout("Logout", "main")
-
-    st.header("Simulation")
-    col1, col2 = st.columns([1, 1])
-
-    # Response Metrics
-    directory = "metrics_level_data"
-    metrics_list = get_excel_names(directory)
-    metrics_selected = col1.selectbox(
-        "Response Metrics",
-        metrics_list,
-        format_func=name_formating,
-        index=0,
-        on_change=reset_inputs,
-    )
-
-    # Target
-    target = name_formating(metrics_selected)
-
-    file_selected = (
-        f".\metrics_level_data\Overview_data_test_panel@#{metrics_selected}.xlsx"
-    )
-
-    # Panel List
-    panel_list = panel_fetch(file_selected)
-
-    # Panel Selected
-    panel_selected = col2.selectbox(
-        "Panel",
-        ["Aggregated"] + panel_list,
-        index=0,
-        on_change=reset_inputs,
-    )
-
-    if "update_rcs" in st.session_state:
-        updated_rcs = st.session_state["update_rcs"]
-    else:
-        updated_rcs = None
-
-    if "first_time" not in st.session_state:
-        st.session_state["first_time"] = True
-
-    # Check if state is initiaized
     is_state_initiaized = st.session_state.get("initialized", False)
-    if not is_state_initiaized or st.session_state["first_time"]:
-        # initialize_data()
-        if panel_selected == "Aggregated":
-            initialize_data(
-                panel=panel_selected,
-                target_file=file_selected,
-                updated_rcs=updated_rcs,
-                metrics=metrics_selected,
-            )
-            panel = None
-        else:
-            initialize_data(
-                panel=panel_selected,
-                target_file=file_selected,
-                updated_rcs=updated_rcs,
-                metrics=metrics_selected,
-            )
-        st.session_state["initialized"] = True
-        st.session_state["first_time"] = False
+    if not is_state_initiaized:
+        initialize_data()
 
-    # Channels List
     channels_list = st.session_state["channels_list"]
 
     # ======================================================== #
@@ -953,16 +731,12 @@ if auth_status == True:
     # ======================================================== #
 
     # print(list(st.session_state.keys()))
+
+    st.header("Simulation")
     main_header = st.columns((2, 2))
     sub_header = st.columns((1, 1, 1, 1))
     _scenario = st.session_state["scenario"]
 
-    if "total_spends_change" not in st.session_state:
-        st.session_state.total_spends_change = 0
-
-    if "total_sales_change" not in st.session_state:
-        st.session_state.total_sales_change = 0
-
     if "total_spends_change_abs" not in st.session_state:
         st.session_state["total_spends_change_abs"] = numerize(
             _scenario.actual_total_spends, 1
@@ -973,16 +747,6 @@ if auth_status == True:
             _scenario.actual_total_sales, 1
         )
 
-    if "total_spends_change_abs_slider" not in st.session_state:
-        st.session_state.total_spends_change_abs_slider = numerize(
-            _scenario.actual_total_spends, 1
-        )
-
-    if "total_sales_change_abs_slider" not in st.session_state:
-        st.session_state.total_sales_change_abs_slider = numerize(
-            _scenario.actual_total_sales, 1
-        )
-
     with main_header[0]:
         st.subheader("Actual")
 
@@ -990,7 +754,9 @@ if auth_status == True:
         st.subheader("Simulated")
 
     with sub_header[0]:
-        st.metric(label="Spends", value=format_numbers(_scenario.actual_total_spends))
+        st.metric(
+            label="Spends", value=format_numbers(_scenario.actual_total_spends)
+        )
 
     with sub_header[1]:
         st.metric(
@@ -1016,52 +782,36 @@ if auth_status == True:
             delta=numerize(_scenario.delta_sales, 1),
         )
 
-    with st.expander("Channel Spends Simulator", expanded=True):
+    with st.expander("Channel Spends Simulator"):
         _columns1 = st.columns((2, 2, 1, 1))
         with _columns1[0]:
+
             optimization_selection = st.selectbox(
-                "Optimize", options=["Media Spends", target], key="optimization_key"
+                "Optimize", options=["Spends", "Sales"], key="optimization_key"
             )
-
         with _columns1[1]:
             st.markdown("#")
-            # if st.checkbox(
-            #     label="Optimize all Channels",
-            #     key="optimze_all_channels",
-            #     value=False,
-            #     # on_change=select_all_channels_for_optimization,
-            # ):
-            #     select_all_channels_for_optimization()
-
             st.checkbox(
                 label="Optimize all Channels",
-                key="optimze_all_channels",
+                key=f"optimze_all_channels",
                 value=False,
                 on_change=select_all_channels_for_optimization,
             )
 
         with _columns1[2]:
             st.markdown("#")
-            # st.button(
-            #     "Optimize",
-            #     on_click=optimize,
-            #     args=(st.session_state["optimization_key"]),
-            #     use_container_width=True,
-            # )
-
-            optimize_placeholder = st.empty()
+            st.button(
+                "Optimize",
+                on_click=optimize,
+                args=(st.session_state["optimization_key"],),
+            )
 
         with _columns1[3]:
             st.markdown("#")
-            st.button(
-                "Reset",
-                on_click=reset_scenario,
-                args=(panel_selected, file_selected, updated_rcs),
-                use_container_width=True,
-            )
+            st.button("Reset", on_click=reset_scenario)
 
         _columns2 = st.columns((2, 2, 2))
-        if st.session_state["optimization_key"] == "Media Spends":
+        if st.session_state["optimization_key"] == "Spends":
             with _columns2[0]:
                 spend_input = st.text_input(
                     "Absolute",
@@ -1069,90 +819,37 @@ if auth_status == True:
                     # label_visibility="collapsed",
                     on_change=update_all_spends_abs,
                 )
-
             with _columns2[1]:
+
                 st.number_input(
-                    "Percent Change",
-                    key="total_spends_change",
-                    min_value=-50,
-                    max_value=50,
+                    "Percent",
+                    key=f"total_spends_change",
                     step=1,
-                    on_change=update_spends,
+                    on_change=update_all_spends,
                 )
-
-            with _columns2[2]:
-                min_value = round(_scenario.actual_total_spends * 0.5)
-                max_value = round(_scenario.actual_total_spends * 1.5)
-                st.session_state["total_spends_change_abs_slider_options"] = [
-                    numerize(value, 1)
-                    for value in range(min_value, max_value + 1, int(1e4))
-                ]
-
-                st.select_slider(
-                    "Absolute Slider",
-                    options=st.session_state["total_spends_change_abs_slider_options"],
-                    key="total_spends_change_abs_slider",
-                    on_change=update_all_spends_abs_slider,
-                )
-
-        elif st.session_state["optimization_key"] == target:
+        elif st.session_state["optimization_key"] == "Sales":
             with _columns2[0]:
+
                 sales_input = st.text_input(
                     "Absolute",
                     key="total_sales_change_abs",
                     on_change=update_sales_abs,
                 )
-
             with _columns2[1]:
                 st.number_input(
-                    "Percent Change",
-                    key="total_sales_change",
-                    min_value=-50,
-                    max_value=50,
+                    "Percent change",
+                    key=f"total_sales_change",
                     step=1,
                     on_change=update_sales,
                 )
-            with _columns2[2]:
-                min_value = round(_scenario.actual_total_sales * 0.5)
-                max_value = round(_scenario.actual_total_sales * 1.5)
-                st.session_state["total_sales_change_abs_slider_options"] = [
-                    numerize(value, 1)
-                    for value in range(min_value, max_value + 1, int(1e5))
-                ]
-
-                st.select_slider(
-                    "Absolute Slider",
-                    options=st.session_state["total_sales_change_abs_slider_options"],
-                    key="total_sales_change_abs_slider",
-                    on_change=update_sales_abs_slider,
-                )
 
-        if (
-            not st.session_state["allow_sales_update"]
-            and optimization_selection == target
-        ):
-            st.warning("Invalid Input")
-
-        if (
-            not st.session_state["allow_spends_update"]
-            and optimization_selection == "Media Spends"
-        ):
-            st.warning("Invalid Input")
-
-        status_placeholder = st.empty()
-
-        # if optimize_placeholder.button("Optimize", use_container_width=True):
-        #     optimize(st.session_state["optimization_key"], status_placeholder)
-        #     st.rerun()
-
-        optimize_placeholder.button(
-            "Optimize",
-            on_click=optimize,
-            args=(st.session_state["optimization_key"], status_placeholder),
-            use_container_width=True,
-        )
+        with _columns2[2]:
+            st.markdown("#")
+            status_placeholder = st.empty()
 
-        st.markdown("""<hr class="spends-heading-seperator">""", unsafe_allow_html=True)
+        st.markdown(
+            """<hr class="spends-heading-seperator">""", unsafe_allow_html=True
+        )
         _columns = st.columns((2.5, 2, 1.5, 1.5, 1))
         with _columns[0]:
             generate_spending_header("Channel")
@@ -1165,7 +862,9 @@ if auth_status == True:
         with _columns[4]:
             generate_spending_header("Optimize")
 
-        st.markdown("""<hr class="spends-heading-seperator">""", unsafe_allow_html=True)
+        st.markdown(
+            """<hr class="spends-heading-seperator">""", unsafe_allow_html=True
+        )
 
         if "acutual_predicted" not in st.session_state:
             st.session_state["acutual_predicted"] = {
@@ -1175,7 +874,9 @@ if auth_status == True:
                 "Delta": [],
             }
         for i, channel_name in enumerate(channels_list):
-            _channel_class = st.session_state["scenario"].channels[channel_name]
+            _channel_class = st.session_state["scenario"].channels[
+                channel_name
+            ]
             _columns = st.columns((2.5, 1.5, 1.5, 1.5, 1))
             with _columns[0]:
                 st.write(channel_name_formating(channel_name))
@@ -1184,8 +885,12 @@ if auth_status == True:
             with _columns[1]:
                 channel_bounds = _channel_class.bounds
                 channel_spends = float(_channel_class.actual_total_spends)
-                min_value = float((1 + channel_bounds[0] / 100) * channel_spends)
-                max_value = float((1 + channel_bounds[1] / 100) * channel_spends)
+                min_value = float(
+                    (1 + channel_bounds[0] / 100) * channel_spends
+                )
+                max_value = float(
+                    (1 + channel_bounds[1] / 100) * channel_spends
+                )
                 ##print(st.session_state[channel_name])
                 spend_input = st.text_input(
                     channel_name,
@@ -1196,11 +901,9 @@ if auth_status == True:
                 if not validate_input(spend_input):
                     st.error("Invalid input")
 
-                channel_name_current = f"{channel_name}_change"
-
                 st.number_input(
-                    "Percent Change",
-                    key=channel_name_current,
+                    "Percent change",
+                    key=f"{channel_name}_change",
                     step=1,
                     on_change=partial(update_data_by_percent, channel_name),
                 )
@@ -1212,10 +915,12 @@ if auth_status == True:
                     * _channel_class.conversion_rate
                 )
                 actual_channel_spends = float(
-                    _channel_class.actual_total_spends * _channel_class.conversion_rate
+                    _channel_class.actual_total_spends
+                    * _channel_class.conversion_rate
                 )
                 spends_delta = float(
-                    _channel_class.delta_spends * _channel_class.conversion_rate
+                    _channel_class.delta_spends
+                    * _channel_class.conversion_rate
                 )
                 st.session_state["acutual_predicted"]["Channel_name"].append(
                     channel_name
@@ -1223,10 +928,12 @@ if auth_status == True:
                 st.session_state["acutual_predicted"]["Actual_spend"].append(
                     actual_channel_spends
                 )
-                st.session_state["acutual_predicted"]["Optimized_spend"].append(
-                    current_channel_spends
+                st.session_state["acutual_predicted"][
+                    "Optimized_spend"
+                ].append(current_channel_spends)
+                st.session_state["acutual_predicted"]["Delta"].append(
+                    spends_delta
                 )
-                st.session_state["acutual_predicted"]["Delta"].append(spends_delta)
                 ## REMOVE
                 st.metric(
                     "Spends",
@@ -1237,32 +944,29 @@ if auth_status == True:
 
             with _columns[3]:
                 # sales
-                current_channel_sales = float(_channel_class.modified_total_sales)
+                current_channel_sales = float(
+                    _channel_class.modified_total_sales
+                )
                 actual_channel_sales = float(_channel_class.actual_total_sales)
                 sales_delta = float(_channel_class.delta_sales)
                 st.metric(
                     target,
-                    format_numbers(current_channel_sales, include_indicator=False),
+                    format_numbers(
+                        current_channel_sales, include_indicator=False
+                    ),
                     delta=numerize(sales_delta, 1),
                     label_visibility="collapsed",
                 )
 
             with _columns[4]:
 
-                # if st.checkbox(
-                #     label="select for optimization",
-                #     key=f"{channel_name}_selected",
-                #     value=False,
-                #     # on_change=partial(select_channel_for_optimization, channel_name),
-                #     label_visibility="collapsed",
-                # ):
-                #     select_channel_for_optimization(channel_name)
-
                 st.checkbox(
                     label="select for optimization",
                     key=f"{channel_name}_selected",
                     value=False,
-                    on_change=partial(select_channel_for_optimization, channel_name),
+                    on_change=partial(
+                        select_channel_for_optimization, channel_name
+                    ),
                     label_visibility="collapsed",
                 )
 
@@ -1274,29 +978,20 @@ if auth_status == True:
             # Bins
             col = channels_list[i]
             x_actual = st.session_state["scenario"].channels[col].actual_spends
-            x_modified = st.session_state["scenario"].channels[col].modified_spends
+            x_modified = (
+                st.session_state["scenario"].channels[col].modified_spends
+            )
 
             x_total = x_modified.sum()
             power = np.ceil(np.log(x_actual.max()) / np.log(10)) - 3
 
-            updated_rcs_key = f"{metrics_selected}#@{panel_selected}#@{channel_name}"
-
-            if updated_rcs and updated_rcs_key in list(updated_rcs.keys()):
-                K = updated_rcs[updated_rcs_key]["K"]
-                b = updated_rcs[updated_rcs_key]["b"]
-                a = updated_rcs[updated_rcs_key]["a"]
-                x0 = updated_rcs[updated_rcs_key]["x0"]
-            else:
-                K = st.session_state["rcs"][col]["K"]
-                b = st.session_state["rcs"][col]["b"]
-                a = st.session_state["rcs"][col]["a"]
-                x0 = st.session_state["rcs"][col]["x0"]
+            K = st.session_state["rcs"][col]["K"]
+            b = st.session_state["rcs"][col]["b"]
+            a = st.session_state["rcs"][col]["a"]
+            x0 = st.session_state["rcs"][col]["x0"]
 
             x_plot = np.linspace(0, 5 * x_actual.sum(), 200)
 
-            # Append current_channel_spends to the end of x_plot
-            x_plot = np.append(x_plot, current_channel_spends)
-
             x, y, marginal_roi = [], [], []
             for x_p in x_plot:
                 x.append(x_p * x_actual / x_actual.sum())
@@ -1306,7 +1001,9 @@ if auth_status == True:
 
             for index in range(len(x_plot)):
                 marginal_roi.append(
-                    a * y[index] * (1 - y[index] / np.maximum(K, np.finfo(float).eps))
+                    a
+                    * y[index]
+                    * (1 - y[index] / np.maximum(K, np.finfo(float).eps))
                 )
 
             x = (
@@ -1321,18 +1018,12 @@ if auth_status == True:
 
             roi = y / np.maximum(x, np.finfo(float).eps)
 
-            roi_current, marginal_roi_current = roi[-1], marginal_roi[-1]
-            x, y, roi, marginal_roi = (
-                x[:-1],
-                y[:-1],
-                roi[:-1],
-                marginal_roi[:-1],
-            )  # Drop data for current spends
-
-            start_value, end_value, left_value, right_value = find_segment_value(
-                x,
-                roi,
-                marginal_roi,
+            start_value, end_value, left_value, right_value = (
+                find_segment_value(
+                    x,
+                    roi,
+                    marginal_roi,
+                )
             )
 
             rgba = calculate_rgba(
@@ -1343,6 +1034,16 @@ if auth_status == True:
                 current_channel_spends,
             )
 
+            # Protecting division by zero by adding a small epsilon to denominators
+            roi_current = current_channel_sales / np.maximum(
+                current_channel_spends, np.finfo(float).eps
+            )
+            marginal_roi_current = (
+                st.session_state["scenario"]
+                .channels[col]
+                .get_marginal_roi("modified")
+            )
+
             with bin_placeholder:
                 st.markdown(
                     f"""
@@ -1360,7 +1061,7 @@ if auth_status == True:
                     unsafe_allow_html=True,
                 )
 
-    with st.expander("See Response Curves", expanded=True):
+    with st.expander("See Response Curves"):
         fig = plot_response_curves()
         st.plotly_chart(fig, use_container_width=True)
 
@@ -1380,11 +1081,19 @@ if auth_status == True:
         )
 
     summary_df = pd.DataFrame(st.session_state["acutual_predicted"])
-    summary_df.drop_duplicates(subset="Channel_name", keep="last", inplace=True)
+    summary_df.drop_duplicates(
+        subset="Channel_name", keep="last", inplace=True
+    )
 
     summary_df_sorted = summary_df.sort_values(by="Delta", ascending=False)
     summary_df_sorted["Delta_percent"] = np.round(
-        ((summary_df_sorted["Optimized_spend"] / summary_df_sorted["Actual_spend"]) - 1)
+        (
+            (
+                summary_df_sorted["Optimized_spend"]
+                / summary_df_sorted["Actual_spend"]
+            )
+            - 1
+        )
         * 100,
         2,
     )
@@ -1412,9 +1121,9 @@ if auth_status != True:
             authenticator.forgot_password("Forgot password")
         )
         if username_forgot_pw:
-            st.session_state["config"]["credentials"]["usernames"][username_forgot_pw][
-                "password"
-            ] = stauth.Hasher([random_password]).generate()[0]
+            st.session_state["config"]["credentials"]["usernames"][
+                username_forgot_pw
+            ]["password"] = stauth.Hasher([random_password]).generate()[0]
             send_email(email_forgot_password, random_password)
             st.success("New password sent securely")
             # Random password to be transferred to user securely

pages/Data_Import.py

@@ -0,0 +1,891 @@
+# Importing necessary libraries
+import streamlit as st
+
+st.set_page_config(
+    page_title="Model Build",
+    page_icon=":shark:",
+    layout="wide",
+    initial_sidebar_state="collapsed",
+)
+
+import numpy as np
+import pandas as pd
+from utilities import set_header, load_local_css, load_authenticator
+import pickle
+
+
+load_local_css("styles.css")
+set_header()
+
+authenticator = st.session_state.get("authenticator")
+if authenticator is None:
+    authenticator = load_authenticator()
+
+name, authentication_status, username = authenticator.login("Login", "main")
+auth_status = st.session_state.get("authentication_status")
+
+# Check for authentication status
+if auth_status != True:
+    st.stop()
+
+
+# Function to validate date column in dataframe
+def validate_date_column(df):
+    try:
+        # Attempt to convert the 'Date' column to datetime
+        df["date"] = pd.to_datetime(df["date"], format="%d-%m-%Y")
+        return True
+    except:
+        return False
+
+
+# Function to determine data interval
+def determine_data_interval(common_freq):
+    if common_freq == 1:
+        return "daily"
+    elif common_freq == 7:
+        return "weekly"
+    elif 28 <= common_freq <= 31:
+        return "monthly"
+    else:
+        return "irregular"
+
+
+# Function to read each uploaded Excel file into a pandas DataFrame and stores them in a dictionary
+st.cache_resource(show_spinner=False)
+
+
+def files_to_dataframes(uploaded_files):
+    df_dict = {}
+    for uploaded_file in uploaded_files:
+        # Extract file name without extension
+        file_name = uploaded_file.name.rsplit(".", 1)[0]
+
+        # Check for duplicate file names
+        if file_name in df_dict:
+            st.warning(
+                f"Duplicate File: {file_name}. This file will be skipped.",
+                icon="⚠️",
+            )
+            continue
+
+        # Read the file into a DataFrame
+        df = pd.read_excel(uploaded_file)
+
+        # Convert all column names to lowercase
+        df.columns = df.columns.str.lower().str.strip()
+
+        # Separate numeric and non-numeric columns
+        numeric_cols = list(df.select_dtypes(include=["number"]).columns)
+        non_numeric_cols = [
+            col
+            for col in df.select_dtypes(exclude=["number"]).columns
+            if col.lower() != "date"
+        ]
+
+        # Check for 'Date' column
+        if not (validate_date_column(df) and len(numeric_cols) > 0):
+            st.warning(
+                f"File Name: {file_name} ➜ Please upload data with Date column in 'DD-MM-YYYY' format and at least one media/exogenous column. This file will be skipped.",
+                icon="⚠️",
+            )
+            continue
+
+        # Check for interval
+        common_freq = common_freq = (
+            pd.Series(df["date"].unique()).diff().dt.days.dropna().mode()[0]
+        )
+        # Calculate the data interval (daily, weekly, monthly or irregular)
+        interval = determine_data_interval(common_freq)
+        if interval == "irregular":
+            st.warning(
+                f"File Name: {file_name} ➜ Please upload data in daily, weekly or monthly interval. This file will be skipped.",
+                icon="⚠️",
+            )
+            continue
+
+        # Store both DataFrames in the dictionary under their respective keys
+        df_dict[file_name] = {
+            "numeric": numeric_cols,
+            "non_numeric": non_numeric_cols,
+            "interval": interval,
+            "df": df,
+        }
+
+    return df_dict
+
+
+# Function to adjust dataframe granularity
+# def adjust_dataframe_granularity(df, current_granularity, target_granularity):
+#     # Set index
+#     df.set_index("date", inplace=True)
+
+#     # Define aggregation rules for resampling
+#     aggregation_rules = {
+#         col: "sum" if pd.api.types.is_numeric_dtype(df[col]) else "first"
+#         for col in df.columns
+#     }
+
+#     resampled_df = df
+#     if current_granularity == "daily" and target_granularity == "weekly":
+#         resampled_df = df.resample("W-MON").agg(aggregation_rules)
+
+#     elif current_granularity == "daily" and target_granularity == "monthly":
+#         resampled_df = df.resample("MS").agg(aggregation_rules)
+
+#     elif current_granularity == "daily" and target_granularity == "daily":
+#         resampled_df = df.resample("D").agg(aggregation_rules)
+
+#     elif current_granularity in ["weekly", "monthly"] and target_granularity == "daily":
+#         # For higher to lower granularity, distribute numeric and replicate non-numeric values equally across the new period
+#         expanded_data = []
+#         for _, row in df.iterrows():
+#             if current_granularity == "weekly":
+#                 period_range = pd.date_range(start=row.name, periods=7)
+#             elif current_granularity == "monthly":
+#                 period_range = pd.date_range(
+#                     start=row.name, periods=row.name.days_in_month
+#                 )
+
+#             for date in period_range:
+#                 new_row = {}
+#                 for col in df.columns:
+#                     if pd.api.types.is_numeric_dtype(df[col]):
+#                         if current_granularity == "weekly":
+#                             new_row[col] = row[col] / 7
+#                         elif current_granularity == "monthly":
+#                             new_row[col] = row[col] / row.name.days_in_month
+#                     else:
+#                         new_row[col] = row[col]
+#                 expanded_data.append((date, new_row))
+
+#         resampled_df = pd.DataFrame(
+#             [data for _, data in expanded_data],
+#             index=[date for date, _ in expanded_data],
+#         )
+
+#     # Reset index
+#     resampled_df = resampled_df.reset_index().rename(columns={"index": "date"})
+
+#     return resampled_df
+
+
+def adjust_dataframe_granularity(df, current_granularity, target_granularity):
+    # Set index
+    df.set_index("date", inplace=True)
+
+    # Define aggregation rules for resampling
+    aggregation_rules = {
+        col: "sum" if pd.api.types.is_numeric_dtype(df[col]) else "first"
+        for col in df.columns
+    }
+
+    # Initialize resampled_df
+    resampled_df = df
+    if current_granularity == "daily" and target_granularity == "weekly":
+        resampled_df = df.resample("W-MON", closed="left", label="left").agg(
+            aggregation_rules
+        )
+
+    elif current_granularity == "daily" and target_granularity == "monthly":
+        resampled_df = df.resample("MS", closed="left", label="left").agg(
+            aggregation_rules
+        )
+
+    elif current_granularity == "daily" and target_granularity == "daily":
+        resampled_df = df.resample("D").agg(aggregation_rules)
+
+    elif current_granularity in ["weekly", "monthly"] and target_granularity == "daily":
+        # For higher to lower granularity, distribute numeric and replicate non-numeric values equally across the new period
+        expanded_data = []
+        for _, row in df.iterrows():
+            if current_granularity == "weekly":
+                period_range = pd.date_range(start=row.name, periods=7)
+            elif current_granularity == "monthly":
+                period_range = pd.date_range(
+                    start=row.name, periods=row.name.days_in_month
+                )
+
+            for date in period_range:
+                new_row = {}
+                for col in df.columns:
+                    if pd.api.types.is_numeric_dtype(df[col]):
+                        if current_granularity == "weekly":
+                            new_row[col] = row[col] / 7
+                        elif current_granularity == "monthly":
+                            new_row[col] = row[col] / row.name.days_in_month
+                    else:
+                        new_row[col] = row[col]
+                expanded_data.append((date, new_row))
+
+        resampled_df = pd.DataFrame(
+            [data for _, data in expanded_data],
+            index=[date for date, _ in expanded_data],
+        )
+
+    # Reset index
+    resampled_df = resampled_df.reset_index().rename(columns={"index": "date"})
+
+    return resampled_df
+
+
+# Function to clean and extract unique values of DMA and Panel
+st.cache_resource(show_spinner=False)
+
+
+def clean_and_extract_unique_values(files_dict, selections):
+    all_dma_values = set()
+    all_panel_values = set()
+
+    for file_name, file_data in files_dict.items():
+        df = file_data["df"]
+
+        # 'DMA' and 'Panel' selections
+        selected_dma = selections[file_name].get("DMA")
+        selected_panel = selections[file_name].get("Panel")
+
+        # Clean and standardize DMA column if it exists and is selected
+        if selected_dma and selected_dma != "N/A" and selected_dma in df.columns:
+            df[selected_dma] = (
+                df[selected_dma].str.lower().str.strip().str.replace("_", " ")
+            )
+            all_dma_values.update(df[selected_dma].dropna().unique())
+
+        # Clean and standardize Panel column if it exists and is selected
+        if selected_panel and selected_panel != "N/A" and selected_panel in df.columns:
+            df[selected_panel] = (
+                df[selected_panel].str.lower().str.strip().str.replace("_", " ")
+            )
+            all_panel_values.update(df[selected_panel].dropna().unique())
+
+        # Update the processed DataFrame back in the dictionary
+        files_dict[file_name]["df"] = df
+
+    return all_dma_values, all_panel_values
+
+
+# Function to format values for display
+st.cache_resource(show_spinner=False)
+
+
+def format_values_for_display(values_list):
+    # Capitalize the first letter of each word and replace underscores with spaces
+    formatted_list = [value.replace("_", " ").title() for value in values_list]
+    # Join values with commas and 'and' before the last value
+    if len(formatted_list) > 1:
+        return ", ".join(formatted_list[:-1]) + ", and " + formatted_list[-1]
+    elif formatted_list:
+        return formatted_list[0]
+    return "No values available"
+
+
+# Function to normalizes all data within files_dict to a daily granularity
+st.cache(show_spinner=False, allow_output_mutation=True)
+
+
+def standardize_data_to_daily(files_dict, selections):
+    # Normalize all data to a daily granularity using a provided function
+    files_dict = apply_granularity_to_all(files_dict, "daily", selections)
+
+    # Update the "interval" attribute for each dataset to indicate the new granularity
+    for files_name, files_data in files_dict.items():
+        files_data["interval"] = "daily"
+
+    return files_dict
+
+
+# Function to apply granularity transformation to all DataFrames in files_dict
+st.cache_resource(show_spinner=False)
+
+
+def apply_granularity_to_all(files_dict, granularity_selection, selections):
+    for file_name, file_data in files_dict.items():
+        df = file_data["df"].copy()
+
+        # Handling when DMA or Panel might be 'N/A'
+        selected_dma = selections[file_name].get("DMA")
+        selected_panel = selections[file_name].get("Panel")
+
+        # Correcting the segment selection logic & handling 'N/A'
+        if selected_dma != "N/A" and selected_panel != "N/A":
+            unique_combinations = df[[selected_dma, selected_panel]].drop_duplicates()
+        elif selected_dma != "N/A":
+            unique_combinations = df[[selected_dma]].drop_duplicates()
+            selected_panel = None  # Ensure Panel is ignored if N/A
+        elif selected_panel != "N/A":
+            unique_combinations = df[[selected_panel]].drop_duplicates()
+            selected_dma = None  # Ensure DMA is ignored if N/A
+        else:
+            # If both are 'N/A', process the entire dataframe as is
+            df = adjust_dataframe_granularity(
+                df, file_data["interval"], granularity_selection
+            )
+            files_dict[file_name]["df"] = df
+            continue  # Skip to the next file
+
+        transformed_segments = []
+        for _, combo in unique_combinations.iterrows():
+            if selected_dma and selected_panel:
+                segment = df[
+                    (df[selected_dma] == combo[selected_dma])
+                    & (df[selected_panel] == combo[selected_panel])
+                ]
+            elif selected_dma:
+                segment = df[df[selected_dma] == combo[selected_dma]]
+            elif selected_panel:
+                segment = df[df[selected_panel] == combo[selected_panel]]
+
+            # Adjust granularity of the segment
+            transformed_segment = adjust_dataframe_granularity(
+                segment, file_data["interval"], granularity_selection
+            )
+            transformed_segments.append(transformed_segment)
+
+        # Combine all transformed segments into a single DataFrame for this file
+        transformed_df = pd.concat(transformed_segments, ignore_index=True)
+        files_dict[file_name]["df"] = transformed_df
+
+    return files_dict
+
+
+# Function to create main dataframe structure
+st.cache_resource(show_spinner=False)
+
+
+def create_main_dataframe(
+    files_dict, all_dma_values, all_panel_values, granularity_selection
+):
+    # Determine the global start and end dates across all DataFrames
+    global_start = min(df["df"]["date"].min() for df in files_dict.values())
+    global_end = max(df["df"]["date"].max() for df in files_dict.values())
+
+    # Adjust the date_range generation based on the granularity_selection
+    if granularity_selection == "weekly":
+        # Generate a weekly range, with weeks starting on Monday
+        date_range = pd.date_range(start=global_start, end=global_end, freq="W-MON")
+    elif granularity_selection == "monthly":
+        # Generate a monthly range, starting from the first day of each month
+        date_range = pd.date_range(start=global_start, end=global_end, freq="MS")
+    else:  # Default to daily if not weekly or monthly
+        date_range = pd.date_range(start=global_start, end=global_end, freq="D")
+
+    # Collect all unique DMA and Panel values, excluding 'N/A'
+    all_dmas = all_dma_values
+    all_panels = all_panel_values
+
+    # Dynamically build the list of dimensions (Panel, DMA) to include in the main DataFrame based on availability
+    dimensions, merge_keys = [], []
+    if all_panels:
+        dimensions.append(all_panels)
+        merge_keys.append("Panel")
+    if all_dmas:
+        dimensions.append(all_dmas)
+        merge_keys.append("DMA")
+
+    dimensions.append(date_range)  # Date range is always included
+    merge_keys.append("date")  # Date range is always included
+
+    # Create a main DataFrame template with the dimensions
+    main_df = pd.MultiIndex.from_product(
+        dimensions,
+        names=[name for name, _ in zip(merge_keys, dimensions)],
+    ).to_frame(index=False)
+
+    return main_df.reset_index(drop=True)
+
+
+# Function to prepare and merge dataFrames
+st.cache_resource(show_spinner=False)
+
+
+def merge_into_main_df(main_df, files_dict, selections):
+    for file_name, file_data in files_dict.items():
+        df = file_data["df"].copy()
+
+        # Rename selected DMA and Panel columns if not 'N/A'
+        selected_dma = selections[file_name].get("DMA", "N/A")
+        selected_panel = selections[file_name].get("Panel", "N/A")
+        if selected_dma != "N/A":
+            df.rename(columns={selected_dma: "DMA"}, inplace=True)
+        if selected_panel != "N/A":
+            df.rename(columns={selected_panel: "Panel"}, inplace=True)
+
+        # Merge current DataFrame into main_df based on 'date', and where applicable, 'Panel' and 'DMA'
+        merge_keys = ["date"]
+        if "Panel" in df.columns:
+            merge_keys.append("Panel")
+        if "DMA" in df.columns:
+            merge_keys.append("DMA")
+        main_df = pd.merge(main_df, df, on=merge_keys, how="left")
+
+    # After all merges, sort by 'date' and reset index for cleanliness
+    sort_by = ["date"]
+    if "Panel" in main_df.columns:
+        sort_by.append("Panel")
+    if "DMA" in main_df.columns:
+        sort_by.append("DMA")
+    main_df.sort_values(by=sort_by, inplace=True)
+    main_df.reset_index(drop=True, inplace=True)
+
+    return main_df
+
+
+# Function to categorize column
+def categorize_column(column_name):
+    # Define keywords for each category
+    internal_keywords = [
+        "Price",
+        "Discount",
+        "product_price",
+        "cost",
+        "margin",
+        "inventory",
+        "sales",
+        "revenue",
+        "turnover",
+        "expense",
+    ]
+    exogenous_keywords = [
+        "GDP",
+        "Tax",
+        "Inflation",
+        "interest_rate",
+        "employment_rate",
+        "exchange_rate",
+        "consumer_spending",
+        "retail_sales",
+        "oil_prices",
+        "weather",
+    ]
+
+    # Check if the column name matches any of the keywords for Internal or Exogenous categories
+    for keyword in internal_keywords:
+        if keyword.lower() in column_name.lower():
+            return "Internal"
+    for keyword in exogenous_keywords:
+        if keyword.lower() in column_name.lower():
+            return "Exogenous"
+
+    # Default to Media if no match found
+    return "Media"
+
+
+# Function to calculate missing stats and prepare for editable DataFrame
+st.cache_resource(show_spinner=False)
+
+
+def prepare_missing_stats_df(df):
+    missing_stats = []
+    for column in df.columns:
+        if (
+            column == "date" or column == "DMA" or column == "Panel"
+        ):  # Skip Date, DMA and Panel column
+            continue
+
+        missing = df[column].isnull().sum()
+        pct_missing = round((missing / len(df)) * 100, 2)
+
+        # Dynamically assign category based on column name
+        # category = categorize_column(column)
+        category = "Media"
+
+        missing_stats.append(
+            {
+                "Column": column,
+                "Missing Values": missing,
+                "Missing Percentage": pct_missing,
+                "Impute Method": "Fill with 0",  # Default value
+                "Category": category,
+            }
+        )
+    stats_df = pd.DataFrame(missing_stats)
+
+    return stats_df
+
+
+# Function to add API DataFrame details to the files dictionary
+st.cache_resource(show_spinner=False)
+
+
+def add_api_dataframe_to_dict(main_df, files_dict):
+    files_dict["API"] = {
+        "numeric": list(main_df.select_dtypes(include=["number"]).columns),
+        "non_numeric": [
+            col
+            for col in main_df.select_dtypes(exclude=["number"]).columns
+            if col.lower() != "date"
+        ],
+        "interval": determine_data_interval(
+            pd.Series(main_df["date"].unique()).diff().dt.days.dropna().mode()[0]
+        ),
+        "df": main_df,
+    }
+
+    return files_dict
+
+
+# Function to reads an API into a DataFrame, parsing specified columns as datetime
+@st.cache_resource(show_spinner=False)
+def read_API_data():
+    return pd.read_excel(r"upf_data_converted.xlsx", parse_dates=["Date"])
+
+
+# Function to set the 'DMA_Panel_Selected' session state variable to False
+def set_DMA_Panel_Selected_false():
+    st.session_state["DMA_Panel_Selected"] = False
+
+
+# Initialize 'final_df' in session state
+if "final_df" not in st.session_state:
+    st.session_state["final_df"] = pd.DataFrame()
+
+# Initialize 'bin_dict' in session state
+if "bin_dict" not in st.session_state:
+    st.session_state["bin_dict"] = {}
+
+# Initialize 'DMA_Panel_Selected' in session state
+if "DMA_Panel_Selected" not in st.session_state:
+    st.session_state["DMA_Panel_Selected"] = False
+
+# Page Title
+st.write("")  # Top padding
+st.title("Data Import")
+
+
+#########################################################################################################################################################
+# Create a dictionary to hold all DataFrames and collect user input to specify "DMA" and "Panel" columns for each file
+#########################################################################################################################################################
+
+
+# Read the Excel file, parsing 'Date' column as datetime
+main_df = read_API_data()
+
+# Convert all column names to lowercase
+main_df.columns = main_df.columns.str.lower().str.strip()
+
+# File uploader
+uploaded_files = st.file_uploader(
+    "Upload additional data",
+    type=["xlsx"],
+    accept_multiple_files=True,
+    on_change=set_DMA_Panel_Selected_false,
+)
+
+# Custom HTML for upload instructions
+recommendation_html = f"""
+<div style="text-align: justify;">
+<strong>Recommendation:</strong> For optimal processing, please ensure that all uploaded datasets including DMA, Panel, media, internal, and exogenous data adhere to the following guidelines: Each dataset must include a <code>Date</code> column formatted as <code>DD-MM-YYYY</code>, be free of missing values.
+</div>
+"""
+st.markdown(recommendation_html, unsafe_allow_html=True)
+
+# Choose Date Granularity
+st.markdown("#### Choose Date Granularity")
+# Granularity Selection
+granularity_selection = st.selectbox(
+    "Choose Date Granularity",
+    ["Daily", "Weekly", "Monthly"],
+    label_visibility="collapsed",
+    on_change=set_DMA_Panel_Selected_false,
+)
+granularity_selection = str(granularity_selection).lower()
+
+# Convert files to dataframes
+files_dict = files_to_dataframes(uploaded_files)
+
+# Add API Dataframe
+if main_df is not None:
+    files_dict = add_api_dataframe_to_dict(main_df, files_dict)
+
+# Display a warning message if no files have been uploaded and halt further execution
+if not files_dict:
+    st.warning(
+        "Please upload at least one file to proceed.",
+        icon="⚠️",
+    )
+    st.stop()  # Halts further execution until file is uploaded
+
+
+# Select DMA and Panel columns
+st.markdown("#### Select DMA and Panel columns")
+selections = {}
+with st.expander("Select DMA and Panel columns", expanded=False):
+    count = 0  # Initialize counter to manage the visibility of labels and keys
+    for file_name, file_data in files_dict.items():
+        # Determine visibility of the label based on the count
+        if count == 0:
+            label_visibility = "visible"
+        else:
+            label_visibility = "collapsed"
+
+        # Extract non-numeric columns
+        non_numeric_cols = file_data["non_numeric"]
+
+        # Prepare DMA and Panel values for dropdown, adding "N/A" as an option
+        dma_values = non_numeric_cols + ["N/A"]
+        panel_values = non_numeric_cols + ["N/A"]
+
+        # Skip if only one option is available
+        if len(dma_values) == 1 and len(panel_values) == 1:
+            selected_dma, selected_panel = "N/A", "N/A"
+            # Update the selections for DMA and Panel for the current file
+            selections[file_name] = {
+                "DMA": selected_dma,
+                "Panel": selected_panel,
+            }
+            continue
+
+        # Create layout columns for File Name, DMA, and Panel selections
+        file_name_col, DMA_col, Panel_col = st.columns([2, 4, 4])
+
+        with file_name_col:
+            # Display "File Name" label only for the first file
+            if count == 0:
+                st.write("File Name")
+            else:
+                st.write("")
+            st.write(file_name)  # Display the file name
+
+        with DMA_col:
+            # Display a selectbox for DMA values
+            selected_dma = st.selectbox(
+                "Select DMA",
+                dma_values,
+                on_change=set_DMA_Panel_Selected_false,
+                label_visibility=label_visibility,  # Control visibility of the label
+                key=f"DMA_selectbox{count}",  # Ensure unique key for each selectbox
+            )
+
+        with Panel_col:
+            # Display a selectbox for Panel values
+            selected_panel = st.selectbox(
+                "Select Panel",
+                panel_values,
+                on_change=set_DMA_Panel_Selected_false,
+                label_visibility=label_visibility,  # Control visibility of the label
+                key=f"Panel_selectbox{count}",  # Ensure unique key for each selectbox
+            )
+
+        # Skip processing if the same column is selected for both Panel and DMA due to potential data integrity issues
+        if selected_panel == selected_dma and not (
+            selected_panel == "N/A" and selected_dma == "N/A"
+        ):
+            st.warning(
+                f"File: {file_name} → The same column cannot serve as both Panel and DMA. Please adjust your selections.",
+            )
+            selected_dma, selected_panel = "N/A", "N/A"
+            st.stop()
+
+        # Update the selections for DMA and Panel for the current file
+        selections[file_name] = {
+            "DMA": selected_dma,
+            "Panel": selected_panel,
+        }
+
+        count += 1  # Increment the counter after processing each file
+
+    # Accept DMA and Panel selection
+    if st.button("Accept and Process", use_container_width=True):
+
+        # Normalize all data to a daily granularity. This initial standardization simplifies subsequent conversions to other levels of granularity
+        with st.spinner("Processing...", cache=True):
+            files_dict = standardize_data_to_daily(files_dict, selections)
+
+            # Convert all data to daily level granularity
+            files_dict = apply_granularity_to_all(
+                files_dict, granularity_selection, selections
+            )
+
+        st.session_state["files_dict"] = files_dict
+        st.session_state["DMA_Panel_Selected"] = True
+
+
+#########################################################################################################################################################
+# Display unique DMA and Panel values
+#########################################################################################################################################################
+
+
+# Halts further execution until DMA and Panel columns are selected
+if "files_dict" in st.session_state and st.session_state["DMA_Panel_Selected"]:
+    files_dict = st.session_state["files_dict"]
+else:
+    st.stop()
+
+# Set to store unique values of DMA and Panel
+with st.spinner("Fetching DMA and Panel values..."):
+    all_dma_values, all_panel_values = clean_and_extract_unique_values(
+        files_dict, selections
+    )
+
+    # List of DMA and Panel columns unique values
+    list_of_all_dma_values = list(all_dma_values)
+    list_of_all_panel_values = list(all_panel_values)
+
+    # Format DMA and Panel values for display
+    formatted_dma_values = format_values_for_display(list_of_all_dma_values)
+    formatted_panel_values = format_values_for_display(list_of_all_panel_values)
+
+# Unique DMA and Panel values
+st.markdown("#### Unique DMA and Panel values")
+# Display DMA and Panel values
+with st.expander("Unique DMA and Panel values"):
+    st.write("")
+    st.markdown(
+        f"""
+    <style>
+    .justify-text {{
+    text-align: justify;
+    }}
+    </style>
+    <div class="justify-text">
+    <strong>Panel Values:</strong> {formatted_panel_values}<br>
+    <strong>DMA Values:</strong> {formatted_dma_values}
+    </div>
+    """,
+        unsafe_allow_html=True,
+    )
+
+    # Display total DMA and Panel
+    st.write("")
+    st.markdown(
+        f"""
+    <div style="text-align: justify;">
+        <strong>Number of DMAs detected:</strong> {len(list_of_all_dma_values)}<br>
+        <strong>Number of Panels detected:</strong> {len(list_of_all_panel_values)}
+    </div>
+    """,
+        unsafe_allow_html=True,
+    )
+    st.write("")
+
+
+#########################################################################################################################################################
+# Merge all DataFrames
+#########################################################################################################################################################
+
+
+# Merge all DataFrames selected
+main_df = create_main_dataframe(
+    files_dict, all_dma_values, all_panel_values, granularity_selection
+)
+merged_df = merge_into_main_df(main_df, files_dict, selections)
+
+# # Display the merged DataFrame
+# st.markdown("#### Merged DataFrame based on selected DMA and Panel")
+# st.dataframe(merged_df)
+
+
+#########################################################################################################################################################
+# Categorize Variables and Impute Missing Values
+#########################################################################################################################################################
+
+
+# Create an editable DataFrame in Streamlit
+st.markdown("#### Select Variables Category & Impute Missing Values")
+
+# Prepare missing stats DataFrame for editing
+missing_stats_df = prepare_missing_stats_df(merged_df)
+
+edited_stats_df = st.data_editor(
+    missing_stats_df,
+    column_config={
+        "Impute Method": st.column_config.SelectboxColumn(
+            options=[
+                "Drop Column",
+                "Fill with Mean",
+                "Fill with Median",
+                "Fill with 0",
+            ],
+            required=True,
+            default="Fill with 0",
+        ),
+        "Category": st.column_config.SelectboxColumn(
+            options=[
+                "Media",
+                "Exogenous",
+                "Internal",
+                "Response_Metric"
+            ],
+            required=True,
+            default="Media",
+        ),
+    },
+    disabled=["Column", "Missing Values", "Missing Percentage"],
+    hide_index=True,
+    use_container_width=True,
+)
+
+# Apply changes based on edited DataFrame
+for i, row in edited_stats_df.iterrows():
+    column = row["Column"]
+    if row["Impute Method"] == "Drop Column":
+        merged_df.drop(columns=[column], inplace=True)
+
+    elif row["Impute Method"] == "Fill with Mean":
+        merged_df[column].fillna(merged_df[column].mean(), inplace=True)
+
+    elif row["Impute Method"] == "Fill with Median":
+        merged_df[column].fillna(merged_df[column].median(), inplace=True)
+
+    elif row["Impute Method"] == "Fill with 0":
+        merged_df[column].fillna(0, inplace=True)
+
+# Display the Final DataFrame and exogenous variables
+st.markdown("#### Final DataFrame")
+final_df = merged_df
+st.dataframe(final_df, hide_index=True)
+
+# Initialize an empty dictionary to hold categories and their variables
+category_dict = {}
+
+# Iterate over each row in the edited DataFrame to populate the dictionary
+for i, row in edited_stats_df.iterrows():
+    column = row["Column"]
+    category = row["Category"]  # The category chosen by the user for this variable
+
+    # Check if the category already exists in the dictionary
+    if category not in category_dict:
+        # If not, initialize it with the current column as its first element
+        category_dict[category] = [column]
+    else:
+        # If it exists, append the current column to the list of variables under this category
+        category_dict[category].append(column)
+
+# Add Date, DMA and Panel in category dictionary
+category_dict.update({"Date": ["date"]})
+if "DMA" in final_df.columns:
+    category_dict["DMA"] = ["DMA"]
+
+if "Panel" in final_df.columns:
+    category_dict["Panel"] = ["Panel"]
+
+# Display the dictionary
+st.markdown("#### Variable Category")
+for category, variables in category_dict.items():
+    # Check if there are multiple variables to handle "and" insertion correctly
+    if len(variables) > 1:
+        # Join all but the last variable with ", ", then add " and " before the last variable
+        variables_str = ", ".join(variables[:-1]) + " and " + variables[-1]
+    else:
+        # If there's only one variable, no need for "and"
+        variables_str = variables[0]
+
+    # Display the category and its variables in the desired format
+    st.markdown(
+        f"<div style='text-align: justify;'><strong>{category}:</strong> {variables_str}</div>",
+        unsafe_allow_html=True,
+    )
+
+# Store final dataframe and bin dictionary into session state
+st.session_state["final_df"], st.session_state["bin_dict"] = final_df, category_dict
+
+if st.button('Save Changes'):
+    
+    with open("Pickle_files/main_df", 'wb') as f:
+        pickle.dump(st.session_state["final_df"], f)
+    with open("Pickle_files/category_dict",'wb') as c:
+        pickle.dump(st.session_state["bin_dict"],c)
+    st.success('Changes Saved!')
+
+
+

pages/actual_data.csv

@@ -0,0 +1,158 @@
+const,clicks_search_decay.2,impressions_tv_lag3,online_edu_trend_lag3,clicks_digital_lag2_decay.3,impressions_streaming_lag2_decay.4,covid_cases_lag3,unemployement_rate_lead4,season,flag_Aug_1,flag_Aug_2,flag_Aug_3,flag_dec_1,flag_dec_-1,flag_dec_-2,flag_dec_-3,flag_easter_-1,flag_easter_-2,flag_may_-1,flag_may_-2,flag_jun_-1,flag_jun_-2,covid_flag1,flag_june28,flag_aug13,flag_sep13,flag_mar_feb,date,total_prospect_id
+1.0,0.03264506089026503,0.0,0.0,0.0,0.11920857922376585,0.0,0.2448979591836735,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2019-11-10,3106
+1.0,0.1203178311529351,0.0,0.0,0.0,0.23575959332216032,0.0,0.2448979591836735,101,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2019-11-17,7809
+1.0,0.037674240888288246,0.0,0.0,0.30427286753070926,0.14866425214344534,0.0,0.2448979591836735,102,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2019-11-24,5658
+1.0,0.114056065999327,0.25459834519940233,0.5700000000000001,0.3210660307498862,0.06375317695001911,0.0,0.2448979591836735,103,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2019-12-01,7528
+1.0,0.15091848146432302,0.04759636387261456,0.58,0.2652143429433443,0.02550166207848893,0.0,0.2380952380952381,104,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2019-12-08,8913
+1.0,0.09691798534505919,0.0,0.41000000000000003,0.27398476053158455,0.22803554179688423,0.0,0.2380952380952381,105,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2019-12-15,7974
+1.0,0.0,0.2185391903071715,0.53,0.3093665823461814,0.3016670242357716,0.0,0.2380952380952381,106,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2019-12-22,5034
+1.0,0.06818143419410627,0.0645557652165116,0.6,0.35005256364095544,0.3915886857834677,0.0,0.2380952380952381,107,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2019-12-29,8296
+1.0,0.19748095587743647,0.0,0.49,0.2866388037412839,0.4644891817948484,0.0,0.2380952380952381,108,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-01-05,10953
+1.0,0.2718903484441833,0.31632836028874944,0.42,0.38339772931601046,0.4758788391710054,0.0,0.2380952380952381,109,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,2020-01-12,11583
+1.0,0.29329394272923165,0.710207473795361,0.56,0.4716341482535363,0.47415700741999534,0.0,0.2380952380952381,110,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,2020-01-19,11650
+1.0,0.3150710926081645,0.6225458397661645,0.66,0.5560651882029227,0.2282082561307921,0.0,0.2380952380952381,111,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-01-26,10086
+1.0,0.23335326208386092,0.5093471390869946,0.65,0.5990392189890996,0.09128427138188955,0.0,0.2993197278911565,112,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-02-02,8454
+1.0,0.18339704064539092,0.46920681970876166,0.66,0.5097387360461574,0.03651393215188798,0.0,0.2993197278911565,113,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-02-09,7842
+1.0,0.1829206162885479,0.5702922924005152,0.64,0.3647117781342298,0.5333315970976881,0.0,0.2993197278911565,114,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-02-16,8528
+1.0,0.17708137647064887,0.4762803199026322,0.62,0.2994390381863003,0.9999999999999999,0.0,0.2993197278911565,115,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-02-23,9230
+1.0,0.2110785179466496,0.31643298954206356,0.65,0.318727924805625,0.5153399788387041,0.0,0.2993197278911565,116,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-03-01,8210
+1.0,0.1922309642774856,0.35110354589746834,0.65,0.3435805763353255,0.20613623376787482,0.0,1.0,117,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-03-08,6573
+1.0,0.1174971533357681,0.4397302099507956,0.64,0.37079693119819457,0.08245451214041095,0.0,1.0,118,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,2020-03-15,4464
+1.0,0.04487177585471158,0.5651604986093057,0.66,0.3797815418753292,0.032981804856164386,3.6661729553753427e-06,1.0,119,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,2020-03-22,5498
+1.0,0.04417426781579725,0.5142518574426083,0.77,0.3239901926717436,0.013192796475509808,0.00016497778299189042,1.0,120,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-03-29,7134
+1.0,0.09508966430933447,0.4246084040047787,1.0,0.22766051203571303,0.005277118590203924,0.01074555293220513,0.8979591836734694,121,0,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,2020-04-05,6507
+1.0,0.1727148072921107,0.3306303340730278,0.92,0.2557126494916798,0.0021108474360815696,0.07506489126131015,0.8979591836734694,122,0,0,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,2020-04-12,6752
+1.0,0.2757761792524949,0.9059477066272279,0.87,0.2910560761584964,0.0008443389744326279,0.11051311756683434,0.8979591836734694,123,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-04-19,7874
+1.0,0.46164669127102737,1.0,0.8200000000000001,0.29288325042575475,0.0003377355897730512,0.1323451775160945,0.8979591836734694,124,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-04-26,8706
+1.0,0.3631365926708698,0.8555262504044332,0.85,0.3143348639913703,0.00013509423590922048,0.12527679605813083,0.8979591836734694,125,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-05-03,9593
+1.0,0.3556269301486625,0.5998066602658987,0.8,0.3573452157072908,5.4838924587260594e-05,0.08418266340132861,0.7482993197278912,126,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-05-10,9554
+1.0,0.3898924329688705,0.31953123019194307,0.76,0.3492819601843694,0.08837696494340691,0.06699197841357364,0.7482993197278912,127,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-05-17,9461
+1.0,0.3270785638817633,0.5040802333471541,0.88,0.37224504100306005,0.12944061135952373,0.04806352744497074,0.7482993197278912,128,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-05-24,8347
+1.0,0.29596428185745655,0.6228739252579004,0.8300000000000001,0.3873711562094451,0.14079607140381442,0.028926104617911456,0.7482993197278912,129,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-05-31,7926
+1.0,0.23446621861142697,0.644779308361226,0.8,0.3519020717491842,0.15750706055823313,0.024482702995996537,0.6938775510204082,130,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-06-07,8606
+1.0,0.2202508917985891,0.726916988225644,0.71,0.32726146750928653,0.0797309833640819,0.022000703905207433,0.6938775510204082,131,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-06-14,7573
+1.0,0.18610614076735926,0.5963517592669729,0.73,0.31618831243754153,0.03501476889363339,0.015086301711369536,0.6938775510204082,132,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,2020-06-21,6983
+1.0,0.1568177529621934,0.6764095796293655,0.75,0.2836099513597926,0.014005944823975384,0.011489786042146325,0.6938775510204082,133,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-06-28,6277
+1.0,0.22774801916471138,0.6466210070345804,0.72,0.25409997289933184,0.006272411362367827,0.00871449311492719,0.5714285714285715,134,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-07-05,7421
+1.0,0.24542124594101095,0.6580063264819511,0.73,0.2516667689694555,0.05947462601462651,0.008318546435746652,0.5714285714285715,135,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-07-12,7852
+1.0,0.24895270375190542,0.32749815383926373,0.68,0.2671053898526598,0.0888609058832765,0.008014254080450499,0.5714285714285715,136,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-07-19,7396
+1.0,0.16285259960994197,0.3666961464656464,0.78,0.26077100654286645,0.12420199588573878,0.008058248155915004,0.5714285714285715,137,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-07-26,7041
+1.0,0.16864346155569104,0.39341698388602436,0.84,0.25893225300958655,0.10423952696584138,0.00920209411799211,0.5714285714285715,138,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-08-02,7470
+1.0,0.22582910125625383,0.41507293852636135,0.8300000000000001,0.2528768986269057,0.08197739941078482,0.009315745479608745,0.5374149659863946,139,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-08-09,8725
+1.0,0.2778946696783185,0.7857143231388266,0.8,0.2772125371796957,0.07178679747906064,0.007237025413910927,0.5374149659863946,140,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-08-16,9657
+1.0,0.3062154076077969,0.434016630925742,0.87,0.33174759696083367,0.12078972986041582,0.006500124649880482,0.5374149659863946,141,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,2020-08-23,10000
+1.0,0.2851073700683267,0.4051792323256236,0.8200000000000001,0.3621387745268235,0.1539969659046611,0.006118842662521447,0.5374149659863946,142,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-08-30,8941
+1.0,0.25999778433367665,0.4113785668398346,0.77,0.3604714968693371,0.1462622685965232,0.006375474769397721,0.4693877551020409,143,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-09-06,8507
+1.0,0.2947500457787596,0.43576671635701947,0.74,0.3084711376902622,0.1030893445960345,0.0060051913009048115,0.4693877551020409,144,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,2020-09-13,9887
+1.0,0.3239559328273078,0.40721834097732834,0.72,0.24061271129609485,0.08422768334333634,0.006456130574415978,0.4693877551020409,145,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-09-20,9627
+1.0,0.3189849597494306,0.4831656702512836,0.68,0.28577062852640756,0.054400116894051116,0.006401137980085348,0.4693877551020409,146,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-09-27,8735
+1.0,0.2930673557404469,0.5423730023996388,0.62,0.32330756771945346,0.02176006539088146,0.007566980979894707,0.45578231292517013,147,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-10-04,8138
+1.0,0.27381401410957934,0.48862464971809444,0.59,0.33668984325037016,0.008704026156352586,0.009172764734349107,0.45578231292517013,148,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-10-11,7966
+1.0,0.21658154029531146,0.5162854532967293,0.55,0.44481231480084876,0.003481610462541034,0.012223020633221393,0.45578231292517013,149,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-10-18,8109
+1.0,0.21772903332032795,0.47368257634991157,0.6,0.46141705479304307,0.0013926441850164136,0.013601501664442522,0.45578231292517013,150,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-10-25,7848
+1.0,0.16712357438522701,0.5132571164009214,0.5,0.38402389059771924,0.0005570576740065655,0.012915927321787332,0.45578231292517013,151,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-11-01,6516
+1.0,0.1814031347156822,0.5409537987241609,0.5,0.2968208337801042,0.00022282306960262618,0.013091903623645349,0.45578231292517013,152,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-11-08,7233
+1.0,0.16852532779394064,0.49490997931858044,0.5,0.22663075929954526,8.912922784105048e-05,0.014624363918992243,0.45578231292517013,153,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-11-15,7409
+1.0,0.10492104198879731,0.4086344123814518,0.41000000000000003,0.21669561761817938,3.565169113642019e-05,0.016127494830696133,0.45578231292517013,154,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-11-22,6232
+1.0,0.16920169406380464,0.45151008168804235,0.49,0.21833619946593313,1.4260676454568076e-05,0.024849320291534072,0.45578231292517013,155,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-11-29,8170
+1.0,0.1305885456099783,0.4543635808918873,0.47000000000000003,0.1596898931167178,5.704270581827231e-06,0.03519159419864792,0.435374149659864,156,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-12-06,7075
+1.0,0.1214984593864375,0.35070760971315756,0.4,0.15417676852356046,2.2817082327308923e-06,0.041732046751037526,0.435374149659864,157,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-12-13,7379
+1.0,0.057042007816384965,0.32470890321593604,0.47000000000000003,0.15442387578570832,9.126832930923571e-07,0.049892947749703036,0.435374149659864,158,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-12-20,5442
+1.0,0.12406882983279183,0.3135816516054531,0.45,0.1671308209739812,3.650733172369429e-07,0.0686930826648678,0.435374149659864,159,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2020-12-27,7735
+1.0,0.24786523070013738,0.3102913429236421,0.42,0.16347790840061424,1.4602932689477716e-07,0.0732574679943101,0.435374149659864,160,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-01-03,9754
+1.0,0.26083059672146286,0.2649240941306087,0.34,0.25327016920452516,5.841173075791087e-08,0.07444897420480709,0.4217687074829932,161,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,2021-01-10,10641
+1.0,0.24028847292133387,0.6513962629200784,0.38,0.3773812732234543,2.3364692303164347e-08,0.08318546435746653,0.4217687074829932,162,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,2021-01-17,10230
+1.0,0.31526302386797916,0.531674302460824,0.47000000000000003,0.3527386460097067,9.345876921265738e-09,0.10258685163731283,0.4217687074829932,163,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-01-24,10352
+1.0,0.2966293410018717,0.44836670500794606,0.47000000000000003,0.3711695518795665,3.738350768506295e-09,0.13234151134313912,0.4217687074829932,164,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-01-31,9216
+1.0,0.20088776123137192,0.3815806999416851,0.45,0.33580461662371014,1.4953403074025183e-09,0.12043744775703538,0.40816326530612246,165,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-02-07,8421
+1.0,0.173394454128539,0.343687050600215,0.48,0.3277941002786073,5.981361229610074e-10,0.11271648751301491,0.40816326530612246,166,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-02-14,9281
+1.0,0.1777198044422716,0.33051072402008147,0.5,0.31487397296804576,2.3925444918440296e-10,0.109699227170741,0.40816326530612246,167,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-02-21,8891
+1.0,0.1850269016675808,0.30627520154343757,0.46,0.3133091660972597,9.570177967376119e-11,0.08255854878209734,0.40816326530612246,168,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-02-28,8169
+1.0,0.2529549962208855,0.298123038215738,0.42,0.3358964981168952,3.828071186950448e-11,0.08351908609640568,0.40816326530612246,169,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-03-07,8724
+1.0,0.213028120324469,0.3267901551549544,0.44,0.3038053348505854,1.531228474780179e-11,0.07285052279626343,0.40816326530612246,170,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-03-14,8194
+1.0,0.16441430466323353,0.25967469209260036,0.5,0.32087357753439977,6.124913899120717e-12,0.07822879852179906,0.40816326530612246,171,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-03-21,8254
+1.0,0.11053130189212229,0.260168451958828,0.42,0.3279459500984871,2.449965559648287e-12,0.07333812379932836,0.40816326530612246,172,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-03-28,7026
+1.0,0.06917021315146277,0.0,0.38,0.37411287881420296,9.799862238593149e-13,0.07465061371735272,0.39455782312925175,173,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-04-04,6412
+1.0,0.06728264676731566,0.0,0.44,0.4347510050616973,3.9199448954372595e-13,0.0732721326861316,0.39455782312925175,174,0,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,2021-04-11,6297
+1.0,0.10167805497311716,0.0,0.43,0.4574504815633023,1.5679779581749037e-13,0.07982724993034271,0.39455782312925175,175,0,0,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,2021-04-18,6687
+1.0,0.1734619149834527,0.0,0.48,0.48912312446006045,6.271911832699615e-14,0.06941165256412136,0.39455782312925175,176,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-04-25,8430
+1.0,0.2040432878056308,0.0,0.46,0.44466429049983563,2.5087647330798465e-14,0.06276854716898124,0.39455782312925175,177,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-05-02,8025
+1.0,0.20788046814877387,0.0,0.48,0.5722675873212515,1.0035058932319387e-14,0.04882242524673344,0.40136054421768713,178,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-05-09,8242
+1.0,0.14929264058846564,0.0,0.5,0.45913415146070335,4.014023572927755e-15,0.033618806000791895,0.40136054421768713,179,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-05-16,8280
+1.0,0.11694210039888364,0.0,0.51,0.39528662679579885,1.6056094291711022e-15,0.025182942030473228,0.40136054421768713,180,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-05-23,7909
+1.0,0.055184035342337234,0.0,0.51,0.3880077087936407,6.422437716684409e-16,0.017652622780132275,0.40136054421768713,181,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-05-30,7574
+1.0,0.04358787034563821,0.0,0.5,0.3863265622647678,2.568975086673764e-16,0.012651962869000308,0.3673469387755103,182,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-06-06,7270
+1.0,0.03833609653008979,0.0,0.46,0.3784495643657444,1.0275900346695056e-16,0.008835476822454577,0.3673469387755103,183,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-06-13,6716
+1.0,0.06111263589867566,0.0,0.48,0.38862024435317233,4.1103601386780226e-17,0.005939200187708055,0.3673469387755103,184,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-06-20,6944
+1.0,0.07119833324643848,0.0,0.44,0.4039000969934476,1.644144055471209e-17,0.004967664354533589,0.3673469387755103,185,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,2021-06-27,6803
+1.0,0.0659956847282599,0.0,0.45,0.4420872417106599,6.576576221884836e-18,0.004359079643941282,0.3537414965986395,186,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-07-04,7019
+1.0,0.12577031397293442,0.0,0.45,0.4950177419852857,2.630630488753935e-18,0.003977797656582247,0.3537414965986395,187,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-07-11,8254
+1.0,0.1502746019886232,0.0,0.45,0.5650602702260171,1.052252195501574e-18,0.0040621196345558795,0.3537414965986395,188,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-07-18,7804
+1.0,0.21001397285486328,0.0,0.42,0.594015126140436,4.209008782006296e-19,0.004952999662712088,0.3537414965986395,189,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-07-25,8212
+1.0,0.23464189851384848,0.0,0.46,0.5484130743981998,1.6836035128025183e-19,0.008076579020691881,0.3537414965986395,190,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-08-01,8378
+1.0,0.23496148203757855,0.0,0.47000000000000003,0.5324473242588711,6.734414051210074e-20,0.01220102359548914,0.3197278911564626,191,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-08-08,9496
+1.0,0.23319893582092505,0.0,0.53,0.5532778727756644,2.6937656204840295e-20,0.020152952735698258,0.3197278911564626,192,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-08-15,9511
+1.0,0.23262329847201318,0.0,0.49,0.7309984534528141,1.0775062481936118e-20,0.029028757460661962,0.3197278911564626,193,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-08-22,9569
+1.0,0.18495638415853394,0.0,0.46,0.8724050615489382,4.310024992774448e-21,0.03698435277382646,0.3197278911564626,194,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-08-29,7928
+1.0,0.2921700012245981,0.0,0.49,1.0,1.7240099971097793e-21,0.03982197064128697,0.3129251700680272,195,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-09-05,7840
+1.0,0.4172971677569805,0.0,0.48,0.8193686075762131,6.896039988439117e-22,0.03868179085216524,0.3129251700680272,196,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-09-12,9521
+1.0,0.5004920981884484,0.0,0.53,0.4496097944711011,2.758415995375647e-22,0.03902274493701515,0.3129251700680272,197,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-09-19,9451
+1.0,0.6383788968475093,0.0,0.47000000000000003,0.3701822126418114,1.1033663981502588e-22,0.03567186285580209,0.3129251700680272,198,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-09-26,8898
+1.0,0.6501651617929107,0.0,0.51,0.34258196039636274,4.413465592601035e-23,0.0352539191388893,0.3129251700680272,199,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-10-03,8441
+1.0,0.6649283374522998,0.0,0.51,0.31355701111053985,1.7653862370404143e-23,0.03635010485254652,0.28571428571428575,200,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-10-10,8788
+1.0,0.6097114754591861,0.0,0.51,0.32306971094469733,7.061544948161657e-24,0.031323781730726925,0.28571428571428575,201,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-10-17,9569
+1.0,0.3964279757062242,0.0,0.51,0.33051520280988034,2.8246179792646632e-24,0.02719933715592967,0.28571428571428575,202,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-10-24,9008
+1.0,0.33105364706311086,0.0,0.47000000000000003,0.3259978333423606,1.1298471917058652e-24,0.025967503042923553,0.28571428571428575,203,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-10-31,8495
+1.0,0.31714045716637634,0.0,0.55,0.3045528431182349,4.519388766823461e-25,0.02263128565353199,0.2653061224489796,204,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-11-07,8807
+1.0,0.28268319082761023,0.0,0.49,0.31370309424641213,1.8077555067293845e-25,0.01786159463858867,0.2653061224489796,205,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-11-14,8385
+1.0,0.15774740707436136,0.0,0.51,0.37945364695975814,7.231022026917538e-26,0.016409790148260033,0.2653061224489796,206,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-11-21,6964
+1.0,0.2836203500514554,0.0,0.55,0.36793503370466,2.892408810767015e-26,0.01882946429880776,0.2653061224489796,207,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-11-28,9340
+1.0,0.33646919882766096,0.0,0.49,0.3299836196379579,1.1569635243068062e-26,0.023555161238286576,0.272108843537415,208,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-12-05,8632
+1.0,0.361268166630245,0.0,0.38,0.3243428164088717,4.6278540972272255e-27,0.029421037966887126,0.272108843537415,209,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-12-12,9271
+1.0,0.21850759166298056,0.0,0.51,0.34100191273497404,1.8511416388908902e-27,0.029549354020325262,0.272108843537415,210,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-12-19,7663
+1.0,0.2156152088113536,0.0,0.43,0.3876459690915292,7.404566555563562e-28,0.04853646375621416,0.272108843537415,211,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2021-12-26,7888
+1.0,0.4122692273972545,0.0,0.42,0.44121852053456856,2.961826622225425e-28,0.07303383144403221,0.272108843537415,212,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-01-02,11088
+1.0,0.5580863257308297,0.0,0.42,0.33648328199770844,1.18473064889017e-28,0.2914790808171166,0.2585034013605442,213,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-01-09,12850
+1.0,0.5441541455767391,0.0,0.45,0.5258301345263098,4.7389225955606806e-29,0.6228644542534939,0.2585034013605442,214,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,2022-01-16,12768
+1.0,0.37953926965668333,0.0,0.51,0.6191133700101356,1.8955690382242722e-29,1.0,0.2585034013605442,215,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,2022-01-23,11023
+1.0,0.3422525462363791,0.0,0.5,0.6600516747429145,7.582276152897087e-30,0.8603298089190655,0.2585034013605442,216,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-01-30,10317
+1.0,0.3679329127754763,0.0,0.49,0.6150147631969254,3.0329104611588346e-30,0.3851571321728674,0.2448979591836735,217,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-02-06,10109
+1.0,0.3530129569359208,0.0,0.49,0.5435710104633258,1.2131641844635335e-30,0.18207314748280565,0.2448979591836735,218,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2022-02-13,10233
+1.0,0.3628237688509028,0.0,0.48,0.5395383650448762,4.852656737854129e-31,0.08532284319045035,0.2448979591836735,219,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2022-02-20,10660
+1.0,0.3535562124344392,0.0,0.49,0.3713089856353334,1.941062695141646e-31,0.04778123212740684,0.2448979591836735,220,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2022-02-27,9862
+1.0,0.35851767100446613,0.0,0.49,0.33021424233802193,7.764250780566529e-32,0.028365180155739026,0.2448979591836735,221,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2022-03-06,10393
+1.0,0.3648140365425708,0.0,0.53,0.29899648842829235,3.105700312226557e-32,0.019053100849085656,0.2448979591836735,222,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2022-03-13,9914
+1.0,0.417768904168966,0.0,0.46,0.30801461857263196,1.242280124890568e-32,0.014096435013418193,0.2448979591836735,223,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2022-03-20,11027
+1.0,0.45364666714531404,0.0,0.5,0.29874033139572204,4.9691204995617213e-33,0.013440190054406007,0.2448979591836735,224,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-03-27,10066
+1.0,0.45997433293937545,0.0,0.45,0.3080341285301519,1.9876481998241388e-33,0.014672024167412121,0.2448979591836735,225,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-04-03,8722
+1.0,0.4245480429075594,0.0,0.46,0.304189689538618,7.950592799291056e-34,0.01936472555029256,0.2448979591836735,226,0,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,2022-04-10,7805
+1.0,0.4463068738641009,0.0,0.54,0.307818077305473,3.1802371197109226e-34,0.027822586558343475,0.2448979591836735,227,0,0,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,2022-04-17,8519
+1.0,0.6012222981571669,0.0,0.53,0.29394180576819906,1.272094847878869e-34,0.033340176856183366,0.2448979591836735,228,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-04-24,10084
+1.0,0.6804106164543928,0.0,0.5,0.28219281269675367,5.088379391460478e-35,0.04576117082899503,0.2448979591836735,229,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-05-01,10291
+1.0,0.62805714350389,0.0,0.54,0.30839694661979145,2.035351756529193e-35,0.05172603422739071,0.2448979591836735,230,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-05-08,9743
+1.0,0.7470007501508245,0.0,0.54,0.3120111152265925,8.141407025566787e-36,0.04952999662712088,0.2448979591836735,231,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-05-15,10759
+1.0,0.6460736106378411,0.0,0.55,0.2905779236460707,3.25656280967673e-36,0.06457597043598129,0.2448979591836735,232,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-05-22,9845
+1.0,0.5732108245519132,0.0,0.52,0.38068837954927237,1.3026251233207076e-36,0.080201199571791,0.2448979591836735,233,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-05-29,9499
+1.0,0.5996683384067256,0.0,0.5,0.3940488499594224,5.210500487782985e-37,0.09049581323048496,0.40680272108843546,234,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-06-05,10021
+1.0,0.5630659455826548,0.0,0.54,0.4539755399873685,2.0842001896133483e-37,0.09128037424293528,0.40680272108843546,235,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-06-12,10112
+1.0,0.5482324249484887,0.0,0.45,0.48814019600803654,8.336800703454939e-38,0.08289217052103649,0.40680272108843546,236,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-06-19,10034
+1.0,0.5485743918729864,0.0,0.47000000000000003,0.475428506654356,3.3347202263835196e-38,0.06987359035649866,0.40680272108843546,237,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,2022-06-26,9209
+1.0,0.5559932625646005,0.0,0.43,0.510072176038165,1.333888035554951e-38,0.06264756346145385,0.40680272108843546,238,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-07-03,10265
+1.0,0.6089718159266746,0.0,0.45,0.44215508529036335,5.33555159223524e-39,0.0627612148230705,0.40680272108843546,239,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-07-10,10033
+1.0,0.6101706458097598,0.0,0.48,0.41550269661979555,2.1342200869095313e-39,0.07072780865510112,0.40680272108843546,240,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-07-17,9790
+1.0,0.6111403594460636,0.0,0.44,0.437146146258812,8.536874847792479e-40,0.07964760745552932,0.40680272108843546,241,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-07-24,9629
+1.0,0.6451477728019566,0.0,0.44,0.4975101423754845,3.4147444392713438e-40,0.0893739643061401,0.40680272108843546,242,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-07-31,10134
+1.0,0.7267513590970145,0.0,0.44,0.5042632593424633,1.3658922758628901e-40,0.09389435556011791,0.40680272108843546,243,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-08-07,12029
+1.0,0.832744074444703,0.0,0.46,0.5840915039533217,5.463514104995084e-41,0.08482790984147467,0.40680272108843546,244,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-08-14,12886
+1.0,0.8546151893753493,0.0,0.49,0.6374603327364593,2.1853506435415578e-41,0.07962194424484169,0.40680272108843546,245,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-08-21,12027
+1.0,0.9999999999999998,0.0,0.55,0.6022458246191313,8.740852589601472e-42,0.07178366646624922,0.40680272108843546,246,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-08-28,11375
+1.0,0.860672618209781,0.0,0.48,0.5735957859704555,3.495791051275827e-42,0.05725095687114135,0.40680272108843546,247,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-09-04,10824
+1.0,0.8622728019659036,0.0,0.54,0.5790428094946118,1.39776643594557e-42,0.050739833702394745,0.40680272108843546,248,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-09-11,12285
+1.0,0.7774120906393625,0.0,0.55,0.7618650061054455,5.585565898134668e-43,0.0440857297883885,0.40680272108843546,249,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-09-18,12146
+1.0,0.6580209603679659,0.0,0.52,0.8137272725878776,2.2287265136062566e-43,0.039975949905412735,0.40680272108843546,250,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-09-25,10881
+1.0,0.9480011027127861,0.0,0.52,0.7867690657367606,8.859907597948911e-44,0.03648941942485079,0.40680272108843546,251,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-10-02,11373
+1.0,0.709096498806814,0.0,0.46,0.7292818780372798,3.4889645827034517e-44,0.04076784326377381,0.40680272108843546,252,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-10-09,10230
+1.0,0.5414415970743589,0.0,0.45,0.6974583695681711,1.340587376605267e-44,0.04368978310920796,0.0,253,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-10-16,11557
+1.0,0.6081525119323576,0.0,0.54,0.6240593695822464,4.812364941659934e-45,0.041156457597043596,0.0,254,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-10-23,10805
+1.0,0.5960421531458853,0.0,0.45,0.5899287906913332,1.3749614119028383e-45,0.03843982343711047,0.0,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-10-30,9709
+1.0,0.848521629204434,0.0,0.47000000000000003,0.6201930426013046,0.0,0.040723849188309305,0.0,256,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-11-06,10098