first commit

main.py

@@ -0,0 +1,455 @@
+from numpy import cov
+import streamlit as st
+st.set_page_config(layout = "wide")
+import pandas as pd
+import matplotlib.pyplot as plt
+
+import numpy as np
+import seaborn as sns
+sns.set(style='white',color_codes=True)
+
+from sklearn.metrics import r2_score, median_absolute_error, mean_absolute_error
+from sklearn.metrics import median_absolute_error, mean_squared_error, mean_squared_log_error
+
+from scipy.optimize import minimize
+import statsmodels.tsa.api as smt
+import statsmodels.api as sm
+
+from tqdm import tqdm_notebook
+from tqdm.notebook import tqdm
+
+
+from itertools import product
+
+
+header=st.container()
+dataset=st.container()
+data_exploration_with_cleaning=st.container()
+features=st.container()
+modelTraining=st.container()
+covid_relationship=st.container()
+
+mystyle = '''
+    <style>
+        .main {
+                background_color:#FFCCFF;
+        }
+    </style>
+    '''
+
+# @st.cache(allow_output_mutation=True)
+def load_data(filename):
+    covid_data=pd.read_csv(filename)
+    return covid_data
+
+with header:
+    st.title('Covid-19 Analysis for predictive analytics')
+    st.text('Aims to provide appropriate analytics and showcasing the relationship between different diseases and covid 19')
+
+with dataset:
+    st.subheader('Dataset 1:ISDH - VR or NBS covid dataset as of July 4, 2022, 9:37 PM (UTC+03:00)')
+    st.subheader('Dataset 2: cdv.gov dataset')
+    covid_data=load_data('data/covid.csv')
+    covid_data.rename(columns = {'_id':'id', 'date':'date', 'agegrp':'age_group'},inplace=True)
+    covid_data['date'] = covid_data['date'].str[:-9]
+
+    covid_data['date'] = pd.to_datetime(covid_data['date'])
+   
+    st.write(covid_data.head(5))
+   
+with data_exploration_with_cleaning:
+    st.subheader('Data exploratory and cleaning')
+    nRow, nCol = covid_data.shape
+    st.write('* **Shape of our data is :** ', nRow, nCol )
+    summary=covid_data.describe()
+    st.write('* **Statistical summary :** ', summary)
+    a=covid_data.isnull().sum()
+    st.write('* **Checking for null values** ', a)
+    w=covid_data['age_group'].unique()
+    st.write('* **Age Group categories** ', w)
+with features:
+    st.subheader('Features of the dataset')
+    
+
+    covid_data.drop("id", axis=1, inplace=True)
+
+    covid_data.to_csv('data/cleaned_data.csv',index=False)
+
+    dat=pd.read_csv('data/cleaned_data.csv')
+    dat['date']= pd.to_datetime(dat['date'])
+    dat.to_csv('data/cleaned_data.csv',index=False)
+    data=pd.read_csv('data/cleaned_data.csv',index_col=['date'], parse_dates=['date'])
+
+    group1 = data.loc[data['age_group'] == '0-19']
+    group2 = data.loc[data['age_group'] == '20-29']
+    group3 = data.loc[data['age_group'] == '30-39']
+    group4 = data.loc[data['age_group'] == '40-49']
+    group5 = data.loc[data['age_group'] == '50-59']
+    group6 = data.loc[data['age_group'] == '60-69']
+    group7 = data.loc[data['age_group'] == '70-79']
+    group8 = data.loc[data['age_group'] == '80+']
+
+    a=plt.figure(figsize=(17, 8))
+    plt.plot(group1.covid_deaths)
+    plt.title('Infection Rate in 0-19 Years')
+    plt.ylabel('Number of Infection')
+    plt.xlabel('Period')
+    plt.grid(False)
+    # plt.show()
+
+    b=plt.figure(figsize=(17, 8))
+    plt.plot(group2.covid_deaths)
+    plt.title('Infection Rate in 20-29 Years')
+    plt.ylabel('Number of Infection')
+    plt.xlabel('Period')
+    plt.grid(False)
+    # plt.show()
+
+    c=plt.figure(figsize=(17, 8))
+    plt.plot(group3.covid_deaths)
+    plt.title('Infection Rate in 30-39 Years')
+    plt.ylabel('Number of Infection')
+    plt.xlabel('Period')
+    plt.grid(False)
+    # plt.show()
+
+    d=plt.figure(figsize=(17, 8))
+    plt.plot(group4.covid_deaths)
+    plt.title('Infection Rate in 40-49 Years')
+    plt.ylabel('Number of Infection')
+    plt.xlabel('Period')
+    plt.grid(False)
+    # plt.show()
+
+    e=plt.figure(figsize=(17, 8))
+    plt.plot(group5.covid_deaths)
+    plt.title('Infection Rate in 50-59 Years')
+    plt.ylabel('Number of Infection')
+    plt.xlabel('Period')
+    plt.grid(False)
+    # plt.show()
+
+    f=plt.figure(figsize=(17, 8))
+    plt.plot(group6.covid_deaths)
+    plt.title('Infection Rate in 60-69 Years')
+    plt.ylabel('Number of Infection')
+    plt.xlabel('Period')
+    plt.grid(False)
+    # plt.show()
+
+    g=plt.figure(figsize=(17, 8))
+    plt.plot(group7.covid_deaths)
+    plt.title('Infection Rate in 70-79 Years')
+    plt.ylabel('Number of Infection')
+    plt.xlabel('Period')
+    plt.grid(False)
+    # plt.show()
+
+    h=plt.figure(figsize=(17, 8))
+    plt.plot(group8.covid_deaths)
+    plt.title('Infection Rate in 80-89 Years')
+    plt.ylabel('Number of Infection')
+    plt.xlabel('Period')
+    plt.grid(False)
+    # plt.show()
+
+    st.pyplot(a)
+    st.pyplot(b)
+    st.pyplot(c)
+    st.pyplot(d)
+    st.pyplot(e)
+    st.pyplot(f)
+    st.pyplot(g)
+    st.pyplot(h)
+
+
+with modelTraining:
+    st.subheader('model training')
+
+    st.write('MODELLING WITH 60-69 years')
+
+    def plot_moving_average(series, window, plot_intervals=False, scale=1.96):
+        rolling_mean = series.rolling(window=window).mean()
+    
+        aa=plt.figure(figsize=(12,8))
+        plt.title('Moving average\n window size = {}'.format(window))
+        plt.plot(rolling_mean, 'g', label='Rolling mean trend')
+        
+        #Plot confidence intervals for smoothed values
+        if plot_intervals:
+            mae = mean_absolute_error(series[window:], rolling_mean[window:])
+            deviation = np.std(series[window:] - rolling_mean[window:])
+            lower_bound = rolling_mean - (mae + scale * deviation)
+            upper_bound = rolling_mean + (mae + scale * deviation)
+            plt.plot(upper_bound, 'r--', label='Upper bound / Lower bound')
+            plt.plot(lower_bound, 'r--')
+                
+        plt.plot(series[window:], label='Actual values')
+        plt.legend(loc='best')
+        plt.grid(True)
+        st.pyplot(aa)
+
+    
+#Smooth by the previous 5 days (by week)
+plot_moving_average(group6.covid_deaths, 5)
+
+#Smooth by the previous month (30 days)
+plot_moving_average(group6.covid_deaths, 30)
+
+#Smooth by previous quarter (90 days)
+plot_moving_average(group6.covid_deaths, 60, plot_intervals=True)
+    
+st.write("Using Exponential smoothening")
+st.markdown('* Determines how fast the weight decreases from previous observations')
+def exponential_smoothing(series, alpha):
+
+    result = [series[0]] # first value is same as series
+    for n in range(1, len(series)):
+        result.append(alpha * series[n] + (1 - alpha) * result[n-1])
+    return result
+  
+def plot_exponential_smoothing(series, alphas):
+ 
+    bb=plt.figure(figsize=(12, 8))
+    for alpha in alphas:
+        plt.plot(exponential_smoothing(series, alpha), label="Alpha {}".format(alpha))
+    plt.plot(series.values, "c", label = "Actual")
+    plt.legend(loc="best")
+    plt.axis('tight')
+    plt.title("Exponential Smoothing")
+    plt.grid(True)
+    st.pyplot(bb)
+
+plot_exponential_smoothing(group6.covid_deaths, [0.05, 0.2])
+
+def double_exponential_smoothing(series, alpha, beta):
+
+    result = [series[0]]
+    for n in range(1, len(series)+1):
+        if n == 1:
+            level, trend = series[0], series[1] - series[0]
+        if n >= len(series): # forecasting
+            value = result[-1]
+        else:
+            value = series[n]
+        last_level, level = level, alpha * value + (1 - alpha) * (level + trend)
+        trend = beta * (level - last_level) + (1 - beta) * trend
+        result.append(level + trend)
+    return result
+
+def plot_double_exponential_smoothing(series, alphas, betas):
+     
+    cc=plt.figure(figsize=(17, 8))
+    for alpha in alphas:
+        for beta in betas:
+            plt.plot(double_exponential_smoothing(series, alpha, beta), label="Alpha {}, beta {}".format(alpha, beta))
+    plt.plot(series.values, label = "Actual")
+    plt.legend(loc="best")
+    plt.axis('tight')
+    plt.title("Double Exponential Smoothing")
+    plt.grid(True)
+    st.pyplot(cc)
+
+    
+plot_double_exponential_smoothing(group6.covid_deaths, alphas=[0.9, 0.02], betas=[0.9, 0.02])
+
+st.subheader("USING SARIMA MODEL")
+def tsplot(y, lags=None, figsize=(12, 7), style='bmh'):
+    
+    if not isinstance(y, pd.Series):
+        y = pd.Series(y)
+        
+    with plt.style.context(style='bmh'):
+        fig = plt.figure(figsize=figsize)
+        layout = (2,2)
+        ts_ax = plt.subplot2grid(layout, (0,0), colspan=2)
+        acf_ax = plt.subplot2grid(layout, (1,0))
+        pacf_ax = plt.subplot2grid(layout, (1,1))
+        
+        y.plot(ax=ts_ax)
+        p_value = sm.tsa.stattools.adfuller(y)[1]
+        ts_ax.set_title('Time Series Analysis Plots\n Dickey-Fuller: p={0:.5f}'.format(p_value))
+        smt.graphics.plot_acf(y, lags=lags, ax=acf_ax)
+        smt.graphics.plot_pacf(y, lags=lags, ax=pacf_ax)
+        plt.tight_layout()
+        st.pyplot(fig)
+tsplot(group6.covid_deaths, lags=30)
+
+# Take the first difference to remove to make the process stationary
+data_diff = group6.covid_deaths - group6.covid_deaths.shift(1)
+
+tsplot(data_diff[1:], lags=30)
+
+import warnings
+warnings.filterwarnings("ignore",category=FutureWarning)
+#Set initial values and some bounds
+ps = range(0, 5)
+d = 1
+qs = range(0, 5)
+Ps = range(0, 5)
+D = 1
+Qs = range(0, 5)
+s = 5
+
+#Create a list with all possible combinations of parameters
+parameters = product(ps, qs, Ps, Qs)
+parameters_list = list(parameters)
+len(parameters_list)
+
+# Train many SARIMA models to find the best set of parameters
+def optimize_SARIMA(parameters_list, d, D, s):
+    """
+        Return dataframe with parameters and corresponding AIC
+        
+        parameters_list - list with (p, q, P, Q) tuples
+        d - integration order
+        D - seasonal integration order
+        s - length of season
+    """
+    
+    results = []
+    best_aic = float('inf')
+    
+    for param in tqdm_notebook(parameters_list):
+        try: model = sm.tsa.statespace.SARIMAX(group6.covid_deaths, order=(param[0], d, param[1]),
+                                               seasonal_order=(param[2], D, param[3], s)).fit(disp=-1)
+        except:
+            continue
+            
+        aic = model.aic
+        
+        #Save best model, AIC and parameters
+        if aic < best_aic:
+            best_model = model
+            best_aic = aic
+            best_param = param
+        results.append([param, model.aic])
+        
+    result_table = pd.DataFrame(results)
+    result_table.columns = ['parameters', 'aic']
+    #Sort in ascending order, lower AIC is better
+    result_table = result_table.sort_values(by='aic', ascending=True).reset_index(drop=True)
+    
+    return result_table
+
+# result_table = optimize_SARIMA(parameters_list, d, D, s)
+
+#Set parameters that give the lowest AIC (Akaike Information Criteria)
+# p, q, P, Q = result_table.parameters[0]
+
+best_model = sm.tsa.statespace.SARIMAX(group6.covid_deaths, order=(1, 1, 1),
+                                       seasonal_order=(1, 1, 1, 7)).fit(disp=-1)
+
+st.write(best_model.summary())
+
+# with covid_relationship:
+st.subheader('Covid Relationship With Other Diseases')
+
+df=pd.read_csv("data/Provisional_COVID-19_Deaths_by_Sex_and_Age.csv")
+df['End Date']=pd.to_datetime(df['End Date'])
+df['Start Date']=pd.to_datetime(df['Start Date'])
+df['Data As Of']=pd.to_datetime(df['Data As Of'])
+for col in df.select_dtypes(include=['datetime64']).columns.tolist():
+    df.style.format({"df[col]":
+            lambda t:t.strftime("%Y-%m-%d")})
+df['Year']=df['Year'].fillna(2020)
+df. drop(["Month","Footnote"], axis=1, inplace=True)
+df=df.dropna()
+Roww, Coll = df.shape
+st.write('dataset 2 shape: ', Roww, Coll)
+df.index=df['End Date']
+
+df=df[df['Age Group'] !='All Ages']
+df.reset_index(drop=True)
+df=df[['Year','Sex','Age Group', 'COVID-19 Deaths', 'Pneumonia Deaths', 'Influenza Deaths']]
+
+jj=sns.lmplot('Pneumonia Deaths','COVID-19 Deaths',data=df,fit_reg=True,scatter_kws={'color':'red','marker':"D","s":20})
+plt.title("Relationship between Covid 19 and Pneumonia")
+st.pyplot(jj)
+
+
+mm=sns.lmplot('Influenza Deaths','COVID-19 Deaths',data=df,fit_reg=True,scatter_kws={'color':'red','marker':"D","s":20})
+plt.title("Relationship between Covid 19 and Influenza")
+st.pyplot(mm)
+
+nn=sns.lmplot('Influenza Deaths','Pneumonia Deaths',data=df,fit_reg=True,scatter_kws={'color':'red','marker':"D","s":20})
+plt.title("Relationship between Pneumonia and Influenza")
+st.pyplot(nn)
+
+df=df[df['Age Group'] !='Under 1 year']
+df=df[df['Age Group'] !='0-17 years']
+df=df[df['Age Group'] !='18-29 years']
+df=df[df['Age Group'] !='30-39 years']
+df=df[df['Age Group'] !='40-49 years']
+
+# Finding the most affected Age Group towards Covid 19
+df.reset_index(drop=True)
+Group_1=df['COVID-19 Deaths'][df['Age Group']=='1-4 years'].to_list()
+Group_2=df['COVID-19 Deaths'][df['Age Group']=='5-14 years'].to_list()
+Group_3=df['COVID-19 Deaths'][df['Age Group']=='15-24 years'].to_list()
+Group_4=df['COVID-19 Deaths'][df['Age Group']=='25-34 years'].to_list()
+Group_5=df['COVID-19 Deaths'][df['Age Group']=='35-44 years'].to_list()
+Group_6=df['COVID-19 Deaths'][df['Age Group']=='45-54 years'].to_list()
+Group_7=df['COVID-19 Deaths'][df['Age Group']=='55-64 years'].to_list()
+Group_8=df['COVID-19 Deaths'][df['Age Group']=='65-74 years'].to_list()
+Group_9=df['COVID-19 Deaths'][df['Age Group']=='75-84 years'].to_list()
+Group_10=df['COVID-19 Deaths'][df['Age Group']=='85 years and over'].to_list()
+
+Infection_rate={'1-4':sum(Group_1),'5-14':sum(Group_2),'15-24':sum(Group_3),'25-34':sum(Group_4),'35-44':sum(Group_5),'45-54':sum(Group_6),'55-64':sum(Group_7),'65-74':sum(Group_8),'75-84':sum(Group_9),'Over 85':sum(Group_10)}
+names=list(Infection_rate.keys())
+values=list(Infection_rate.values())
+
+vv=plt.figure(figsize=(12, 8))
+plt.bar(range(len(Infection_rate)),values,tick_label=names)
+plt.xlabel('Age group{Years}')
+plt.ylabel('Number of Infections')
+plt.title("Covid Infection Rate in various Age group categories")
+# plt.show()
+st.pyplot(vv)
+
+df.to_csv('data/provisional_data.csv',index=False)
+provisional_data=pd.read_csv('data/provisional_data.csv',index_col=['Year'],parse_dates=['Year'])
+provisional_data.rename(columns = {'COVID-19 Deaths':'COVID_Deaths', 'Pneumonia Deaths':'Pneumonia_Deaths','Influenza Deaths':'Influenza_Deaths'}, inplace = True)
+
+# Analysis of infection rate per Gender
+Male_Covid=provisional_data['COVID_Deaths'][provisional_data['Sex']=='Male'].to_list()
+Female_Covid=provisional_data['COVID_Deaths'][provisional_data['Sex']=='Female'].to_list()
+Female_Pneumonia=provisional_data['Pneumonia_Deaths'][provisional_data['Sex']=='Female'].to_list()
+Male_Pneumonia=provisional_data['Pneumonia_Deaths'][provisional_data['Sex']=='Male'].to_list()
+Female_Influenza=provisional_data['Influenza_Deaths'][provisional_data['Sex']=='Female'].to_list()
+Male_Influenza=provisional_data['Influenza_Deaths'][provisional_data['Sex']=='Male'].to_list()
+
+Gender_Infection_rate={'F_Covid':sum(Female_Covid),'M_Covid':sum(Male_Covid),'F_Pneum..':sum(Female_Pneumonia),'M_Pneum..':sum(Male_Pneumonia),'F_Influenza':sum(Female_Influenza),'M_Influenza':sum(Male_Influenza)}
+names=list(Gender_Infection_rate.keys())
+values=list(Gender_Infection_rate.values())
+
+zz=plt.figure()
+plt.bar(range(len(Gender_Infection_rate)),values,tick_label=names,color=['black', 'red', 'green', 'blue', 'cyan','pink'],width=0.3)
+plt.xlabel('Gender')
+plt.ylabel('Number of Infections')
+plt.title("Analysis of infection rate per Gender")
+# plt.show()
+st.pyplot(zz)
+
+# Finding the highest recorded value of detected covid death
+provisional_data["COVID_Deaths"].max()
+
+# Finding the highest recorded value of detected Pneumonia_Deaths
+provisional_data["Pneumonia_Deaths"].max()
+
+# Finding the highest recorded value of detected Influenza_Deaths
+provisional_data["Influenza_Deaths"].max()
+
+st.subheader('Finding Correlation between different diseases')
+
+# The correlation between Covid 19 and Pneumonia
+correlation1=provisional_data['COVID_Deaths']. corr(provisional_data['Pneumonia_Deaths'])
+st.write('The correlation between Covid 19 and Pneumonia',correlation1) 
+
+# The correlation between Covid 19 and Influenza
+correlation2=provisional_data['COVID_Deaths']. corr(provisional_data['Influenza_Deaths'])
+st.write('The correlation between Covid 19 and Influenza',correlation2) 
+
+# The correlation between Pneumonia and Influenza Disease
+correlation3=provisional_data['Pneumonia_Deaths']. corr(provisional_data['Influenza_Deaths'])
+st.write('The correlation between Pneumonia and Influenza Disease',correlation3) 

requirements.txt

@@ -0,0 +1,107 @@
+altair==4.2.0
+argon2-cffi==21.3.0
+argon2-cffi-bindings==21.2.0
+asttokens==2.0.5
+attrs==21.4.0
+backcall==0.2.0
+beautifulsoup4==4.11.1
+bleach==5.0.1
+blinker==1.4
+cachetools==5.2.0
+certifi==2022.6.15
+cffi==1.15.1
+charset-normalizer==2.1.0
+click==8.1.3
+commonmark==0.9.1
+cycler==0.11.0
+debugpy==1.6.0
+decorator==5.1.1
+defusedxml==0.7.1
+entrypoints==0.4
+executing==0.8.3
+fastjsonschema==2.15.3
+fonttools==4.34.2
+gitdb==4.0.9
+GitPython==3.1.27
+idna==3.3
+importlib-metadata==4.12.0
+ipykernel==6.15.0
+ipython==8.4.0
+ipython-genutils==0.2.0
+ipywidgets==7.7.1
+jedi==0.18.1
+Jinja2==3.1.2
+joblib==1.1.0
+jsonschema==4.6.1
+jupyter-client==7.3.4
+jupyter-core==4.10.0
+jupyterlab-pygments==0.2.2
+jupyterlab-widgets==1.1.1
+kiwisolver==1.4.3
+MarkupSafe==2.1.1
+matplotlib==3.5.2
+matplotlib-inline==0.1.3
+mistune==0.8.4
+nbclient==0.6.6
+nbconvert==6.5.0
+nbformat==5.4.0
+nest-asyncio==1.5.5
+notebook==6.4.12
+numpy==1.23.0
+packaging==21.3
+pandas==1.4.3
+pandocfilters==1.5.0
+parso==0.8.3
+patsy==0.5.2
+pexpect==4.8.0
+pickleshare==0.7.5
+Pillow==9.2.0
+prometheus-client==0.14.1
+prompt-toolkit==3.0.30
+protobuf==3.20.1
+psutil==5.9.1
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyarrow==8.0.0
+pycparser==2.21
+pydeck==0.7.1
+Pygments==2.12.0
+Pympler==1.0.1
+pyparsing==3.0.9
+pyrsistent==0.18.1
+python-dateutil==2.8.2
+pytz==2022.1
+pytz-deprecation-shim==0.1.0.post0
+pyzmq==23.2.0
+requests==2.28.1
+rich==12.4.4
+scikit-learn==1.1.1
+scipy==1.8.1
+seaborn==0.11.2
+semver==2.13.0
+Send2Trash==1.8.0
+six==1.16.0
+sklearn==0.0
+smmap==5.0.0
+soupsieve==2.3.2.post1
+stack-data==0.3.0
+statsmodels==0.13.2
+streamlit==1.10.0
+terminado==0.15.0
+threadpoolctl==3.1.0
+tinycss2==1.1.1
+toml==0.10.2
+toolz==0.11.2
+tornado==6.2
+tqdm==4.64.0
+traitlets==5.3.0
+typing-extensions==4.3.0
+tzdata==2022.1
+tzlocal==4.2
+urllib3==1.26.9
+validators==0.20.0
+watchdog==2.1.9
+wcwidth==0.2.5
+webencodings==0.5.1
+widgetsnbextension==3.6.1
+zipp==3.8.0