|
|
import pandas as pd |
|
|
import numpy as np |
|
|
from warnings import warn |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def check_missing(data,output_path=None): |
|
|
""" |
|
|
check the total number & percentage of missing values |
|
|
per variable of a pandas Dataframe |
|
|
""" |
|
|
|
|
|
result = pd.concat([data.isnull().sum(),data.isnull().mean()],axis=1) |
|
|
result = result.rename(index=str,columns={0:'total missing',1:'proportion'}) |
|
|
if output_path is not None: |
|
|
result.to_csv(output_path+'missing.csv') |
|
|
print('result saved at', output_path, 'missing.csv') |
|
|
return result |
|
|
|
|
|
|
|
|
def drop_missing(data,axis=0): |
|
|
""" |
|
|
Listwise deletion: |
|
|
excluding all cases (listwise) that have missing values |
|
|
|
|
|
Parameters |
|
|
---------- |
|
|
axis: drop cases(0)/columns(1),default 0 |
|
|
|
|
|
Returns |
|
|
------- |
|
|
Pandas dataframe with missing cases/columns dropped |
|
|
""" |
|
|
|
|
|
data_copy = data.copy(deep=True) |
|
|
data_copy = data_copy.dropna(axis=axis,inplace=False) |
|
|
return data_copy |
|
|
|
|
|
|
|
|
def add_var_denote_NA(data,NA_col=[]): |
|
|
""" |
|
|
creating an additional variable indicating whether the data |
|
|
was missing for that observation (1) or not (0). |
|
|
""" |
|
|
|
|
|
data_copy = data.copy(deep=True) |
|
|
for i in NA_col: |
|
|
if data_copy[i].isnull().sum()>0: |
|
|
data_copy[i+'_is_NA'] = np.where(data_copy[i].isnull(),1,0) |
|
|
else: |
|
|
warn("Column %s has no missing cases" % i) |
|
|
|
|
|
return data_copy |
|
|
|
|
|
|
|
|
def impute_NA_with_arbitrary(data,impute_value,NA_col=[]): |
|
|
""" |
|
|
replacing NA with arbitrary values. |
|
|
""" |
|
|
|
|
|
data_copy = data.copy(deep=True) |
|
|
for i in NA_col: |
|
|
if data_copy[i].isnull().sum()>0: |
|
|
data_copy[i+'_'+str(impute_value)] = data_copy[i].fillna(impute_value) |
|
|
else: |
|
|
warn("Column %s has no missing cases" % i) |
|
|
return data_copy |
|
|
|
|
|
|
|
|
def impute_NA_with_avg(data,strategy='mean',NA_col=[]): |
|
|
""" |
|
|
replacing the NA with mean/median/most frequent values of that variable. |
|
|
Note it should only be performed over training set and then propagated to test set. |
|
|
""" |
|
|
|
|
|
data_copy = data.copy(deep=True) |
|
|
for i in NA_col: |
|
|
if data_copy[i].isnull().sum()>0: |
|
|
if strategy=='mean': |
|
|
data_copy[i+'_impute_mean'] = data_copy[i].fillna(data[i].mean()) |
|
|
elif strategy=='median': |
|
|
data_copy[i+'_impute_median'] = data_copy[i].fillna(data[i].median()) |
|
|
elif strategy=='mode': |
|
|
data_copy[i+'_impute_mode'] = data_copy[i].fillna(data[i].mode()[0]) |
|
|
else: |
|
|
warn("Column %s has no missing" % i) |
|
|
return data_copy |
|
|
|
|
|
|
|
|
def impute_NA_with_end_of_distribution(data,NA_col=[]): |
|
|
""" |
|
|
replacing the NA by values that are at the far end of the distribution of that variable |
|
|
calculated by mean + 3*std |
|
|
""" |
|
|
|
|
|
data_copy = data.copy(deep=True) |
|
|
for i in NA_col: |
|
|
if data_copy[i].isnull().sum()>0: |
|
|
data_copy[i+'_impute_end_of_distri'] = data_copy[i].fillna(data[i].mean()+3*data[i].std()) |
|
|
else: |
|
|
warn("Column %s has no missing" % i) |
|
|
return data_copy |
|
|
|
|
|
|
|
|
def impute_NA_with_random(data,NA_col=[],random_state=0): |
|
|
""" |
|
|
replacing the NA with random sampling from the pool of available observations of the variable |
|
|
""" |
|
|
|
|
|
data_copy = data.copy(deep=True) |
|
|
for i in NA_col: |
|
|
if data_copy[i].isnull().sum()>0: |
|
|
data_copy[i+'_random'] = data_copy[i] |
|
|
|
|
|
random_sample = data_copy[i].dropna().sample(data_copy[i].isnull().sum(), random_state=random_state) |
|
|
random_sample.index = data_copy[data_copy[i].isnull()].index |
|
|
data_copy.loc[data_copy[i].isnull(), str(i)+'_random'] = random_sample |
|
|
else: |
|
|
warn("Column %s has no missing" % i) |
|
|
return data_copy |
|
|
|
