File size: 3,640 Bytes
000de75 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 | """
Derive features from comorbidities dataset for 2 models:
Parallel model 1: uses both hospital and community exacerbation events
Parallel model 2: uses only hospital exacerbation events
"""
import numpy as np
import pandas as pd
import sys
import os
import yaml
import model_h
with open("./training/config.yaml", "r") as config:
config = yaml.safe_load(config)
# Specify which model to generate features for
model_type = config["model_settings"]["model_type"]
# Setup log file
log = open("./training/logging/process_comorbidities_" + model_type + ".log", "w")
sys.stdout = log
# Dataset to process - set through config file
data_to_process = config["model_settings"]["data_to_process"]
# Load cohort data
if data_to_process == "forward_val":
exac_data = pd.read_pickle("./data/patient_labels_forward_val_hosp_comm.pkl")
patient_details = pd.read_pickle("./data/patient_details_forward_val.pkl")
else:
exac_data = pd.read_pickle("./data/patient_labels_" + model_type + ".pkl")
patient_details = pd.read_pickle("./data/patient_details.pkl")
exac_data = exac_data[["StudyId", "IndexDate"]]
patient_details = exac_data.merge(
patient_details[["StudyId", "PatientId"]],
on="StudyId",
how="left",
)
comorbidities = pd.read_csv(
config["inputs"]["raw_data_paths"]["comorbidities"], delimiter="|"
)
comorbidities = patient_details.merge(comorbidities, on="PatientId", how="left")
# Only keep records submitted before index date
comorbidities["Created"] = pd.to_datetime(comorbidities["Created"], utc=True)
comorbidities["TimeSinceSubmission"] = (
comorbidities["IndexDate"] - comorbidities["Created"]
).dt.days
comorbidities = comorbidities[comorbidities["TimeSinceSubmission"] > 0]
# If multiple records submitted for same patient keep the most recent record (in relation
# to index date)
comorbidities = comorbidities.sort_values(
by=["StudyId", "IndexDate", "TimeSinceSubmission"]
)
comorbidities = comorbidities.drop_duplicates(
subset=["StudyId", "IndexDate"], keep="first"
)
# Get list of comorbidities captured in the service
comorbidity_list = list(comorbidities)
comorbidity_list = [
e
for e in comorbidity_list
if e
not in ("PatientId", "Id", "StudyId", "IndexDate", "TimeSinceSubmission", "Created")
]
# Map True/False values to integers
bool_mapping = {True: 1, False: 0}
comorbidities[comorbidity_list] = (
comorbidities[comorbidity_list].replace(bool_mapping).fillna(0)
)
# Get comorbidity counts for each patient
comorbidities["Comorbidities"] = comorbidities[comorbidity_list].sum(axis=1)
# Drop comorbidities columns from train data but retain AsthmaOverlap
comorbidity_list.remove("AsthmaOverlap")
comorbidities = comorbidities.drop(columns=comorbidity_list)
comorbidities = comorbidities.drop(columns=["Id", "Created", "TimeSinceSubmission"])
# Bin number of comorbidities
comorb_bins = [0, 1, 3, np.inf]
comorb_labels = ["No comorbidities", "1-2", "3+"]
comorbidities["Comorbidities"] = model_h.bin_numeric_column(
col=comorbidities["Comorbidities"], bins=comorb_bins, labels=comorb_labels
)
comorbidities = comorbidities.drop(columns=["PatientId"])
# Save data
os.makedirs(config["outputs"]["processed_data_dir"], exist_ok=True)
if data_to_process == "forward_val":
comorbidities.to_pickle(
os.path.join(
config["outputs"]["processed_data_dir"],
"comorbidities_forward_val_" + model_type + ".pkl",
)
)
else:
comorbidities.to_pickle(
os.path.join(
config["outputs"]["processed_data_dir"],
"comorbidities_" + model_type + ".pkl",
)
)
|