Supplement 3: Hierarchical Clustering Model¶

# import libraries
from models import select_attributes
from models import cut_hierarchy_columns

import pandas as pd
import numpy as np
import timeit
import pickle

import matplotlib as mpl
import scipy.cluster.hierarchy as shc
import matplotlib.pyplot as plt

from scipy.cluster.hierarchy import fcluster

from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import roc_auc_score
import numpy as np
from joblib import Parallel, delayed


import matplotlib.pyplot as plt

import matplotlib as mpl
mpl.rcParams.update(mpl.rcParamsDefault)

start = timeit.default_timer()

# list of values for inverse of regularization strength
c_list = [0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1,1.5,2]

split_ids = []
cluster_n = []
cs = []
source = []
AUCs = []
prec = []
rec =  []
vars_cnt = []
vars_lists = []
ys_test = []
ys_pred = []

for i in range (0,30): 
    # read one of the 30 split data
    tr_path = "../data/30_splits_data/binary-transformed_tr_"+str(i)+".csv"
    ts_path = "../data/30_splits_data/binary-transformed_ts_"+str(i)+".csv"
     
    train = pd.read_csv(tr_path)
    test = pd.read_csv(ts_path)
    
    # Select the list of original predictors
    symptoms = train.columns.tolist()
    symptoms.remove('TestPositive')
    
    # For each cutoff point run tehe lasso model
    for cc in range(2,len(symptoms)):
        
        # create the hierarchical clusters based on the selected cutoff point
        XT, Xt, yT, yt = cut_hierarchy_columns(train, test, cc)
        
        # build the model for each value fo the inverse of regularization strength
        for c in c_list:
            auc, recall, precision, valid_cols, y_test, y_pred = select_attributes(XT, yT, Xt, yt, c)
            
            # add results to the list
            split_ids.append(i)
            cluster_n.append(cc)
            cs.append(c)
            source.append('cut_hierarchy')
            AUCs.append(auc)
            prec.append(precision)
            rec.append(recall)
            vars_cnt.append(len(valid_cols))
            vars_lists.append(valid_cols)
            if(y_test is not np.nan):
                ys_test.append(yt.values.tolist())
                ys_pred.append(y_pred.tolist())
            else:
                ys_test.append(np.nan)
                ys_pred.append(np.nan)
                

            print(f'ID {i}, number of clusters={cc:02}, C={c:.2f}, AUC={auc:.5f}, Precision={precision:.5f}, Recall={recall:.5f}, cls# {len(valid_cols)}') 

        
stop = timeit.default_timer()
print('Time: ', stop - start) 

# create dataframe for all results
ff = pd.DataFrame({'split_ids'    : split_ids,
                   'cluster_n'    : cluster_n,
                   'cs'           : cs,
                   'source'       : source,
                   'AUCs'         : AUCs,
                   'prec'         : prec,
                   'rec'          : rec,
                   'vars_cnt'     : vars_cnt,
                   'vars_lists'   : vars_lists,
                   'y_test'       : ys_test,
                   'y_pred'       : ys_pred
                  })

# identify the list of unique selected predictors
unq_var = ff['vars_lists'].values.tolist()
for i in range(0, len(unq_var)):
    unq_var[i] = [sub.replace(':', ',') for sub in unq_var[i]]
    
sympt_lists = []
for i in range(0, len(unq_var)):
    l = ",".join(unq_var[i])
    l2 = list(set(l.split(',')))
    sympt_lists.append(l2)

# Add the list of unique predictors and its count to the dataframe
ff['sympt_lists'] = sympt_lists
ff['sympt_cnt'] = ff['sympt_lists'].apply(lambda x :len(x))

# save the results
ff.to_csv('../data/results/hierarchical_model.csv', index=False)

table = pd.pivot_table(ff, values=['AUCs', 'vars_cnt', 'sympt_cnt']
                       , index=['cluster_n', 'cs']
                       , aggfunc=np.mean).round(decimals=4)


table['sympt_cnt'] = table['sympt_cnt'].round().astype(int)
table['vars_cnt'] = table['vars_cnt'].round().astype(int)

# read data
path = "../data/preprocessed.csv"
df   = pd.read_csv(path)
df.columns  = [s.replace('_',' ') for s in df.columns]

metric = "cityblock" 
method = "complete"

X, Xt, y, yt = cut_hierarchy_columns(df, df, n_cluster)

Xy_pickle = 'BinaryDataX_hierarchical.p'
pickle.dump({'X': X, 'y': y}, open(Xy_pickle, 'wb'))

def do_validation(fold, tr, te, c= _C):
    data = pickle.load(open(Xy_pickle, 'rb'))
    X, y = data['X'], data['y']
    
    X_tr, X_te = X.iloc[tr], X.iloc[te]
    y_tr, y_te = y.iloc[tr].values.ravel(), y.iloc[te].values.ravel()
    
    print(f'fold {fold:02}')
        
    regressor = LogisticRegression(penalty='l1', solver='saga', C=c, n_jobs=-1, max_iter=5000*2)
    regressor.fit(X_tr, y_tr)
    
    y_pr = regressor.predict_proba(X_te)[:,1]
    
    score = roc_auc_score(y_te, y_pr)
    print(f'fold {fold:02}, score={score:.5f}')
    return score, regressor.coef_[0]
    

skf = StratifiedKFold(n_splits=24, shuffle=True, random_state=37)

outputs = Parallel(n_jobs=-1)(delayed(do_validation)(fold, tr, te, _C) 
                              for fold, (tr, te) in enumerate(skf.split(X, y)))

scores = pd.Series([score for score, _ in outputs])
coefs = pd.DataFrame([coef for _, coef in outputs], columns=X.columns)

def get_profile(df, col):
    s = df[col]
    
    s_pos = s[s > 0]
    s_neg = s[s < 0]
    
    n = df.shape[0]
    p_pos = len(s_pos) / n
    p_neg = len(s_neg) / n
    
    return {
        'field': col,
        'n_pos': len(s_pos),
        'n_neg': len(s_neg),
        'pct_pos': p_pos, 
        'pct_neg': p_neg, 
        'is_valid': 1 if p_pos >= 0.95 or p_neg >= 0.95 else 0
    }

valid_coefs = pd.DataFrame([get_profile(coefs, c) for c in coefs.columns]).sort_values(['is_valid'], ascending=False)
valid_coefs = valid_coefs[valid_coefs.is_valid == 1]

y_pred = regressor.predict_proba(X[valid_cols])[:,1]

t = X[valid_cols].copy()
t['y_pred'] = y_pred
t['y_actual'] = y
t.to_csv("../data/results/prediction_hierarchical_model.csv", index=False)

c = pd.Series(regressor.coef_[0], valid_cols)

plt.style.use('ggplot')

i = pd.Series([regressor.intercept_[0]], index=['intercept'])
s = pd.concat([c[c > 0], c[c < 0]]).sort_index()
s = pd.concat([i, s])
color = ['r' if v > 0 else 'b' for v in s]

ax = s.plot(kind='bar', color=color, figsize=(20, 4))
_ = ax.set_title(f'Logistic Regression, validated auc={scores.mean():.5f}')

s_odds = np.exp(s)
color = ['r' if v > 1 else 'b' for v in s_odds]

ax = s_odds.plot(kind='bar', color=color, figsize=(20, 4))
_ = ax.set_title(f'Logistic Regression, coefficient odds')