\documentclass[AMA,STIX1COL]{WileyNJD-v2}
\usepackage{moreverb}
\usepackage{xr}
\externaldocument{CARS}
%\usepackage{doublespace}
%\usepackage[style=authoryear,backend=biber]{biblatex} 
%\addbibresource{survival.bib} 
%\usepackage{graphics}
%\usepackage{natbib}
%\usepackage{sectsty}
%\usepackage[section]{placeins}

\DeclareMathOperator*{\argmin}{arg\,min}
\makeatletter
\DeclareRobustCommand{\rvdots}{%
	\vbox{
		\baselineskip4\p@\lineskiplimit\z@
		\kern-\p@
		\hbox{.}\hbox{.}\hbox{.}
}}
\makeatother

\newcommand\BibTeX{{\rmfamily B\kern-.05em \textsc{i\kern-.025em b}\kern-.08em
T\kern-.1667em\lower.7ex\hbox{E}\kern-.125emX}}

\articletype{Research Article}%

\received{07, 03, 2018}
\revised{19, 12, 2018}
\accepted{<day> <Month>, <year>}

\input{abbreviations}

\begin{document}

\appendix

\section{\textbf {Proof of consistency of CAR survival scores}} \label{proofOverview}

In this section we prove that the estimator $\hat{\btheta}$ of the CAR survival score defined in Equation (8) of the manuscript is consistent for $\btheta$. We first define
\begin{align}
& \bar{Y}_w = \frac{1}{n} \sum_{i=1}^{n} w_i \log (\tilde{T}_{i}) \, ,\\
& S_{Y;w}^2 = \frac{1}{n}\sum_{i=1}^n w_{i} \left( \log (\tilde{T} _{i} ) - \bar{Y}_w \right)^2 \, , \label{syw2}\\
& \bar{X}_j = \frac{1}{n} \sum_{i=1}^{n} X_{i,j} \, ,\\
& S_{X_j}^2 = \frac{1}{n-1}\sum_{i=1}^n \left( X_{i,j} - \bar{X}_j \right)^2 \, ,\\
& S_{X_j,Y;w} = \frac{1}{n} \sum_{i=1}^n w_{i} (X_{ij} - \bar{X}_j) \left( \log (\tilde{T}_{i} ) - \bar{Y}_w \right) \, , \label{sxyw}\\
& \bR_{\bX, Y} = \left(
\frac{S_{X_j,Y;w}}
{\sqrt{S_{X_j}^2} \sqrt{S_{Y;w}^2}}
\right)_{j = 1, \dots , d} \, ,
 \label{CARSdefLast}
\end{align}
where $\tilde{T}_i$ and $X_{ij}$, $i=1,\ldots , n$, $j=1,\ldots , d$, refer to the sample variables of the observed response and the covariates, respectively. The proof is partitioned into four parts: First the consistency of the IPC weighted mean $\bar{Y}_w$ is shown (Section \ref{proofWeightedMean}). Second the consistency of the weighted sample variance $S_{Y;w}^2$ of the response is shown (Section \ref{proofWeightedVar}). Third the consistency of the weighted sample covariance $S_{X_j,Y;w}$ of the response and the covariates is shown (Section \ref{proofWeightedCoVar}). In the last part the consistency of $\hat{\btheta}$ is derived by combining all previous parts (Section \ref{proofCARS}).

\subsection{\textbf{Consistency of the IPC weighted mean}} \label{proofWeightedMean}

To show the consistency of $\bar{Y}_w$ it is sufficient to embed this estimator in the framework of unbiased estimation equations. \citep{huberEstEq} Generally, the estimating equation
\begin{align} \label{unbEstEqWeight}
& \frac{1}{n} \sum_{i=1}^{n} \psi(\tilde{Y}_i, w_i , \Theta) = 0
\end{align}
for some parameter $\Theta$ (here, $\Theta \equiv \mu_Y = E(Y)$) is {\em unbiased} if
\begin{align} \label{expectZero}
& E \left( \psi(\tilde{Y}_i, w_i , \Theta) \right) = 0 \ \ \ \forall \ i=1, \ldots, n \, ,
\end{align}
where $\psi$ is a real-valued function of $\tilde{Y}_i$, $w_i$ and $\Theta$. The IPC weights $w_i$ are defined as in Equation (9) of the manuscript, i.e.,
\begin{align} %\label{unbEstEqWeight}
& w_i = \frac{I(C_i \geq T_i)}{\hat{G}_n(\log(\tilde{T}_i))} \, ,
\end{align}
where $\hat{G}_n$ is the Kaplan-Meier estimator of the survival function of the censoring distribution. Note that the Kaplan-Meier estimator is consistent for the true survival function $G$ and can therefore be replaced by $G$ in asymptotic analysis. (This will be done in the following, see also Carroll et al.\@ (2006). \citep{carrolletal}) Under some regularity conditions \citep{huberEstEq} (e.g.\@ measureability, continuity and uniqueness of solutions) the estimator $\hat{\Theta}$ solving (\ref{unbEstEqWeight}) is consistent for $\Theta$ if Equation (\ref{expectZero}) holds. \\

As stated in Section \ref{sampleLevel} of the manuscript, the above scenario can be extended by allowing the censoring survival function to depend on the covariates. For this it needs to be assumed that $\hat{G}_n(\log(\tilde{T}_i)|{\boldsymbol x}_i$) is a consistent estimator of the true conditional censoring distribution $G(\log(\tilde{T}_i)|{\boldsymbol x}_i)$, where ${\boldsymbol x}_i$ is a vector containing the sample variables $X_{i1}, \ldots , X_{id}$. In practice, $\hat{G}_n$ is usually obtained by fitting a multivariable time-to-event model to the survival times with event indicator $1-\Delta$. Correct specification of this model requires, in particular, the selection of the correct set of variables to include in the model. It should be noted that the covariates to be included in the censoring model do not necessarily have to be the same as the covariates considered for CARS score estimation. In IPC weighting, which bears similarities to propensity score weighting in retrospective epidemiological studies, \citep{propensity} the covariates for the censoring model are often selected a priori based on context knowledge (e.g., demographic covariates and context-specific clinical risk factors). If an a priori choice is not possible and if the number of candidate variables for the censoring model is large, data-driven variable selection for the censoring model (which is a largely unexplored topic in survival analysis) may be done along similar lines as the ones proposed by Brookhart et al.\@ (2006) and Schneeweiss (2009) \citep{propensity, schneeweiss} for propensity score modeling. \\

From now on assume that $G(\cdot |{\boldsymbol x})$ is known and that $G(\cdot |{\boldsymbol x}) > \nu > 0$ for a small real number $\nu$. Next consider the estimation function
\begin{align} \label{unbiasedPsi}
\psi(\tilde{Y}_i, w_i , \mu_{\log(T)}) = w_i \log (\tilde{T}_{i}) -  \mu_{\log(T)}
\end{align}
for the weighted mean $\bar{Y}_w$. By definition, one obtains
\begin{align} 
& E\left( w_i \log (\tilde{T}_{i}) \right) = \int \int\limits_{-\infty}^{\infty} \int\limits_{-\infty}^{\infty} \frac{I\left( \log(C_i) \geq \log(T_i)\right) }{G(\log(\tilde{T}_i)|{\boldsymbol x}_i)} \log(\tilde{T}_i) f_{\log(T)|{\boldsymbol x}}\left( \log(T)\right) \nonumber \\ 
& f_{\log(C)|{\boldsymbol x}} \left( \log(C)\right) \,f_{\boldsymbol x} ({\boldsymbol x}) \ d\log(T) \ d\log(C) \ d{\boldsymbol x}  \, ,
\end{align}
where the first integral is over the covariates ${\boldsymbol x} \in \mathbb{R}^d$, where $f_{\log(T)|{\boldsymbol x}}$ and $f_{\log(C)|{\boldsymbol x}}$ denote the conditional probability density functions of $\log (T)|{\boldsymbol x}$ and $\log(C) | {\boldsymbol x}$, respectively, and where $f_{\boldsymbol{x}}$ is the probability density function of the covariates ${\boldsymbol x}$. The following steps show that (\ref{unbiasedPsi}) defines an unbiased estimation equation for the parameter $\mu_Y \equiv\mu_{\log(T)}$:

\begin{align} \label{unbiasedMean} 
& E\left( w_i \log (\tilde{T}_{i}) \right) = \int \int\limits_{-\infty}^{\infty} \frac{1}{G(\log(T_i)|{\boldsymbol x}_i)} \log(T_i) f_{\log(T)|{\boldsymbol x}}\left( \log(T)\right) \nonumber \\ & \int\limits_{\log(T_i)}^{\infty} f_{\log(C)|{\boldsymbol x}} \left( \log(C)\right) \ d\log(C) f_{\boldsymbol x} ({\boldsymbol x}) \ d\log(T) \ d{\boldsymbol x}\\
& = \int \int\limits_{-\infty}^{\infty} \frac{G( \log(T_i)|{\boldsymbol x}_i ) }{G(\log(T_i)|{\boldsymbol x}_i)} \log(T_i) f_{\log(T)|{\boldsymbol x}}\left( \log(T)\right) \, f_{\boldsymbol x} ({\boldsymbol x}) \ d\log(T) \ d{\boldsymbol x} \\
& = \mu_{\log(T)} \, .
\end{align}

\subsection{\textbf{Consistency of the IPC weighted variance}} \label{proofWeightedVar}

To prove the consistency of the IPC weighted variance $S_{Y;w}^2$ for $\sigma_Y^2 = \sigma_{\log(T)}^2$, we consider the estimation function
\begin{align} \label{unbiasedEstVarY}
& \psi\left( \tilde{Y}_i, w_i , \sigma_{\log(T)}^2\right)  = w_i \left( \log (\tilde{T}_{i} ) - \mu_{\log(T)} \right)^2 - \sigma_{\log(T)}^2 \, ,
\end{align}
where, compared to the definition of $S_{Y;w}^2$ in (\ref{syw2}), the weighted mean $\bar{Y}_w$ has been replaced by the true expectation $\mu_{\log(T)}$.

The following steps show that (\ref{unbiasedEstVarY}) defines an unbiased estimation equation for the parameter $\sigma_Y^2 \equiv\sigma_{\log(T)}^2$:

\begin{align} \label{unbiasedVarY} 
& E\left( w_i  \left( \log (\tilde{T}_{i} ) - \mu_{\log(T)} \right)^2 \right) = \int \int\limits_{-\infty}^{\infty} \frac{1}{G(\log(T_i)|{\boldsymbol x}_i)} \left( \log ({T}_{i} ) - \mu_{\log(T)} \right)^2 f_{\log(T)|{\boldsymbol x}}\left( \log(T)\right) \nonumber \\ & \int\limits_{\log(T_i)}^{\infty} f_{\log(C)|{\boldsymbol x}}\left( \log(C)\right) \ d\log(C) \ f_{\boldsymbol x} ({\boldsymbol x}) \ d\log(T) \ d{\boldsymbol x} \\
& = \int \int\limits_{-\infty}^{\infty} \frac{G( \log(T_i)|{\boldsymbol x}_i ) }{G(\log(T_i)|{\boldsymbol x}_i)} \left( \log ({T}_{i} ) - \mu_{\log(T)} \right)^2 f_{\log(T)|{\boldsymbol x}}\left( \log(T)\right) \, f_{\boldsymbol x} ({\boldsymbol x}) \ d\log(T) \ d{\boldsymbol x} \\
& = \sigma_{\log(T)}^2 \, .
\end{align}

As shown, for example, in Carroll et al.\@ (2006), \citep{carrolletal} the estimator of $\sigma_{\log(T)}^2$ solving $1/n \sum_i \psi\big( \tilde{Y}_i, w_i , \sigma_{\log(T)}^2\big) = 0$ remains a consistent estimator if the true expectation $\mu_{\log(T)}$ in (\ref{unbiasedEstVarY}) is replaced by
the weighted mean $\bar{Y}_w$ in the definition of $S_{Y;w}^2$. This is because $\bar{Y}_w$ is a consistent estimator of  $\mu_{\log(T)}$, as shown above.

\subsection{\textbf{Consistency of the IPC weighted covariance}} \label{proofWeightedCoVar}

Following the same strategy as in the previous Section \ref{proofWeightedVar}, we consider the following estimation function
for the covariance $\sigma_{X_j,Y} \equiv \sigma_{X_j,\log (T)}$ between covariate $X_j$, $j\in\{1,\ldots , d\}$, and $Y$:
\begin{align} \label{unbiasedEstCovXY}
& \psi\left( \tilde{Y}_i, w_i , \sigma_{X_j, \log(T)}\right)  = w_{i} (X_{ij} - \mu_j) \left( \log (\tilde{T}_{i} ) - \mu_{\log(T)} \right) - \sigma_{X_j, \log(T)} \, ,
\end{align}
where, compared to the definition of $S_{X_j,Y;w}$ in (\ref{sxyw}), the weighted mean $\bar{Y}_w$ and the mean $\bar{X}_j$ have been replaced by the true expectations $\mu_{\log(T)}$ and $\mu_j$, respectively.

The following steps show that (\ref{unbiasedEstCovXY}) defines an unbiased estimation equation for the parameter $\sigma_{X_j, \log (T)}$:
\begin{align} \label{unbiasedCov} 
& E\left( w_{i} (X_{ij} - \mu_j) \left( \log (\tilde{T}_{i} )- \mu_{\log(T)} \right)\right) = \int\int\limits_{-\infty}^{\infty} \frac{1}{G(\log(T_i)|{\boldsymbol x}_i)}
(X_{ij} - \mu_j) \left( \log ({T}_{i} ) - \mu_{\log(T)}\right)
f_{\log(T)|{\boldsymbol x}} \left( \log(T)\right) \nonumber \\
& \int\limits_{\log(T_i)}^{\infty} 
f_{\log(C)|{\boldsymbol x}}\left( \log(C)\right) \ d\log(C) \ f_{\boldsymbol x} ({\boldsymbol x}) \ d\log(T) \ d{\boldsymbol x} \\
& = \int\int\limits_{-\infty}^{\infty} \frac{G( \log(T_i)|{\boldsymbol x}_i ) }{G(\log(T_i)|{\boldsymbol x}_i)}
(X_{ij} - \mu_j) \left( \log ({T}_{i} ) - \mu_{\log(T)}\right)
f_{\log(T)|{\boldsymbol x}}\left( \log(T)\right) f_{\boldsymbol x} ({\boldsymbol x}) \ d\log(T) \ d{\boldsymbol x} \\
& = \sigma_{X_j, \log(T)}^2 \, .
\end{align}

Analogous to the previous subsection, the estimator of $\sigma_{X_j, \log(T)}$ solving $1/n \sum_i \psi\big( \tilde{Y}_i, w_i , \sigma_{X_j, \log(T)}\big) = 0$ remains a consistent estimator if the true expectations $\mu_j$ and $\mu_{\log(T)}$ in (\ref{unbiasedEstCovXY}) are replaced by
the mean $\bar{X}_j$ and the weighted mean $\bar{Y}_w$, respectively, in the definition of $S_{X_j,Y;w}$. This is because $\bar{X}_j$ and $\bar{Y}_w$ are consistent estimators of $\mu_j$ and  $\mu_{\log(T)}$, respectively.

\subsection{\textbf{Combination of results}} \label{proofCARS}

In this subsection all previous consistency proofs for the weighted mean, weighted variance and weighted covariance are combined to prove the consistency of the CARS score estimator $\hat{\btheta} = \bR_{\text{shrink}}^{-1/2} \, \bR_{\bX, Y}$. First consider the quantity $\bR_{\bX, Y}$. Denoting by~$\boldsymbol{P}_{\boldsymbol{X}, Y}$ the vector of true pairwise correlations between the covariates and the response, it follows from the previous subsections that

\begin{align}
& \underset{n \rightarrow \infty}{\lim} \boldsymbol{R}_{\boldsymbol{X},Y} = \underset{n \rightarrow \infty}{\lim} \left( \frac{s_{Y, X_j; w}} { s_{X_j} s_{Y; w} } \right)_{j=1, \ldots, d} = \boldsymbol{P}_{\boldsymbol{X}, Y} \,  .
\end{align}

Next consider the shrinkage estimator $\boldsymbol{R}_{\text{Shrink}}$ \citep{shrinkCovar} of the correlations between the covariates $\boldsymbol{R}_{\boldsymbol{X}}$, which is defined by

\begin{align}
& \boldsymbol{R}_{\text{Shrink}} = \lambda \boldsymbol{I}_d + (1 - \lambda) \boldsymbol{R}_{\boldsymbol{X}} \, ,
\end{align}
where the shrinkage parameter $\lambda$ is estimated by
\begin{align}
& \hat{\lambda} = \frac{\sum_{j \neq k} \widehat{Var} (\hat{r}_{j,k})} {\sum_{j \neq k} \hat{r}_{j,k}^2}
\end{align}
and
\begin{align}
& \hat{r}_{j,k} = \dfrac{\frac{1} {n-1} \sum_{i=1}^n \left( x_{i,j} - \bar{x}_j \right) \left( x_{i,k} - \bar{x}_k \right)} {\sqrt{\frac{1} {n-1} \sum_{i=1}^n \left( x_{i,j} - \bar{x}_j \right)^2} \sqrt{\frac{1} {n-1} \sum_{i=1}^n \left( x_{i,k} - \bar{x}_k \right)^2}} 
\end{align}

is the sample correlation between the $j$-th and $k$-th covariate. In the limit, the estimator $\hat{\lambda}$ converges to zero, as

\begin{align}
& \lim_{n \rightarrow \infty} \hat{\lambda} = \frac{\lim_{n \rightarrow \infty} \sum_{j \neq k} \widehat{Var} (\hat{r}_{j,k})} {\lim_{n \rightarrow \infty} \sum_{j \neq k} \hat{r}_{j,k}^2} \\
& = \frac{\sum_{j \neq k} \lim_{n \rightarrow \infty}\widehat{Var} (\hat{r}_{j,k})} {\sum_{j \neq k} \lim_{n \rightarrow \infty} \hat{r}_{j,k}^2} \\
& = \frac{\sum_{j \neq k} 0} {\sum_{j \neq k} r_{j,k}^2} = 0 \, .
\end{align}

As a consequence, $\boldsymbol{R}_{\text{Shrink}}$  approaches the estimator $\boldsymbol{R}_{\boldsymbol{X}}$ in the limit, which is itself a consistent estimator of the covariates' correlation matrix $\boldsymbol{P}_{\boldsymbol{X}}$. Combining all previous results, and applying the continuous mapping theorem as well as Slutsky's theorem, proves the consistency of the CARS score estimator $\hat{\btheta} = \bR_{\text{shrink}}^{-1/2} \bR_{\bX, Y}$ for the population level CARS score $\btheta = \boldsymbol{P}_{\boldsymbol{X}}^{-1/2} \boldsymbol{P}_{\boldsymbol{X}, Y}$.

\clearpage

\section{\textbf {Additional results obtained from the simulation study}} \label{simCARS}

\subsection{\textbf {CARS simulation with low absolute covariate correlations}} \label{simCARSlowCor}

\begin{figure}[!htbp]
	\centering
	\includegraphics[width=\textwidth]{carPRAUC_noVarSigXexpVar.pdf}
	\caption{Results of the simulation study. PR-AUC of CARS, Cox and Cox $L_1$ scores stratified by relative number of relevant variables (relVar) and explained variance (expVar) with low absolute covariate correlations $(\rho=\pm 0.25)$ and censoring rate of $25\%$. Each boxplot summarizes the results of $2700$ simulation runs (3 sample sizes x 3 number of covariates x 300 repetitions).}
	\label{simCARSlowCorPlotNoVarExpVar}
\end{figure}

\begin{figure}
	\centering
	\includegraphics[scale = 0.3]{carRankCor.pdf}
	\caption{Results of the simulation study, scenario with low absolute correlations ($\rho = \pm 0.25$).
		The boxplots visualize the rank correlations of the estimated and the true covariate orderings, as obtained from variable selection by CARS and Cox scores. The censoring rate was equal to $25\%$. Each boxplot shows the results of $24300$ simulation runs (3 explained variance ratios x 3 signal to noise ratios x 3 sample sizes x 3 number of covariates x 300 repetitions).}
	\label{figCor}
\end{figure}

\begin{figure}[htbp]
	\centering
	\includegraphics[scale = 0.3]{carPRAUC.pdf}
	\caption{Results of the simulation study, scenario with low absolute correlations ($\rho = \pm 0.25$). The boxplots visualize the PR-AUC values obtained from variable selection by CARS scores, Cox scores and $L_1$-penalized Cox regression. The censoring rate was equal to $25\%$. The average prevalance of the positive class (computed from all simulations) is displayed by the dashed line. Note that the boxplots contain the PR-AUC values corresponding to all three rates of influential covariates ($1\%$, $5\%$, $10\%$). Each boxplot shows the results of $24300$ simulation runs (3 explained variance ratios x 3 signal to noise ratios x 3 sample sizes~x 3 number of covariates x 300 repetitions).}
	\label{figPRAUC}
\end{figure}

\clearpage

\subsection{\textbf {CARS simulation with high absolute covariate correlations}} \label{simCARShighCor}

\begin{figure}[!htbp]
	\centering
	\includegraphics[width=\textwidth]{carPRAUC_noVarSigXexpVar_highCor.pdf}
	\caption{Results of the simulation study. PR-AUC of CARS, Cox and Cox $L_1$ scores stratified by relative number of relevant variables (relVar) and explained variance (expVar) with high absolute covariate correlations $\rho=\pm 0.75$ and censoring rate of~$25\%$. Each boxplot shows the results of $2700$ simulation runs (3 sample sizes x 3 number of covariates x 300 repetitions).}
	\label{simCARShighCorPlotNoVarExpVar}
\end{figure}

\begin{figure}[!t]
	\centering
	\includegraphics[scale=0.3]{carRankCor_highCor.pdf}
	\caption{Results of the simulation study, scenario with high absolute correlations ($\rho = \pm 0.75$).
		The boxplots visualize the rank correlations of the estimated and the true covariate orderings, as obtained from variable selection by CARS and Cox scores. The censoring rate was equal to $25\%$. Each boxplot shows the results of $24300$ simulation runs (3 explained variance ratios x \protect\linebreak 3 signal to noise ratios x 3 sample sizes x 3 number of covariates x 300 repetitions).}
	\label{figCor-highCor}
\end{figure}

\begin{figure}[htbp]
	\centering
	\includegraphics[scale=0.3]{carPRAUC_highCor.pdf}
	\caption{Results of the simulation study, scenario with high absolute correlations ($\rho = \pm 0.75$). The boxplots visualize the PR-AUC values obtained from variable selection by CARS scores, Cox scores and $L_1$-penalized Cox regression. The censoring rate was equal to $25\%$. The average prevalance of the positive class (computed from all simulations) is displayed by the dashed line. Note that the boxplots contain the PR-AUC values corresponding to all three rates of influential covariates ($1\%$, $5\%$, $10\%$). Each boxplot shows the results of $24300$ simulation runs (3 explained variance ratios x 3 signal to noise ratios x 3 sample sizes x 3 number of covariates x 300 repetitions). Note that almost all PR-AUC values of the Cox score approach are below the PR-AUC of a random classifier.} 
	\label{figPRAUC-highCor}
\end{figure}

\clearpage

\subsection{\textbf {CARS simulation with high absolute covariate correlations and censoring rate 0.75}} \label{simCARShighCorHighCens}

\begin{figure}[!htbp]
	\centering
	\includegraphics[width=\textwidth]{carPRAUC_noVarSigXexpVar_highCor_highCens.pdf}
	\caption{Results of the simulation study. PR-AUC of CARS, Cox and Cox $L_1$ scores stratified by relative number of relevant variables (relVar) and explained variance (expVar) with high absolute covariate correlations ($\rho = \pm 0.75$) and censoring rate of $75\%$. Each boxplot shows the results of $2700$ simulation runs (3 sample sizes x 3 number of covariates x 300 repetitions).}
	\label{simCARShighCorHighCensPlotNoVarExpVar}
\end{figure}

\begin{figure}[!htbp]
	\centering
	\includegraphics[scale=0.3]{carRankCor_highCor_highCens.pdf}
	\caption{Results of the simulation study, scenario with high absolute correlations ($\rho = \pm 0.75$) and a high censoring rate of $75\%$. The boxplots visualize the rank correlations of the estimated and the true covariate orderings, as obtained from variable selection by CARS and Cox scores. Each boxplot shows the results of $24300$ simulation runs (3 explained variance ratios x 3 signal to noise ratios x 3 sample sizes x 3 number of covariates x 300 repetitions).}
	\label{figCor-highCor-highCens}
\end{figure}

\begin{figure}[htbp]
	\centering
	\includegraphics[scale=0.3]{carPRAUC_highCor_highCens.pdf}
	\caption{Results of the simulation study, scenario with high absolute correlations ($\rho = \pm 0.75$) and a high censoring rate of~$75\%$. The boxplots visualize the PR-AUC values obtained from variable selection by CARS scores, Cox scores and $L_1$-penalized Cox regression. The average prevalance of the positive class (computed from all simulations) is displayed by the dashed line. Note that the boxplots contain the PR-AUC values corresponding to all three rates of influential covariates ($1\%$, $5\%$, $10\%$). Each boxplot shows the results of $24300$ simulation runs (3 explained variance ratios x 3 signal to noise ratios x 3 sample sizes x \protect\linebreak 3 number of covariates x 300 repetitions).  Note that almost all PR-AUC values of the Cox score approach are below the PR-AUC of a random classifier.}
	\label{figPRAUC-highCor-highCens}
\end{figure}

\clearpage

\subsection{\textbf {CARS simulation with low absolute covariate correlations and Weibull distribution}} \label{simCARSlowCor_Weibull}

\begin{figure}[!htbp]
	\centering
	\includegraphics[width=\textwidth]{carPRAUC_nXd_Weibull.pdf}
	\caption{Results of the simulation study. PR-AUC of CARS, Cox and Cox $L_1$ scores stratified by sample size ($n$) and number of covariates ($d$) with low absolute covariate correlations $(\rho=\pm 0.25)$ and low censoring rate of $25\%$. Each boxplot summarizes the results of $2700$ simulation runs (3 explained variance ratios x 3 signal to noise ratios x 300 repetitions).}
	\label{simCARSlowCorPlotNxP_Weibull}
\end{figure}

\begin{figure}[!htbp]
	\centering
	\includegraphics[width=\textwidth]{carPRAUC_noVarSigXexpVar_Weibull.pdf}
	\caption{Results of the simulation study. PR-AUC of CARS, Cox and Cox $L_1$ scores stratified by relative number of relevant variables (relVar) and explained variance (expVar) with low absolute covariate correlations $(\rho=\pm 0.25)$ and censoring rate of~$25\%$. Each boxplot summarizes the results of $2700$ simulation runs (3 sample sizes x 3 number of covariates x 300 repetitions).}
	\label{simCARSlowCorPlotNoVarExpVar_Weibull}
\end{figure}

\begin{figure}
	\centering
	\includegraphics[scale = 0.3]{carRankCor_Weibull.pdf}
	\caption{Results of the simulation study, scenario with low absolute correlations ($\rho = \pm 0.25$).
			The boxplots visualize the rank correlations of the estimated and the true covariate orderings, as obtained from variable selection by CARS and Cox scores. The censoring rate was equal to $25\%$. Each boxplot shows the results of $24300$ simulation runs (3 explained variance ratios x \protect\linebreak 3 signal to noise ratios x 3 sample sizes x 3 number of covariates x 300 repetitions).}
	\label{figCor_Weibull}
\end{figure}

\begin{figure}[htbp]
	\centering
	\includegraphics[scale = 0.3]{carPRAUC_Weibull.pdf}
	\caption{Results of the simulation study, scenario with low absolute correlations ($\rho = \pm 0.25$). The boxplots visualize the PR-AUC values obtained from variable selection by CARS scores, Cox scores and $L_1$-penalized Cox regression. The censoring rate was equal to $25\%$. The average prevalance of the positive class (computed from all simulations) is displayed by the dashed line. Note that the boxplots contain the PR-AUC values corresponding to all three rates of influential covariates ($1\%$, $5\%$, $10\%$). Each boxplot shows the results of $24300$ simulation runs (3 explained variance ratios x 3 signal to noise ratios x 3 sample sizes~x 3 number of covariates x 300 repetitions).}
	\label{figPRAUC_Weibull}
\end{figure}

\clearpage

\subsection{\textbf {CARS simulation with high absolute covariate correlations and Weibull distribution}} \label{simCARShighCor_Weibull}

\begin{figure}[!htbp]
	\centering
	\includegraphics[width=\textwidth]{carPRAUC_nXd_highCor_Weibull.pdf}
	\caption{Results of the simulation study. PR-AUC of CARS, Cox and Cox $L_1$ scores stratified by sample size ($n$) and number of covariates ($d$) with high absolute covariate correlations $\rho=\pm 0.75$ and censoring rate of $25\%$. Each boxplot shows the results of $2700$ simulation runs (3 explained variance ratios x 3 signal to noise ratios x 300 repetitions).}
	\label{simCARShighCorPlotNxP_Weibull}
\end{figure}

\begin{figure}[!htbp]
	\centering
	\includegraphics[width=\textwidth]{carPRAUC_noVarSigXexpVar_highCor_Weibull.pdf}
	\caption{Results of the simulation study. PR-AUC of CARS, Cox and Cox $L_1$ scores stratified by relative number of relevant variables (relVar) and explained variance (expVar) with high absolute covariate correlations $\rho=\pm 0.75$ and censoring rate of $25\%$. Each boxplot shows the results of $2700$ simulation runs (3 sample sizes x 3 number of covariates x 300 repetitions).}
	\label{simCARShighCorPlotNoVarExpVar_Weibull}
\end{figure}

\begin{figure}[!t]
	\centering
	\includegraphics[scale=0.3]{carRankCor_highCor_Weibull.pdf}
	\caption{Results of the simulation study, scenario with high absolute correlations ($\rho = \pm 0.75$). The boxplots visualize the rank correlations of the estimated and the true covariate orderings, as obtained from variable selection by CARS and Cox scores. The censoring rate was equal to $25\%$. Each boxplot shows the results of $24300$ simulation runs (3 explained variance ratios x 3 signal to noise ratios x 3 sample sizes x 3 number of covariates x 300 repetitions).}
	\label{figCor-highCor_Weibull}
\end{figure}

\begin{figure}[htbp]
	\centering
	\includegraphics[scale=0.3]{carPRAUC_highCor_Weibull.pdf}
	\caption{Results of the simulation study, scenario with high absolute correlations ($\rho = \pm 0.75$): The boxplots visualize the PR-AUC values obtained from variable selection by CARS scores, Cox scores and $L_1$-penalized Cox regression. The censoring rate was equal to $25\%$. The average prevalance of the positive class (computed from all simulations) is displayed by the dashed line. Note that the boxplots contain the PR-AUC values corresponding to all three rates of influential covariates ($1\%$, $5\%$, $10\%$). Each boxplot shows the results of $24300$ simulation runs (3 explained variance ratios x 3 signal to noise ratios x 3 sample sizes~x 3 number of covariates x 300 repetitions). Note that all PR-AUC values of the Cox score approach are below the PR-AUC of a random classifier.}
	\label{figPRAUC-highCor_Weibull}
\end{figure}

\clearpage

\subsection{\textbf {Runtime in the low correlation, low censoring scenario}} \label{simCARSruntime}

\begin{figure}[!htbp]
	\centering
	\includegraphics[width=\textwidth]{cars_compTime_nXd.pdf}
	\caption{Simulation study on computational efficiency. The boxplots show the log-transformed run-times (in seconds) of the three methods (variable selection by CARS scores, Cox scores and $L_1$-penalized Cox regression). Run-times were measured without parallelization. Each boxplot shows the results of $2700$ simulation runs (3 explained variance ratios x 3 signal to noise ratios x 300 repetitions).}
	\label{compTime}
\end{figure}

\clearpage

\subsection{\textbf {Construction of correlation matrix}} \label{constrCorr}

In this section we provide more details on the construction of the correlation matrices used in all scenarios of the simulation (see Section \ref{simulation}). First we created a preparatory design matrix $\boldsymbol{B}$ whose diagonal consisted of three equally sized blocks $\boldsymbol{A}_\xi$, with $\xi = \{0.25, 0.5, 0.75\}$ referring to the desired absolute correlations. Within each block, half of the correlations were positive and the other half were negative. The first half of positive $\xi$ correlations were inserted columwise into the upper triangle part of the block matrix  $\boldsymbol{A}_\xi$. The remaining empty entries of the upper triangle part of $\boldsymbol{A}_\xi$ were filled with the negative $\xi$ correlations. For example consider the case with 12 covariates: Then each block matrix $\boldsymbol{A}_\xi \in \mathbb{R}^{4\times 4}$ would be given by the matrix

\begin{align}
\boldsymbol{A}_\xi = \left( 
\begin{matrix}
1 & \xi & \xi & -\xi \\
\xi & 1 & \xi & -\xi \\
\xi & \xi & 1 & -\xi \\
-\xi & -\xi & -\xi & 1 \\
\end{matrix} 
\right)
\end{align}

In the respective block diagonal matrix ${\boldsymbol B}$, all correlations between variables belonging to different blocks were set to zero. Then the matrix ${\boldsymbol B}$ was converted to the nearest possible positive definite matrix $\boldsymbol{\tilde{B}}$, as measured by a weighted Frobenius Norm of the elementwise differences between the specified and the new matrix. For further details on the algorithm to minimize the deviations between $\boldsymbol{\tilde{B}}$ and $\boldsymbol{B}$ we refer to Higham (2002). \cite{nearPD} A histogram with relative frequencies of the correlation matrix entries, after applying the algorithm by Higham, above the diagonal based on $1000$ covariates is shown in Figure \ref{relFreqCorrSim}.

\begin{figure}[!htbp]
	\centering
	\includegraphics[width=\textwidth]{RelFreq_Correlations.pdf}
	\caption{Construction of correlation matrices for the simulation study. The top figure shows the histogram (100 equally sized intervals) of the first block of the final correlation matrix with $1000$ covariates. Only correlations above the diagonal are shown. The middle figure shows the histogram of the second block and the lower figure gives the relative frequencies of the third block.}
	\label{relFreqCorrSim}
\end{figure}

\clearpage

\section{\textbf{Additional material used for the data analysis}}

\subsection{\textbf{CARS scores: Selected variables in data analysis}}

\begin{table}[htbp]
	\centering
	\begin{tabular}{|c|r|r|} 
		\hline
		Gene symbol & CARS score & Q-value \\
		\hline
		% Proline rich 4 (lacrimal)
		NM\_007244 & 0.1063 & 0.0461 \\
		\hline
		% KRIT1, ankyrin repeat containing
		NM\_004912 & -0.1035 & 0.0461 \\
		\hline
		% BMX non-receptor tyrosine kinase
		NM\_203281 & -0.1016 & 0.0461 \\
		\hline
		% Baculoviral IAP repeat-containing 5 (survivin)
		NM\_001012271 & -0.0944 & 0.0814 \\
		\hline
		% Thymosin-like 8
		NM\_021992 & -0.0939 & 0.0835 \\
		\hline
		% MLLT11 myeloid/lymphoid or mixed-lineage leukemia (trithorax homolog, Drosophila)
		NM\_006818 & 0.0939 & 0.0836 \\
		\hline
		% ARP2 actin-related protein 2 homolog (yeast)
		NM\_005722 & -0.0928 & 0.0877 \\
		\hline
		% IL18R1 interleukin 18 receptor 1
		NM\_003855 & -0.0923 & 0.0895 \\
		\hline
		% BTB and CNC homology 1, basic leucine zipper transcription factor 1
		NM\_001186 & -0.0916 & 0.0918 \\
		\hline
		% SPINK5 serine peptidase inhibitor, Kazal type 5
		NM\_006846 & 0.0906 & 0.0965 \\
		\hline
	\end{tabular}
	\caption{Prostate cancer application. CARS scores and q-values below $\alpha=0.1$ ordered by absolute score magnitude. Gene symbol refers to the accession number in the NCBI database. \citep{databaseNCBI}} \label {tabCompDatAnalysisProst}
\end{table}

\begin{table}[htbp]
	\centering
	\begin{tabular}{|c|r|r|}
		\hline
		Gene Symbol & CARS score & Q-value \\		
		\hline
		% L76224.1 
		GRIN2C & -0.0479   & $1.0560 * 10^{-6}$  \\
		% NM\_014785.1
		RGP1 & 0.0446   & $6.8075 * 10^{-6}$ \\ 
		% AF065389.1 
		TSPAN5 & 0.0446   & $6.8776 * 10^{-6}$ \\ 
		% AI824113 
		RGS12 & 0.0441 & $7.7700 * 10^{-6}$ \\  
		% NM\_000419.2 
		ITGA2B & 0.0404 & $1.1334 * 10^{-4}$  \\ 
		% NM\_003172.1 
		SURF1 & -0.0402  & $1.3162 * 10^{-4}$ \\ 
		% NM\_016010.1 
		ZC2HC1A & 0.0386 & $3.7071 * 10^{-4}$  \\ 
		% BF062335 
		MCM9 & 0.0369   & $1.0399 * 10^{-3}$  \\ 
		% AC004893 
		TRRAP & 0.0368   & $1.0732 * 10^{-3}$  \\ 
		% AF243499.2 
		SLC4A5 & 0.0360  & $1.7160 * 10^{-3}$ \\ 
		% AI685892 
		FEZ2 & -0.0356   & $2.1184 * 10^{-3}$ \\ 
		% NM\_015644.1 
		ARPC4 /// ARPC4-TTLL3 /// TTLL3 & 0.0347 & $3.2360 * 10^{-3}$ \\ 
		% NM\_007035.2 
		KERA & 0.0347  & $3.3192 * 10^{-3}$ \\ 
		% AB014586.1  
		GPR98 & 0.0341  & $4.4523 * 10^{-3}$ \\ 
		% AK026589.1  
		SEPT6 & 0.0336  & $5.6147 * 10^{-3}$ \\ 
		% AB002365.1  
		PRUNE2 & 0.0334 & $6.1200 * 10^{-3}$ \\ 
		% AI738980 
		PLEKHG3 & 0.0332  & $6.6953 * 10^{-3}$ \\ 
		% AW973834 
		AW973834$^1$ & 0.0329 & $7.4615 * 10^{-3}$ \\ 
		% NM\_030878.1 
		CYP2C8 & 0.0326  & $8.2900 * 10^{-3}$  \\ 
		% NM\_022034.1 
		CUZD1 & 0.0326  & $8.4730 * 10^{-3}$ \\ 
		% NM\_003793.2 
		CTSF & 0.0324  & $8.8681 * 10^{-3}$   \\ 
		% AI122905 
		KIAA0485 & 0.0323  & $9.0860 * 10^{-3}$ \\ 
		\hline
	\end{tabular}
	\caption{Breast cancer application. CARS scores and q-values below $\alpha=0.01$ ordered by absolute score magnitude. Gene symbol refers to the accession number in the NCBI database \citep{databaseNCBI} or proteins.} \label {tabCompDatAnalysisBreast1}
\end{table}

\clearpage

\subsection{\textbf{Summary of gene enrichment analysis}}

\begin{table}[htbp]
	\centering
	\begin{tabular}{|c|c|r|r|r|r|}
		\hline
		GO.ID &	Term &	Ann &	Signif	& Expect & p-value \\
		\hline
		GO:0070735	& Protein-glycine ligase activity &	2 &	1 &	0 &	0.0020 \\
		GO:0070736	& Protein-glycine ligase activity &	2 &	1 &	0 &	0.0020 \\
		GO:0005088	& Ras guanyl-nucleotide exchange factor & 373 &	3 &	0.35 &	0.0057 \\
		GO:0008510	& Sodium:bicarbonate symporter activity	& 10 &	1 &	0.01 & 0.0099 \\
		GO:0101020	& Estrogen 16-alpha-hydroxylase activity & 10 &	1 &	0.01 & 0.0099 \\
		GO:0005085	& Guanyl-nucleotide exchange factor activity & 470 & 3 & 0.44 & 0.0107 \\
		GO:0004972	& NMDA glutamate receptor activity & 13	 & 1 &	0.01 & 0.0128 \\
		GO:0070051	& Fibrinogen binding &	13	& 1	& 0.01	& 0.0128 \\
		GO:0033695	& Oxidoreductase activity acting on CH & 15	& 1 &	0.01 & 0.0148 \\
		GO:0034875	& Caffeine oxidase activity	& 15 & 1 & 0.01 & 0.0148 \\
		GO:0016881	& Acid-amino acid ligase activity & 21 & 1 & 0.02 & 0.0207 \\
		GO:0005452	& Inorganic anion exchanger activity & 22 & 1 & 0.02 & 0.0217 \\
		GO:0015106	& Bicarbonate transmembrane transporter activity & 	23 & 1 & 0.02 & 0.0226 \\
		GO:0016725	& Oxidoreductase activity, acting on CH & 23 & 1 & 0.02 & 0.0226 \\
		GO:0015301	& Anion:anion antiporter activity & 26 & 1 & 0.02 & 0.0255 \\
		GO:0004970	& Ionotropic glutamate receptor activity & 29 & 1 & 0.03 & 0.0284 \\
		GO:0005234	& Extracellular-glutamate-gated ion channel & 30 & 1 & 0.03 & 0.0294 \\
		GO:0008324	& Cation transmembrane transporter activity & 703 & 3 & 0.66 & 0.0309 \\
		GO:0004129	& Cytochrome-c oxidase activity & 33 & 1 & 0.03 & 0.0323 \\
		GO:0015002	& Heme-copper terminal oxidase activity & 33 & 1 & 0.03 & 0.0323 \\
		GO:0016676 & Oxidoreductase activity & 33 & 1 & 0.03 & 0.0323 \\
		GO:0008391	& Arachidonic acid monooxygenase activity & 34 & 1 & 0.03 & 0.0333 \\
		GO:0008392 & Arachidonic acid epoxygenase activity & 34 & 1 & 0.03 & 0.0333 \\
		GO:0016675 & Oxidoreductase activity & 34 & 1 & 0.03 & 0.0333 \\
		GO:0070330 & Aromatase activity & 34 & 1 & 0.03 & 0.0333 \\
		GO:0017112	& Rab guanyl-nucleotide exchange factor activity & 39 & 1 & 0.04 & 0.0381 \\
		GO:0008066	& Glutamate receptor activity & 42 & 1 & 0.04 & 0.0409 \\
		GO:0016712	& Oxidoreductase activity & 42 & 1 & 0.04 & 0.0409 \\
		GO:0022824	& Transmitter-gated ion channel activity & 44 & 1 & 0.04 & 0.0429 \\
		GO:0022835	& Transmitter-gated channel activity & 44 & 1 & 0.04 & 0.0429 \\
		GO:0015296	& Anion:cation symporter activity & 46 & 1 & 0.04 & 0.0448 \\
		GO:0008395	& Steroid hydroxylase activity & 50 & 1 & 0.05 & 0.0486 \\
		\hline
	\end{tabular}
	\caption{Breast cancer application. Gene enrichment analyis based on Fisher's test with \textit{R} package \textit{topGO}. The first column gives the gene ontology (GO) \citep{Consortium01082001} identification number and column "Term" gives additional details. The number in column "Ann" shows how many genes are annotated with the GO term. Column "Signif" gives the number of significant genes of the GO term with respect to q-value $0.05$. "Expect" shows the expected number of significant genes under the null hypothesis, e.g. no genes are enriched. The last column "p-value" gives the p-value for Fisher's test for enrichment.} \label{tabTopGO}
\end{table}

\clearpage

\subsection{\textbf{CARS scores diagnostic plots}} 

\begin{figure}[!htbp]
	\centering
	\includegraphics[width=\textwidth]{SwedishWatchful_varSel.pdf}
	\caption{Diagnostic plots obtained in the prostate cancer application (density of empirical null model, cumulative distribution function and local FDR). The first two plots show that the mixture model fits the empirical distribution well.}
	\label{nullModel}
\end{figure}

\begin{figure}[!htbp]
	\centering
	\includegraphics[width=\textwidth]{BreastCancer_varSel.pdf}
	\caption{Diagnostic plots obtained in the breast cancer application (density of empirical null model, cumulative distribution function and local FDR). The first two plots show that the mixture model fits the empirical distribution well.}
	\label{BreastNullModel}
\end{figure}

\clearpage

\bibliography{wileyNJD-AMS}

\end{document}
