\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage[super,sort,comma]{natbib}

\title{Supplementary Material}
\date{}
\usepackage{natbib}
\usepackage{graphicx}
\usepackage{color,soul}

\setlength\parindent{0pt}
\renewcommand{\figurename}{Fig.}
\usepackage{geometry}
\geometry{margin=1in}

\newcommand{\hlr}[1]{{\sethlcolor{white}\hl{#1}}}

\usepackage[table,xcdraw]{xcolor}

%% Stylefile to load MEDIMA template
%\usepackage{medima}
\usepackage{framed,multirow}
\usepackage{amsmath,amssymb,amsfonts}
\graphicspath{{Figures/}}
\usepackage{arydshln}
\usepackage{comment}
\usepackage{subcaption}
%\usepackage[table,xcdraw]{xcolor}
\renewcommand{\figurename}{Fig.}
 \usepackage{multirow}
% Following three lines are needed for this document.
% If you are not loading colors or url, then these are
% not required.
\usepackage{url}
\usepackage{xcolor}
\usepackage{color,soul}
\usepackage{hyperref}
\renewcommand{\thetable}{S\arabic{table}}
\renewcommand{\thefigure}{S\arabic{figure}}

\begin{document}

\maketitle
\section{Results}
\subsection{Quantitative evaluation}
Tables \ref{tab:lesion_eval_supp}, \ref{tab:cs_esion_eval_supp} and \ref{tab:patient_level_eval_supp} present detailed versions of the Tables 3, 4 and 5 presented in the manuscript. Model rankings for each metric are presented in brackets beside the model performance. The sum of ranks column shows how the individual metric ranks were aggregated to derive the final rank for each model. 
\begin{table*}[!htbp]
\centering
\caption{Detecting cancer: Lesion-level evaluation on cohorts C1-test and cohort C2. Mean and standard deviation values for each metric in each cohort is reported. Numbers in brackets indicate model rank in individual metric. Column ``Sum of ranks'' represents the sum of the individual metric ranks. Column ``Final rank'' represents rank of model based on sum of individual ranks. Note, the lower the sum rank, the higher ranked (i.e. better) the model. The first three rows above the dotted lines in both cohorts represent the MRI-only models, while the rows below the dotted lines represent the MRI+correlated features-based models. CorrSigNIA outperforms all other models in both cohorts.}
\label{tab:lesion_eval_supp}
\resizebox{\textwidth}{!}{%
\begin{tabular}{|l|l|l|l|l|l|l|l|l|l|c|c|}
\hline
 \multicolumn{12}{|c|}{\textbf{Evaluation on cohort C1-test}} \\ 
 \multicolumn{12}{|c|}{Patients with cancer lesion volumes $\geq$ 250~$mm^3$ N = 37, number of cancerous lesions = 44} \\ 
 \hline
\hline

Model                                           & ROC AUC                    & PR AUC                     & Sensitivity                & Specificity                & Precision                  & NPV                        & F1 Score                   & Dice                 & Accuracy                   & \begin{tabular}[c]{@{}c@{}}Sum of \\ Ranks\end{tabular} & {\begin{tabular}[c]{@{}c@{}}Final\\ Rank\end{tabular}} \\ \hline
SPCNet                                          & \textbf{0.82$\pm$0.30 (1)} & 0.76$\pm$0.36 (5)          & 0.62$\pm$0.47 (11)         & 0.75$\pm$0.30 (5)          & 0.44$\pm$0.41 (5)          & 0.84$\pm$0.22 (4)          & 0.48$\pm$0.40 (7)          & 0.27$\pm$0.26 (13)   & 0.72$\pm$0.24 (5)          & 56                                                      & {5}                                                    \\ 
U-Net                                           & 0.82$\pm$0.34 (2)          & \textbf{0.81$\pm$0.34 (1)} & 0.84$\pm$0.35 (4)          & 0.30$\pm$0.31 (11)         & 0.35$\pm$0.27 (11)         & 0.57$\pm$0.47 (11)         & 0.45$\pm$0.25 (11)         & 0.30$\pm$0.20 (2)    & 0.43$\pm$0.26 (12)         & 65                                                      & {8}                                                    \\ 
BrU-Net                                  & 0.80$\pm$ 0.32 (5)         & 0.78$\pm$0.34 (2)          & 0.82$\pm$0.35 (5)          & 0.39$\pm$0.38 (10)         & 0.37$\pm$0.29 (10)         & 0.57$\pm$0.46 (10)         & 0.47$\pm$0.28 (9)          & 0.27$\pm$0.17 (8)    & 0.52$\pm$0.27 (10)         & 69                                                      & {9}                                                    \\ \hdashline
\textbf{CorrSigNIA}         & 0.81$\pm$0.31 (3)          & 0.73$\pm$0.38 (8)          & 0.73$\pm$0.43 (6)          & 0.72$\pm$0.33 (7)          & \textbf{0.50$\pm$0.41 (1)} & 0.86$\pm$0.25 (3)          & \textbf{0.55$\pm$0.39 (1)} & 0.30$\pm$0.24 (4)    & 0.72$\pm$0.26 (6)          & \textbf{39}                                             & {\textbf{1}}                                           \\ 

SPCNet + CR-3-S               & 0.79$\pm$0.31 (8)          & 0.72$\pm$0.38 (10)         & 0.62$\pm$0.46 (10)         & 0.80$\pm$0.21 (2)          & 0.45$\pm$0.39 (4)          & \textbf{0.88$\pm$0.15 (1)} & 0.49$\pm$0.38 (6)          & 0.27$\pm$0.23 (10)   & \textbf{0.75$\pm$0.17 (1)} & 52                                                      & {4}                                                    \\ 

SPCNet + CR-123-I        & 0.79$\pm$0.33 (9)          & 0.72$\pm$0.37 (9)          & 0.66$\pm$0.47 (9)          & 0.77$\pm$0.29 (4)          & 0.48$\pm$0.42 (3)          & 0.87$\pm$0.21 (2)          & 0.52$\pm$0.41 (3)          & 0.29$\pm$0.24 (6)    & 0.75$\pm$0.23 (3)          & 48                                                      & {2}                                                    \\ 

SPCNet + CR-123-S       & 0.73$\pm$0.35 (12)         & 0.70$\pm$0.38 (12)         & 0.55$\pm$0.48 (13)         & \textbf{0.82$\pm$0.24 (1)} & 0.41$\pm$0.41 (8)          & 0.83$\pm$0.21 (5)          & 0.44$\pm$0.40 (12)         & 0.27$\pm$0.24 (12)   & 0.75$\pm$0.20 (2)          & 77                                                      & {11}                                                   \\ 

SPCNet + CR-4-I          & 0.80$\pm$0.34 (6)          & 0.77$\pm$0.36 (4)          & 0.58$\pm$0.47 (12)         & 0.80$\pm$0.30 (3)          & 0.44$\pm$0.42 (6)          & 0.82$\pm$0.25 (6)          & 0.47$\pm$0.42 (10)         & 0.28$\pm$0.23 (7)    & 0.73$\pm$0.24 (4)          & 58                                                      & {7}                                                    \\ 

SPCNet + CR-4-S           & 0.79$\pm$0.34 (10)         & 0.75$\pm$0.37 (6)          & 0.69$\pm$0.44 (7)          & 0.72$\pm$0.32 (6)          & \textbf{0.50$\pm$0.41 (1)}          & 0.82$\pm$0.28 (7)          & 0.54$\pm$0.39 (2)          & 0.30$\pm$0.23 (3)    & 0.71$\pm$0.27 (7)          & 49                                                      & {3}                                                    \\ 

U-Net + CR-3-I            & 0.80$\pm$0.34 (6)          & 0.77$\pm$0.34 (3)          & \textbf{0.89$\pm$0.29 (1)}          & 0.27$\pm$0.33 (12)         & 0.34$\pm$0.23 (12)         & 0.46$\pm$0.48 (12)         & 0.47$\pm$0.24 (8)          & 0.29$\pm$0.17 (5)    & 0.44$\pm$0.26 (11)         & 70                                                      & {10}                                                   \\ 


U-Net + CR-123-I          & 0.80$\pm$0.31 (4)          & 0.72$\pm$0.38 (10)         & 0.88$\pm$0.32 (2)          & 0.41$\pm$0.37 (9)          & 0.40$\pm$0.29 (9)          & 0.65$\pm$0.46 (9)          & 0.51$\pm$0.28 (4)          & \textbf{0.31$\pm$0.21 (1)}    & 0.55$\pm$0.28 (9)          & 57                                                      & {6}                                                    \\ 

BrU-Net + CR-3-I   & 0.77$\pm$0.38 (11)         & 0.75$\pm$0.38 (7)          & 0.88$\pm$0.32 (2) & 0.10$\pm$0.25 (13)         & 0.30$\pm$0.22 (13)         & 0.20$\pm$0.39 (13)         & 0.42$\pm$0.23 (13)         & 0.27$\pm$0.17 (8) & 0.32$\pm$0.23 (13)         & 93                                                      & {13}                                                   \\ 

BrU-Net + CR-123-I & 0.72$\pm$0.39 (13)         & 0.66$\pm$0.40 (13)         & 0.69$\pm$0.44 (7)          & 0.58$\pm$0.38 (8)          & 0.43$\pm$0.38 (7)          & 0.70$\pm$0.38 (8)          & 0.49$\pm$0.37 (5)          & 0.27$\pm$0.23 (10)    & 0.62$\pm$0.30 (8)          & 79                                                      & {12}                                                   \\ \hline
\hline
\multicolumn{12}{|c|}{\textbf{Evaluation on cohort C2}} \\ 
 \multicolumn{12}{|c|} {Patients with cancer N = 147, number of cancerous lesions = 189}  \\ \hline
\hline
Model                                           & ROC AUC                    & PR AUC                     & Sensitivity                & Specificity                & Precision                  & NPV                        & F1 Score                   & Dice                 & Accuracy                   & \begin{tabular}[c]{@{}c@{}}Sum of \\ Ranks\end{tabular} & {\begin{tabular}[c]{@{}c@{}}Final\\ Rank\end{tabular}} \\ \hline
SPCNet & 0.75$\pm$0.36 (6) & 0.76$\pm$0.35 (6) & 0.50$\pm$0.47 (7) & 0.77$\pm$0.36 (3) & 0.42$\pm$0.43(4) & 0.69$\pm$0.33 (3) & 0.43$\pm$0.42 (6) & 0.25$\pm$0.25 (6) & 0.69$\pm$0.24 (3) & 44 & 5 \\ 

U-Net & 0.78$\pm$0.33 (4) & 0.77$\pm$0.34 (4) & 0.76$\pm$0.39 (3) & 0.57$\pm$0.39 (7) & \textbf{0.50$\pm$0.35 (1)} & 0.66$\pm$0.41 (5) & \textbf{0.56$\pm$0.33 (1)} & 0.33$\pm$0.23 (2) & 0.63$\pm$0.26 (5) & 32 & 3 \\ 

BrU-Net & {0.79$\pm$ 0.35 (3)} & {0.79$\pm$0.34 (3)} & {0.89$\pm$0.29 (2)} & {0.27$\pm$0.36 (8)} & {0.41$\pm$0.26 (5)} & {0.41$\pm$0.47 (8)} & {0.53$\pm$0.25 (2)} & {\textbf{0.33$\pm$0.20 (1)}} & {0.48$\pm$0.26 (8)} & {40} & {4} \\ \hdashline

\textbf{CorrSigNIA} & \textbf{0.81$\pm$0.31 (1)} & 0.79$\pm$0.33 (2) & 0.58$\pm$0.46 (4) & 0.76$\pm$0.36 (4) & 0.49$\pm$0.43 (2) & 0.71$\pm$0.33 (2) & 0.49$\pm$0.41 (4) & 0.30$\pm$0.26 (4) & 0.71$\pm$0.25 (2) & \textbf{25} & \textbf{1} \\ 

SPCNet + CR-123-I & {0.77$\pm$0.33 (5)} & {0.76$\pm$0.34 (5)} & {0.55$\pm$0.47 (5)} & {\textbf{0.81$\pm$0.33 (1)}} & {0.47$\pm$0.44 (3)} & {\textbf{0.75$\pm$0.29 (1)}} & {0.48$\pm$0.42 (5)} & {0.30$\pm$0.27 (5)} & {\textbf{0.74$\pm$0.22 (1)}} & {31} & {2} \\ 

{U-Net + CR-3-I} & {0.49$\pm$0.43 (9)} & {0.54$\pm$0.39 (9)} & {0.40$\pm$0.46 (9)} & {0.59$\pm$0.40 (6)} & {0.30$\pm$0.38 (9)} & {0.56$\pm$0.36 (7)} & {0.31$\pm$0.37 (9)} & {0.19$\pm$0.22 (9)} & {0.53$\pm$0.29 (7)} & {74} & {9} \\

{U-Net + CR-123-I} & {0.67$\pm$0.38 (7)} & {0.66$\pm$0.38 (7)} & {0.52$\pm$0.47 (6)} & {0.67$\pm$0.39 (5)} & {0.41$\pm$0.41 (6)} & {0.65$\pm$0.35 (6)} & {0.42$\pm$0.39 (7)} & {0.24$\pm$0.24 (7)} & {0.62$\pm$0.28 (6)} & {57} & {7} \\
 
{BrU-Net + CR-3-I} & {0.81$\pm$0.33 (2)} & {\textbf{0.81$\pm$0.33(1)}} & {\textbf{0.90$\pm$0.27(1)}} & {0.12$\pm$0.26 (9)} & {0.36$\pm$0.20 (8)} & {0.22$\pm$0.40 (9)} & {0.49$\pm$0.20 (3)} & {0.32$\pm$0.19 (3)} & {0.39$\pm$0.21 (9)} & {45} & {6} \\

{BrU-Net + CR-123-I} & {0.61$\pm$0.40 (8)} & {0.62$\pm$0.39 (8)} & {0.41$\pm$0.46 (8)} & {0.78$\pm$0.34 (2)} & {0.37$\pm$0.44 (7)} & {0.68$\pm$0.30 (4)} & {0.36$\pm$0.41 (8)} & {0.21$\pm$0.25 (8)} & {0.66$\pm$0.25 (4)} & {57} & {7} \\ \hline
\end{tabular}% 
}
\end{table*}


% Please add the following required packages to your document preamble:
% \usepackage{graphicx}
\begin{table*}[!htbp]
\centering
\caption{Detecting clinically significant cancer: Lesion-level evaluation on cohorts C1-test and cohort C2. Mean and standard deviation values for each metric in each cohort is reported. Numbers in brackets indicate model rank in individual metric. Column ``Sum of ranks'' represents the sum of the individual metric ranks. Column ``Final rank'' represents rank of model based on sum of individual ranks. Note, the lower the sum rank, the higher ranked (i.e. better) the model. The first three rows above the dotted lines in both cohorts represent the MRI-only models, while the rows below the dotted lines represent the MRI+correlated features-based models. CorrSigNIA ranks second in cohort C1-test, closely following another CorrSigNIA variant, whereas it ranks first in cohort C2.}
\label{tab:cs_esion_eval_supp}
\resizebox{\textwidth}{!}{%
\begin{tabular}{|l|l|l|l|l|l|l|l|l|l|l|c|c|}
\hline
 \multicolumn{13}{|c|}{\textbf{Evaluation on cohort C1-test}} \\ 
 \multicolumn{13}{|c|}{Patients with clinically significant cancer with lesion volume$\geq$ 250~$mm^3$ N = 34, number of clinically significant cancerous lesions = 37} \\ 
 \hline
\hline

Model & ROC AUC & PR AUC & Sensitivity & Specificity & Precision & NPV & F1 Score & Dice & Accuracy & \begin{tabular}[c]{@{}l@{}}Agg cancer \\ overlap\end{tabular} & \begin{tabular}[c]{@{}c@{}}Sum of\\ Ranks\end{tabular} & \begin{tabular}[c]{@{}c@{}}Final\\ Rank\end{tabular} \\ \hline

SPCNet & 0.83$\pm$0.30 (3) & 0.77$\pm$0.37 (3) & 0.68$\pm$0.47 (10) & 0.76$\pm$0.29 (4) & 0.47$\pm$0.41 (4) & 0.88$\pm$0.21 (3) & 0.52$\pm$0.41(4) & 0.29$\pm$0.25 (8) & 0.75$\pm$0.24 (5) & 0.18$\pm$0.28 (9) & 53 & 4 \\ 

U-Net & \textbf{0.84$\pm$0.32 (1)} & \textbf{0.81$\pm$0.34 (1)} & 0.88$\pm$0.34 (4) & 0.33$\pm$0.31 (10) & 0.35$\pm$0.28 (10) & 0.65$\pm$0.46 (10) & 0.46$\pm$0.27 (10) & 0.30$\pm$0.20 (5) & 0.46$\pm$0.28 (11) & 0.30$\pm$0.33 (5) & 67 & 7 \\ 

Branched   U-Net & 0.81$\pm$ 0.32 (8) & 0.75$\pm$0.38 (9) & 0.84$\pm$0.36 (5) & 0.39$\pm$0.37 (9) & 0.34$\pm$0.28 (11) & 0.62$\pm$0.46 (11) & 0.45$\pm$0.28 (11) & 0.27$\pm$0.18 (11) & 0.51$\pm$0.27 (10) & 0.20$\pm$0.26 (7) & 92 & 12 \\ \hdashline

\textbf{CorrSigNIA} & 0.82$\pm$0.31 (5) & 0.74$\pm$0.38 (10) & 0.79$\pm$0.40 (6) & 0.72$\pm$0.33 (6) & \textbf{0.53$\pm$0.40 (1)} & 0.88$\pm$0.25 (4) & \textbf{0.59$\pm$0.39 (1)} & \textbf{0.32$\pm$0.24 (1)} & 0.73$\pm$0.27 (6) & 0.20$\pm$0.29 (8) & 48 & 2 \\


SPCNet + CR-3-S & 0.79$\pm$0.31 (11) & 0.70$\pm$0.39 (12) & 0.66$\pm$0.47 (11) & 0.79$\pm$0.22 (3) & 0.45$\pm$0.39 (6) & \textbf{0.91$\pm$0.13 (1)} & 0.51$\pm$0.40 (5) & 0.28$\pm$0.23 (10) & \textbf{0.77$\pm$0.18 (1)} & 0.14$\pm$0.26 (13) & 73 & 8 \\ 

SPCNet + CR-123-I & 0.83$\pm$0.32 (4) & 0.76$\pm$0.37 (5) & 0.71$\pm$0.46 (9) & 0.76$\pm$0.29 (4) & 0.50$\pm$0.41 (2) & 0.89$\pm$0.21 (2) & 0.55$\pm$0.41 (3) & 0.31$\pm$0.23 (3) & 0.76$\pm$0.23 (3) & 0.18$\pm$0.28 (9) & \textbf{44} & \textbf{1} \\ 

SPCNet + CR-123-S & 0.77$\pm$0.34 (12) & 0.73$\pm$0.39 (11) & 0.60$\pm$0.48 (13) & \textbf{0.81$\pm$0.25 (1)} & 0.44$\pm$0.41 (7) & 0.86$\pm$0.20 (5) & 0.48$\pm$0.42 (9) & 0.29$\pm$0.24 (7) & 0.77$\pm$0.21 (2) & 0.17$\pm$0.27 (10) & 77 & 9 \\ 

SPCNet + CR-4-I & 0.82$\pm$0.33 (7) & 0.78$\pm$0.36 (2) & 0.63$\pm$0.47 (12) & 0.80$\pm$0.30 (2) & 0.47$\pm$0.43 (5) & 0.85$\pm$0.25 (6) & 0.51$\pm$0.43 (6) & 0.29$\pm$0.23 (6) & 0.76$\pm$0.25 (4) & 0.16$\pm$0.26 (12) & 62 & 6 \\ 

SPCNet + CR-4-S & 0.81$\pm$0.32 (8) & 0.76$\pm$0.37 (5) & 0.72$\pm$0.44 (7) & 0.72$\pm$0.32 (5) & 0.50$\pm$0.41 (2) & 0.85$\pm$0.29 (7) & 0.55$\pm$0.39 (2) & 0.31$\pm$0.23 (3) & 0.72$\pm$0.27 (7) & 0.21$\pm$0.29 (6) & 52 & 3 \\ 

U-Net   + CR-3-I & 0.82$\pm$0.32 (6) & 0.75$\pm$0.36 (7) & \textbf{0.93$\pm$0.25 (1)} & 0.24$\pm$0.28 (11) & 0.31$\pm$0.19 (12) & 0.48$\pm$0.49 (12) & 0.44$\pm$0.20 (12) & 0.28$\pm$0.18 (9) & 0.41$\pm$0.23 (12) & 0.36$\pm$0.29 (4) & 86 & 10 \\ 

U-Net   + CR-123-I & 0.83$\pm$0.29 (2) & 0.75$\pm$0.37 (8) & 0.90$\pm$0.29 (3) & 0.41$\pm$0.37 (8) & 0.39$\pm$0.29 (9) & 0.66$\pm$0.46 (9) & 0.50$\pm$0.27 (7) & 0.31$\pm$0.21 (2) & 0.53$\pm$0.29 (9) & 0.46$\pm$0.34 (2) & 59 & 5 \\ 

BrU-Net + CR-3-I & 0.80$\pm$0.36 (10) & 0.77$\pm$0.37 (3) & 0.91$\pm$0.28 (2) & 0.09$\pm$0.20 (12) & 0.26$\pm$0.17 (13) & 0.22$\pm$0.41 (13) & 0.39$\pm$0.19 (13) & 0.26$\pm$0.16 (13) & 0.29$\pm$0.18 (13) & \textbf{0.53$\pm$0.30 (1)} & 93 & 13 \\ 

BrU-Net + CR-123-I & 0.73$\pm$0.41 (13) & 0.70$\pm$0.40 (13) & 0.72$\pm$0.44 (7) & 0.56$\pm$0.38 (7) & 0.41$\pm$0.37 (8) & 0.70$\pm$0.39 (8) & 0.49$\pm$0.37 (8) & 0.27$\pm$0.23 (12) & 0.60$\pm$0.31 (8) & 0.42$\pm$0.36 (3) & 87 & 11 \\ 
\hline
\hline 
 \multicolumn{13}{|c|}{\textbf{Evaluation on cohort C2}} \\ 
 \multicolumn{13}{|c|}{Patients with clinically significant cancer N = 90, number of clinically significant cancerous lesions = 110} \\ 
 \hline
\hline

Model & ROC AUC & PR AUC & Sensitivity & Specificity & Precision & NPV & F1 Score & Dice & Accuracy & \begin{tabular}[c]{@{}l@{}}Agg cancer \\ overlap\end{tabular} & \begin{tabular}[c]{@{}c@{}}Sum of\\ Ranks\end{tabular} & \begin{tabular}[c]{@{}c@{}}Final\\ Rank\end{tabular} \\ \hline

SPCNet & 0.79$\pm$0.34 (6) & 0.79$\pm$0.34 (6) & 0.58$\pm$0.47 (7) & 0.74$\pm$0.37 (3) & 0.48$\pm$0.42 (4) & 0.71$\pm$0.34 (4) & 0.49$\pm$0.41(7) & 0.29$\pm$0.25 (6) & 0.70$\pm$0.25 (4) & 0.14$\pm$0.24 (9) & 56 & 7 \\ 

U-Net & 0.83$\pm$0.30 (4) & 0.81$\pm$0.32 (4) & 0.81$\pm$0.37 (3) & 0.58$\pm$0.39 (7) & 0.53$\pm$0.34 (3) & 0.68$\pm$0.41 (6) & \textbf{0.60$\pm$0.32 (1)} & \textbf{0.38$\pm$0.23 (1)} & 0.67$\pm$0.26 (5) & 0.26$\pm$0.28 (2) & 36 & 3 \\ 

Branched   U-Net & 0.84$\pm$ 0.31 (3) & 0.82$\pm$0.33 (3) & \textbf{0.94$\pm$0.22 (1)} & 0.26$\pm$0.35 (8) & 0.42$\pm$0.25 (7) & 0.41$\pm$0.48 (8) & 0.55$\pm$0.23 (4) & 0.35$\pm$0.19 (3) & 0.48$\pm$0.25 (8) & 0.24$\pm$0.24 (4) & 49 & 4 \\ \hdashline

\textbf{CorrSigNIA} & \textbf{0.86$\pm$0.26 (1)} & 0.83$\pm$0.31 (2) & 0.67$\pm$0.43 (4) & 0.74$\pm$0.38 (4) & \textbf{0.55$\pm$0.41 (1)} & 0.72$\pm$0.36 (3) & 0.57$\pm$0.39 (2) & 0.35$\pm$0.25 (5) & 0.74$\pm$0.25 (2) & 0.24$\pm$0.31 (5) & \textbf{29} & \textbf{1} \\ 

SPCNet + CR-123-I & 0.81$\pm$0.30 (5) & 0.79$\pm$0.33 (5) & 0.65$\pm$0.45 (5) & 0.80$\pm$0.34 (2) & 0.54$\pm$0.42 (2) & \textbf{0.77$\pm$0.31 (1)} & 0.56$\pm$0.41 (3) & 0.36$\pm$0.21 (2) & \textbf{0.76$\pm$0.21 (1)} & 0.21$\pm$0.29 (7) & 33 & 2 \\ 

U-Net   + CR-3-I & 0.55$\pm$0.44 (9) & 0.59$\pm$0.40 (9) & 0.46$\pm$0.47 (9) & 0.62$\pm$0.40 (6) & 0.34$\pm$0.39 (9) & 0.61$\pm$0.36 (7) & 0.36$\pm$0.39 (9) & 0.22$\pm$0.24 (9) & 0.58$\pm$0.29 (7) & 0.15$\pm$0.24 (8) & 82 & 9 \\ 

U-Net   + CR-123-I & 0.72$\pm$0.37 (7) & 0.70$\pm$0.37 (7) & 0.61$\pm$0.46 (6) & 0.70$\pm$0.37 (5) & 0.48$\pm$0.42 (4) & 0.70$\pm$0.35 (5) & 0.50$\pm$0.41 (5) & 0.28$\pm$0.25 (7) & 0.67$\pm$0.28 (6) & 0.26$\pm$0.31 (3) & 55 & 6 \\ 

BrU-Net + CR-3-I & 0.86$\pm$0.31 (2) & \textbf{0.85$\pm$0.30 (1)} & 0.93$\pm$0.24 (2) & 0.12$\pm$0.23 (9) & 0.35$\pm$0.17 (8) & 0.25$\pm$0.42 (9) & 0.49$\pm$0.18 (6) & 0.35$\pm$0.19 (3) & 0.39$\pm$0.19 (9) & \textbf{0.62$\pm$0.30 (1)} & 50 & 5 \\ 

BrU-Net + CR-123-I & 0.67$\pm$0.39 (8) & 0.67$\pm$0.38 (8) & 0.49$\pm$0.47 (8) & \textbf{0.80$\pm$0.33 (1)} & 0.44$\pm$0.45 (6) & 0.73$\pm$0.30 (2) & 0.44$\pm$0.43 (8) & 0.25$\pm$0.26 (8) & 0.71$\pm$0.25 (3) & 0.23$\pm$0.30 (6) & 58 & 8 \\ \hline
\end{tabular}%
}
\end{table*}



\begin{table*}[!htbp]
\centering
\caption{Patient-level evaluation on cohorts C1-test and C3-test (Total number of subjects N = 70, including 55 men with cancer from C1-test and 15 men with normal MRI from C3-test. Numbers in brackets represent model rank in particular metric. Column ``Sum of ranks'' represents the sum of the individual metric ranks. Column ``Final rank'' represents rank of model based on sum of individual ranks.  Note, the lower the sum rank, the higher ranked (i.e. better) the model. The first three rows above the dotted lines in both cohorts represent the MRI-only models, while the rows below the dotted lines represent the MRI+correlated features-based models. CorrSigNIA outperforms all other models in patient-level evaluation.}
\label{tab:patient_level_eval_supp}
\resizebox{\textwidth}{!}{%
\begin{tabular}{|l|l|l|l|l|l|l|l|l|}
\hline
Model                                      & Sensitivity       & Specificity       & Precision         & NPV               & F1 score         & Accuracy          & \begin{tabular}[c]{@{}c@{}}Sum of\\ Ranks\end{tabular} & \begin{tabular}[c]{@{}c@{}}Final\\ Rank\end{tabular}    \\ \hline
SPCNet                                     & 0.71 (10)         & 0.80 (3)          & 0.93 (3)          & 0.43 (3)          & 0.80 (9)          & 0.73 (4)          & 32         & 3          \\ 

U-Net                                       & 0.87 (4)          & 0.00 (12)         & 0.76 (13)         & 0.00 (12)         & 0.81 (8)          & 0.69 (11)         & 60          & 13         \\ 

Branched U-Net                             & 0.87  (4)         & 0.13 (9)          & 0.79 (9)          & 0.22 (9)          & 0.83 (5)          & 0.71 (8)          & 44          & 9          \\ \hdashline

\textbf{CorrSigNIA}              & 0.78 (6) & \textbf{0.87 (1)} & \textbf{0.96 (1)} & \textbf{0.52 (1)} & 0.86 (2) & \textbf{0.80 (1)} & \textbf{12} & \textbf{1} \\

SPCNet + CR-3-S   & 0.73 (9)          & 0.53 (7)          & 0.85 (8)          & 0.35 (8)          & 0.78 (12)         & 0.69 (11)         & 55          & 12         \\ 
SPCNet + CR-123-I  & 0.71 (10)         & 0.80 (3)          & 0.93 (3)          & 0.43 (3)          & 0.80 (9)          & 0.73 (4)          & 32          & 3          \\


SPCNet + CR-123-S & 0.67 (11)         & 0.87 (1)          & 0.95 (2)          & 0.42 (5)          & 0.79 (11)         & 0.71 (8)          & 38          & 7          \\ 


SPCNet + CR-4-I   & 0.67 (11)         & 0.80 (3)          & 0.93 (3)          & 0.40 (6)          & 0.78 (12)         & 0.70 (13)         & 48          & 10         \\

SPCNet + CR-4-S  & 0.78 (6)          & 0.73 (6)          & 0.91 (6)          & 0.48 (2)          & 0.84 (3)          & 0.77 (3)          & 26          & 2          \\ 

U-Net + CR-3-I                         & 0.91 (1)          & 0.07 (10)         & 0.78 (12)         & 0.17 (10)         & 0.84 (3)          & 0.73 (4)          & 40          & 8          \\ 

U-Net + CR-123-I                      & 0.89 (3)          & 0.07 (10)         & 0.78 (11)         & 0.14 (11)         & 0.83 (5)          & 0.71 (8)          & 48          & 10        \\ 

BrU-Net + CR-3-I             & \textbf{0.91 (1)}          & 0.00 (12)            & 0.77 (9)          & 0.00 (12)         & \textbf{0.83 (1)}          & 0.71 (2)          & 37          & 5         \\ 

BrU-Net + CR-123-I             & 0.78 (6)          & 0.53 (7)          & 0.86 (7)          & 0.40 (6)          & 0.82 (7)          & 0.73 (4)          & 37          & 5         \\ \hline

\end{tabular}%
}
\end{table*}


\newpage
\subsection{Qualitative analysis of false negative and false positive predictions}

To assess the clinical utility of CorrSigNIA, false negative and false positive predictions were qualitatively analysed in cohorts C1-test and C3-test.

\figurename~\ref{fig:fn_median_cs} shows slices from apex to base of a patient in cohort C1-test, where CorrSigNIA missed the clinically significant lesion of volume 548~mm$^3$ on the left of the image (right of the patient). The false negative prediction could be due to the small size of the lesion and the subtle appearance of the lesion on MRI. It may however be noted that CorrSigNIA correctly identifies the false positive annotation of the radiologist (red) on the right of the image (left of patient) as benign tissue.

%\textcolor{purple}{\figurename~\ref{fig:fn_largest_cs} shows slices from apex to base of the patient with the largest missed clinically significant (lesion volume 2669~mm$^3$) in cohort C1-test on the right of the image (left of the patient). CorrSigNIA also shows false positive predictions for this patient on the anterior side of the prostate, which corresponds to some hypointensity in the T2w and ADC image, next to the benign prostatic hyperplasia (BPH) nodules.}

\figurename~\ref{fig:fp} shows slices from apex to base of a man with normal prostate MRI in cohort C3-test. CorrSigNIA generated a false positive prediction (predominantly indolent cancer) for this patient on the right anterior side of the prostate (left side of image).

\begin{figure*} [!htbp]
    \centering
     \includegraphics[page=2, trim=5 300 5 250, clip, width=\linewidth]{corrsignia_supp_figs.pdf}
    \caption{CorrSigNIA false negative predictions in a patient from cohort C1-test, shown from apex to base. Registered (a) T2w image, (b) ADC image, and (c) histopathology image. T2w image overlaid with (d) ground truth labels: cancer from expert pathologist (black outline), aggressive cancer (yellow) and indolent cancer (green) histologic grade labels, pixels within pathologist outline without automated histologic grade labels (brown), (e) radiologist annotations (red), together with pathologist cancer annotations (black outline), (f)  CorrSigNIA predictions, together with pathologist cancer annotations (black outline). CorrSigNIA misses the cancerous lesion on the left of the image (lesion volume 548~mm$^3$), but it also correctly identifies the false positive annotation of the radiologist as benign tissue.}
    \label{fig:fn_median_cs}
\end{figure*}

\begin{comment}

\begin{figure*} [!htbp]
    \centering
     \includegraphics[page=2, trim=3 270 3 250, clip, width=\linewidth]{corrsignia_supp_fig2.pdf}
    \caption{CorrSigNIA false negative predictions in the patient with the largest missed lesion from cohort C1-test, shown from apex to base. Registered (a) T2w image, (b) ADC image, (c) histopathology image,(d) T2w image overlaid with ground truth labels: cancer from expert pathologist (black outline), aggressive cancer (yellow) and indolent cancer (green) histologic grading, pixels within pathologist outline without automated histologic grade labels shown in brown, (e) T2w image overlaid with radiologist annotations (red outline), together with pathologist cancer annotations (black outline), (f) T2w image overlaid with CorrSigNIA predictions, together with pathologist cancer annotations (black outline). CorrSigNIA misses the cancerous lesion (lesion volume 2669~mm$^3$) on the right of the image, and has false positive predictions on the anterior side of the prostate.}
    \label{fig:fn_largest_cs}
\end{figure*}

\end{comment}

\begin{figure*} [!htbp]
    \centering
     \includegraphics[page=3, trim=3 100 3 50, clip, width=\linewidth]{corrsignia_supp_figs.pdf}
    \caption{CorrSigNIA false positive predictions in a man with normal prostate MRI from cohort C3-test, shown from apex to base. Registered (a) T2w image, (b) ADC image, (c) T2w image overlaid with CorrSigNIA predictions. CorrSigNIA generated a false positive prediction (predominantly indolent cancer) in the right anterior prostate (left side of image).}
    \label{fig:fp}
\end{figure*}
\end{document}