Algorithm 3 Top-K influential Sub-signals.

1:procedureTopkSub-Signals($X,x,model,w_{size},k$)  2:    $mae\_signal\leftarrow$ empty list  3:    $subsignals\leftarrow$ empty list  4:    $s\leftarrow$Segment($x,w_{size}$)  5:    for $i\leftarrow 1,\left|s\right|$ do  6:        $x_{occ}\leftarrow$occlude($x,w_{size},i$)  7:        $X_{occ}\leftarrow (X\setminus \left\{x\right\})\cup \left\{x_{occ}\right\}$  8:        $y_{occ}\leftarrow$predict($X_{occ},model$)  9:        $y\leftarrow$predict($X,model$) 10:        $ϵ\leftarrow$mae($y,y_{occ}$) 11:        Append $(ϵ,s^i)$ to $mae\_signal$ 12:    end for 13:    $mae\_signal\leftarrow$ReverseSort($mae\_signal$) 14:    for $j\leftarrow 1,k$ do 15:        Append $mae\_signal.get\left(i\right)\left[1\right]$ to $subsignals$ 16:    end for 17:    return $subsignals$ 18:end procedure