Algorithm 2 Contrastive Sample Identification (CSI) Task

Require: Positive video features $V^{+}$, Positive text features $T^{+}$, Negative text features $T^{-}$, Negative video features $V^{-}$ Ensure: Binary classification of fused features as matching or mismatching   1: function FindMostSimilarNegativeSample($V^{+}$, $T^{-}$, $T^{+}$)   2:     $similarit{y}_{max}\leftarrow -\infty$   3:     $neg\_sampl{e}_{max}\leftarrow \mathrm{null}$   4:     for each $t^{-}$ in $T^{-}$ do   5:         $similarity\leftarrow \mathrm{Similarity}(V^{+},t^{-})$   6:         if $similarity>similarit{y}_{max}$ then   7:            $similarit{y}_{max}\leftarrow similarity$   8:            $neg\_sampl{e}_{max}\leftarrow t^{-}$   9:         end if 10:     end for 11:     return $neg\_sampl{e}_{max}$ 12: end function 13: function FuseFeatures(v, t) 14:     $t_{enc}\leftarrow$ AddEncToken(t)                                    ▹ Prepend ‘enc_token’ to text feature 15:     $f\leftarrow \mathrm{BERT}(t_{enc},v)$            ▹ Input text to BERT and video feature to Cross Attention 16:     return f                                                ▹ Output from BERT corresponding to enc_token 17: end function 18: function ContrastiveSampleIdentification($V^{+}$, $T^{+}$, $T^{-}$, $V^{-}$) 19:     $P_{match}\leftarrow \left[\right]$ 20:     $P_{mismatch}\leftarrow \left[\right]$ 21:     $neg\_text\leftarrow \mathrm{FindMostSimilarNegativeSample}(V^{+},T^{-})$ 22:     $neg\_video\leftarrow \mathrm{FindMostSimilarNegativeSample}(T^{+},V^{-})$ 23:     $fuse{d}_{match}\leftarrow \mathrm{FuseFeatures}(V^{+},T^{+})$ 24:     $fuse{d}_{mismatch\_text}\leftarrow \mathrm{FuseFeatures}(V^{+},neg\_text)$ 25:     $fuse{d}_{mismatch\_video}\leftarrow \mathrm{FuseFeatures}(neg\_video,T^{+})$ 26:     Append $fuse{d}_{match}$ to $P_{match}$ 27:     Append $fuse{d}_{mismatch\_text}$ and $fuse{d}_{mismatch\_video}$ to $P_{mismatch}$ 28:     for each $fused\_feature$ in $P_{match}\cup P_{mismatch}$ do 29:         $classification\leftarrow \mathrm{Classifier}(fused\_feature)$ 30:         if $fused\_feature$ is in $P_{match}$ then 31:            Expect $classification$ to be 1 32:         else 33:            Expect $classification$ to be 0 34:         end if 35:     end for 36: end function