#!/bin/sh -x
#
# Script name: ScriptB_Mapping-DNAm-Metrics_bam.sh
#
# Purpose of script: Mapping, Methylation, PCR duplicate and OnTarget rates, generation of bam files with or without PCR duplicates
#
# Date Created: 2022/12/29
#
# Written by Keita Hasegawa and Kazuhiko Nakabayashi for the following study:
#
# Keita Hasegawa et al.
# "A capture methyl-seq protocol with improved efficiency and cost-effectiveness using pre-pooling and enzymatic conversion"
# 
# 
# Contact information:
# Kazuhiko Nakabayashi, Ph.D.
# nakabaya-k@ncchd.go.jp 
# Laboratory of Developmental Genomics, Research Institute, National Center for Child Health and Development, Tokyo, Japan
#
#
# Required Files in same working directory
# #1: this script
# #2: paired-end fastq files= "$sample"_R1_*.fastq.gz and "$sample"_R2_*.fastq.gz
# #3: bait file(tab-delimited bed file)
#   #3-1 file name = bait_*.bed (example: bait_EPIC.bed)
#   #3-2 contain only first 3 columns (Chromosome StartPosition EndPosition)
#   #3-3 include header (exapmle: chr start end)
#   #3-4 include "chr" in 1st clumn (example: chr1)
#
#
# Usage
# ./this_script $1 $2
# specify genome version at $1 in the command line to run this script
# specify genome version for IGVTools at $2 (use "hg19" for hs37d5)
# below is an example
# ./this_script hs37d5 hg19
#
#
# Output 7 Directories and 4 Files in Directory of "$sample"
# Dir1: Step1_trimgalore
#  File01: *.fastq.gz_trimming_report.txt
#  File02: stderr and stdout
#  Dir1-1: fastqc
#    File01: *fastqc.html
#    File02: *fastqc.zip
# Dir2: Step2_bismark_hisat2
#  File01: *bismark_hisat2_PE_hs37d5_report.txt
#  File02: stderr and stdout
# Dir3: Step3_bismark_meth_extractor (include both with duplication version and without duplication version)
#  File01: CpG_context_* 
#  File02: *.M-bias.txt
#  File03: *.bedGraph.gz
#  File04: *.bismark.cov.gz
#  File05: *_splitting_report.txt
#  File06: stderr and stdout
# Dir4: Step4_methylKit (include both with duplication version and without duplication version)
#  File01: *.CpG_report.txt
#  File02: *_methylKit.txt
# Dir5: Step5_bam_tdf
#  File01: *_bismark_hisat2_pe_*.bam (with duplication)
#  File02: *_bismark_hisat2_pe_*_sort.bam (with duplication)
#  File03: *_bismark_hisat2_pe_*_sort.bam.bai (with duplication)
#  File04: *_bismark_hisat2_pe_*_nsort_nodup.bam (without duplication)
#  File05: *_bismark_hisat2_pe_*_sort_nodup.bam.bai (without duplication)
#  File06: *_final_mapped.bam (without duplication)
#  File07: *_final_mapped.bam.bai (without duplication)
#  File08: *_final.bed
#  File09: offtarget_read_*.bed
#  File10: ontarget_read_*.bed
#  File11: how_to_remake_erased_bam_files.txt
# Dir6: Step6_result
#  File01: mapping_rate.txt
#  File02: Result_MapMethOnt_${sample}.txt
# Dir7: "Step7_fastq_files"
#  File01: original paired-end fastq files
#  File02: fastq_files_removed.txt
#  File03: input_fastq_files.txt
#
# File1: command_options.txt
# File2: flagstat.txt
# File3: mapping_summaries.txt
# File4: out.txt and err.txt
#
#
# Definition of Ontarget Rate
# # Ontarget rate = (ontarget reads) / (total reads = (ontarget reads) + (offtarget reads) )
#

# This script requires the following tools:
export PATH=/home/xxx/yyyy/TrimGalore-0.6.6/:$PATH    # Add the path to TrimGalore
export PATH=/home/xxx/yyyy/cutadapt-3.2/:$PATH        # Add the path to cutadapt
export PATH=/home/xxx/yyyy/FastQC-0.11.9/:$PATH       # Add the path to FastQC
export PATH=/home/xxx/yyyy/hisat2-2.1.0/:$PATH        # Add the path to hisat2
export PATH=/home/xxx/yyyy/samtools-1.12/bin/:$PATH   # Add the path to samtools
export PATH=/home/xxx/yyyy/Bismark-0.22.3/:$PATH      # Add the path to Bismark
export PATH=/home/xxx/yyyy/IGVTools/:$PATH            # Add the path to IGVTools
export PATH=/home/xxx/yyyy/bedtools-2.30.0/bin:$PATH  # Add the path to bedtools
export PATH=/home/xxx/yyyy/R/R-4.1.3/bin:$PATH        # Add the path to R

# This script requires a reference genome fasta file and bwa indexes for bwa "samse" procesure:
# Specify the path to hg19 reference fasta indexes (/home/xxx/yyyy/zzzzzzzz/) and its prefix (hg19.fa)
genome=/home/xxx/yyyy/zzzzzzzz/hg19.fa                


#  Specify the parameters and names below before starting run this script
sample=`basename $PWD`

# Please refer to the following website to prepare the Bisulfite_Genome for hisat2
# https://felixkrueger.github.io/Bismark/options/genome_preparation/
genome=/path/to/the/directory/containing/Bisulfite_Genome

## Analysis of Mapping Rate, Methylation Rate and PCR duplicate Rate (twelve Steps) ##
date
ls -1 *.sh > command_options.txt
echo '$1=' $1 >> command_options.txt
echo '$2=' $2 >> command_options.txt
ls -l *fastq* > input_fastq_files.txt

# Map-Step1: combine and rename fastq.gz
cat *_R1*.fastq.gz > ${sample}_R1.fastq.gz
cat *_R2*.fastq.gz > ${sample}_R2.fastq.gz


# Map-Step2: Adapter trimming by using "TrimGalore"
trim_galore \
  --quality 20 \
  --phred33 \
  --fastqc \
  --illumina \
  --stringency 1 \
  -e 0.1 \
  --length 20 \
  --paired ${sample}_R1.fastq.gz ${sample}_R2.fastq.gz \
  > trim_galore.stdout 2> trim_galore.stderr
## ${samaple}_R1_val_1.fq.gz and {samaple}_R2_val_2.fq.gz will be produced ##
mv ${sample}_R1_val_1.fq.gz ${sample}_val_1.fq.gz
mv ${sample}_R2_val_2.fq.gz ${sample}_val_2.fq.gz


# Map-Step3: Mapping pair-end reads by using "Bismark"
bismark \
  --hisat2 -p 2 \
  --non_directional $genome \
  -1 ${sample}_val_1.fq.gz -2 ${sample}_val_2.fq.gz \
  > bismark.stdout 2> bismark.stderr
# ${sample}_val_1_bismark_hisat2_pe.bam and ${sample}_val_1_bismark_hisat2_PE_report.txt will be produced
mv ${sample}_val_1_bismark_hisat2_pe.bam ${sample}_bismark_hisat2_pe_$1.bam
mv ${sample}_val_1_bismark_hisat2_PE_report.txt ${sample}_bismark_hisat2_PE_$1_report.txt


# Map-Step4: bismark_meth_call with PCR duplication
bismark_methylation_extractor -p --bedGraph --comprehensive \
--merge_non_CpG --multicore 8 \
${sample}_bismark_hisat2_pe_$1.bam > extractor.stdout 2> extractor.stderr
#.bedGraph.gz and .bismark.cov.gz will be produced
coverage2cytosine --coverage_threshold 1 --genome_folder $genome \
 ${sample}_bismark_hisat2_pe_$1.bismark.cov.gz -o ${sample}_$1


# Map-Step5: Generate methylKit.txt without removing PCR duplication
echo -e "chrBase\\tchr\\tbase\\tstrand\\tcoverage\\tfreqC\\tfreqT" > ${sample}_$1_methylKit.txt
cat ${sample}_$1.CpG_report.txt | \
awk 'BEGIN{OFS="\t"}$3=="+"{print $1"."$2,$1,$2,"F",$4+$5,100*$4/($4+$5),100*$5/($4+$5)}$3=="-"{print $1"."$2,$1,$2,"R",$4+$5,100*$4/($4+$5),100*$5/($4+$5)}' | \
sort -V -k2,2 -k3,3 >> ${sample}_$1_methylKit.txt


# Map-Step6: samtools to fixmate, sort, mark and remove duplicates; igvtools to generate tdf files
samtools fixmate -m ${sample}_bismark_hisat2_pe_$1.bam tmp_fixmate.bam
samtools sort tmp_fixmate.bam -o ${sample}_bismark_hisat2_pe_$1_sort.bam
samtools markdup -r ${sample}_bismark_hisat2_pe_$1_sort.bam ${sample}_bismark_hisat2_pe_$1_sort_nodup.bam
samtools index ${sample}_bismark_hisat2_pe_$1_sort.bam
samtools index ${sample}_bismark_hisat2_pe_$1_sort_nodup.bam
igvtools count -z 5 -w 25 ${sample}_bismark_hisat2_pe_$1_sort.bam ${sample}_bismark_hisat2_pe_$1_sort.bam.tdf $2
igvtools count -z 5 -w 25 ${sample}_bismark_hisat2_pe_$1_sort_nodup.bam ${sample}_bismark_hisat2_pe_$1_sort_nodup.bam.tdf $2


# Map-Step7: bismark_meth_call without PCR duplication
samtools sort -n ${sample}_bismark_hisat2_pe_$1_sort_nodup.bam -O bam -o ${sample}_bismark_hisat2_pe_$1_nsort_nodup.bam
bismark_methylation_extractor -p --bedGraph --comprehensive --merge_non_CpG --multicore 8 \
${sample}_bismark_hisat2_pe_$1_nsort_nodup.bam > extractor2.stdout 2> extractor2.stderr
#.bedGraph.gz and .bismark.cov.gz will be produced
coverage2cytosine --coverage_threshold 1 --genome_folder $genome \
${sample}_bismark_hisat2_pe_$1_nsort_nodup.bismark.cov.gz -o ${sample}_nodup_$1


# Map-Step8: Generate methylKit.txt without PCR duplication
echo -e "chrBase\\tchr\\tbase\\tstrand\\tcoverage\\tfreqC\\tfreqT" > ${sample}_nodup_$1_methylKit.txt
cat ${sample}_nodup_$1.CpG_report.txt | \
awk 'BEGIN{OFS="\t"}$3=="+"{print $1"."$2,$1,$2,"F",$4+$5,100*$4/($4+$5),100*$5/($4+$5)}$3=="-"{print $1"."$2,$1,$2,"R",$4+$5,100*$4/($4+$5),100*$5/($4+$5)}' | \
sort -V -k2,2 -k3,3 >> ${sample}_nodup_$1_methylKit.txt


# Map-Step9: Generate "mapping_summaries.txt" and "flatstat.txt"
rm mapping_summaries.txt
touch mapping_summaries.txt
echo -e "Mapping summaries for $sample\n" >> mapping_summaries.txt
echo -e "=== script(.sh) name:\t ` ls -1d *.sh `\n" >> mapping_summaries.txt
echo "=== original fastq/fq files:" >> mapping_summaries.txt
ls -1d *fastq *fq *fastq.gz *fq.gz >> mapping_summaries.txt
echo "" >> mapping_summaries.txt
echo "=== cutadapt summary ==="  >> mapping_summaries.txt
cat ${sample}_R1.fastq*_trimming_report.txt | grep Input >> mapping_summaries.txt
echo "" >> mapping_summaries.txt
cat ${sample}_R1.fastq*_trimming_report.txt | grep  -A 7 "Total reads processed:" >> mapping_summaries.txt
echo "= = = = = " >> mapping_summaries.txt
cat ${sample}_R2.fastq*_trimming_report.txt | grep Input >> mapping_summaries.txt
echo "" >> mapping_summaries.txt
cat ${sample}_R2.fastq*_trimming_report.txt | grep  -A 7 "Total reads processed:" >> mapping_summaries.txt
echo "=== hisat2 summary ==="  >> mapping_summaries.txt
echo -e "genome index path: $genome " >> mapping_summaries.txt
echo "`cat bismark.stderr | awk '/>>> Writing bisulfite mapping results/,/Processed/'`" >> mapping_summaries.txt
echo "" >> mapping_summaries.txt

echo "### samtools flagstat ${sample}_bismark_hisat2_pe_$1_sort.bam  ###" >> flagstat.txt
samtools flagstat ${sample}_bismark_hisat2_pe_$1_sort.bam  >> flagstat.txt
echo "" >> flagstat.txt

echo "### samtools flagstat ${sample}_bismark_hisat2_pe_$1_sort_nodup.bam ###" >> flagstat.txt
samtools flagstat ${sample}_bismark_hisat2_pe_$1_sort_nodup.bam >> flagstat.txt
echo "" >> flagstat.txt


# Map-Step10: clean-up
mkdir Step7_fastq_files Step1_trimgalore Step2_bismark_hisat2 Step3_bismark_meth_extractor Step4_methylKit Step5_bam_tdf Step6_result

rm ${sample}_*val_?.fq*
mv ${sample}_R[12].fastq.gz input_fastq_files.txt Step7_fastq_files
ls -l *fq.gz *fastq.gz *fq *fastq > fastq_files_removed.txt;  rm *fq.gz *fastq.gz *fq *fastq
mv fastq_files_removed.txt Step7_fastq_files

mv ${sample}_*_trimming_report.txt trim_galore.std* Step1_trimgalore
mkdir fastqc; mv ${sample}_R[12]_val_[12]_fastqc.* fastqc
mv fastqc Step1_trimgalore

mv ${sample}_bismark_hisat2_PE_$1_report.txt bismark.std* Step2_bismark_hisat2

mv *.bismark.cov.gz *.bedGraph.gz Step3_bismark_meth_extractor
rm Non_CpG_context_${sample}_bismark_hisat2_pe_$1*.txt #keep this file if you need non-CG analysis
mv CpG_context_${sample}_bismark_hisat2_pe_$1*.txt extractor*.std* Step3_bismark_meth_extractor
mv ${sample}_bismark_hisat2_pe_$1*.txt Step3_bismark_meth_extractor #M-bias, splitting_report

mv ${sample}*_$1_methylKit.txt Step4_methylKit  # two files
mv ${sample}*_$1.CpG_report.txt Step4_methylKit # two files

ls -l *.bam* >  how_to_remake_erased_bam_files.txt; echo "" >> how_to_remake_erased_bam_files.txt
mv ${sample}_bismark_hisat2_pe_$1* Step5_bam_tdf
mv Step5_bam_tdf/${sample}_bismark_hisat2_pe_$1_sort_nodup.bam Step5_bam_tdf/${sample}_final_mapped.bam
rm tmp_fixmate.bam
cat *.sh | grep ^samtools | grep -v flagstat >> how_to_remake_erased_bam_files.txt
mv how_to_remake_erased_bam_files.txt Step5_bam_tdf

rm -rf igv.log


# Map-Step11: mapping_rate.txt
echo date > tmp.txt ; echo $PWD >> tmp.txt ; echo ${sample}_bismark_hisat2_pe_$1 >> tmp.txt; echo "" >> tmp.txt

read_pairs_in_fastq=`cat mapping_summaries.txt | grep "Total reads processed" | head -1 | awk '{print $4}' |  sed 's/,//g' `
read_pairs_after_QC=` cat mapping_summaries.txt | grep "Reads written (passing filters)" | head -1 | rev | cut -d ' ' -f2 | rev | sed 's/,//g' `
read_pairs_mapped=` cat flagstat.txt | grep read1 | head -1 | cut -d' ' -f1 `
read_pairs_nodup=` cat flagstat.txt | grep read1 | tail -1 | cut -d' ' -f1 `
echo "read_pairs_in_fastq:" $read_pairs_in_fastq >> tmp.txt
echo "read_pairs_after_QC:" $read_pairs_after_QC >> tmp.txt
echo "read_pairs_mapped:  " $read_pairs_mapped   >> tmp.txt
echo "read_pairs_nodup:   " $read_pairs_nodup    >> tmp.txt
echo "" >> tmp.txt
echo "" | awk -v nodup="$read_pairs_nodup" -v mapped="$read_pairs_mapped"       '{print "PCR_duplicate_rate  = "(mapped-nodup)/mapped}' >> tmp.txt
echo "" >> tmp.txt
echo "" | awk -v mapped="$read_pairs_mapped" -v QCpassed="$read_pairs_after_QC" '{print "mapping_rate_withdup= "mapped/QCpassed}' >> tmp.txt
echo "" | awk -v nodup="$read_pairs_nodup"   -v QCpassed="$read_pairs_after_QC" '{print "mapping_rate_nodup  = "nodup/QCpassed}'  >> tmp.txt

mv tmp.txt mapping_rate.txt
mv mapping_rate.txt Step6_result


# Map-Step12: Specify variables
 Mapping_rate_with_dup=`cat Step6_result/mapping_rate.txt | grep 'mapping_rate_withdup' | sed 's/ //g' | sed 's/=/\t/g' | cut -f2`
 Mapping_rate_nodup=`cat Step6_result/mapping_rate.txt | grep 'mapping_rate_nodup' | sed 's/ //g' | sed 's/=/\t/g' | cut -f2`

 CpG_methyl=`cat Step2_bismark_hisat2/${sample}_bismark_hisat2_PE_$1_report.txt | grep 'C methylated in CpG context' | sed 's/ //g' | sed 's/:/\t/g' | sed 's/%//g' | awk '{print $2/100}' | cut -f1`
 CHG_methyl=`cat Step2_bismark_hisat2/${sample}_bismark_hisat2_PE_$1_report.txt | grep 'C methylated in CHG context' | sed 's/ //g' | sed 's/:/\t/g' | sed 's/%//g' | awk '{print $2/100}' | cut -f1`
 CHH_methyl=`cat Step2_bismark_hisat2/${sample}_bismark_hisat2_PE_$1_report.txt | grep 'C methylated in CHH context' | sed 's/ //g' | sed 's/:/\t/g' | sed 's/%//g' | awk '{print $2/100}' | cut -f1`

 PCR_dup_rate=`cat Step6_result/mapping_rate.txt | grep 'PCR_duplicate_rate' | sed 's/ //g' | sed 's/=/\t/g' | cut -f2`


## Analysis of Ontarget Rate (seven Steps) ##

# ONT-Step1: Specify the directory include target file and Move into Step5_bam_tdf
DIRBAM=Step5_bam_tdf
cd "$DIRBAM"


# ONT-Step2: generate "$sample"_final_mapped.bam.bai by using samtools
samtools index "$sample"_final_mapped.bam


# ONT-Step3: edit bait_*.bed and remove "chr" from $1
cat ../../../bait_* | sed -e '1d' | sed 's/chr//g' | awk '{print $1,$2,$3}' | sed 's/ /\t/g' > bait_nochr.bed


# ONT-Step4: generate "$sample"_final.bed from "$sample"_final_mapped.bam by usin bedtools
bedtools bamtobed -i "$sample"_final_mapped.bam > "$sample"_final_0.bed
cat "$sample"_final_0.bed | awk '{print $1,$2,$3}' | sed 's/ /\t/g' > "$sample"_final.bed


# ONT-Step5: calculate Ontarget Rate and OffTarget Rate by usin bedtools
bedtools intersect -wa -u -a "$sample"_final.bed -b bait_nochr.bed > ontarget_read_"$sample".bed
bedtools intersect -v -a "$sample"_final.bed -b bait_nochr.bed > offtarget_read_"$sample".bed


# ONT-Step6: Specify variables
Total_reads=`wc -l "$sample"_final.bed | awk '{print $1}' | cut -f1`
Ontarget_reads=`wc -l ontarget_read_"$sample".bed | awk '{print $1}' | cut -f1`
Offtarget_reads=`wc -l offtarget_read_"$sample".bed | awk '{print $1}' | cut -f1`

Ontarget_rate=`awk "BEGIN { print $Ontarget_reads/$Total_reads }" | cut -f1`
Offtarget_rate=`awk "BEGIN { print $Offtarget_reads/$Total_reads }" | cut -f1`


# ONT-Step7: clean-up
rm -rf bait_nochr.bed
rm -rf "$sample"_final_0.bed
cd ../


## The Last Step: Write the MappingRate,MethylationRate, PCR duplicationRate and OntargetRate to Result_MapMethOnt_${sample}.txt in Step6_result ##
echo \
sample_name,\
read_pairs_in_fastq,read_pairs_after_QC,read_pairs_mapped,read_pairs_nodup,PCR_dup_rate,\
Mapping_rate_with_dup,Mapping_rate_nodup,\
CpG_methyl,CHG_methyl,CHH_methyl,\
Ontarget_rate,Offtarget_rate,Total_reads,Ontarget_reads,Offtarget_reads\
| sed 's/,/\t/g' > Step6_result/Result_MapMethOnt_${sample}.txt

echo \
"\
"$sample",\
"$read_pairs_in_fastq","$read_pairs_after_QC","$read_pairs_mapped","$read_pairs_nodup","$PCR_dup_rate",\
"$Mapping_rate_with_dup","$Mapping_rate_nodup",\
"$CpG_methyl","$CHG_methyl","$CHH_methyl",\
"$Ontarget_rate","$Offtarget_rate","$Total_reads","$Ontarget_reads","$Offtarget_reads"\
"\
| sed 's/ //g' | sed 's/,/\t/g' >> Step6_result/Result_MapMethOnt_${sample}.txt


date

exit
