#!/bin/bash
#SBATCH -D /some-path/results/BatchXX/work/e4/3dfbede5151f34f8e16c805ca407ae
#SBATCH -J nf-NFCORE_VIRALRECON_ILLUMINA_PICARD_COLLECTMULTIPLEMETRICS_(4142023023_S11)
#SBATCH -o /some-path/results/BatchXX/work/e4/3dfbede5151f34f8e16c805ca407ae/.command.log
#SBATCH --no-requeue
#SBATCH --signal B:USR2@30
#SBATCH -t 04:00:00
#SBATCH --mem 6144M
#SBATCH -p normal
# NEXTFLOW TASK: NFCORE_VIRALRECON:ILLUMINA:PICARD_COLLECTMULTIPLEMETRICS (4142023023_S11)
set -e
set -u
NXF_DEBUG=${NXF_DEBUG:=0}; [[ $NXF_DEBUG > 1 ]] && set -x
NXF_ENTRY=${1:-nxf_main}

nxf_tree() {
    local pid=$1

    declare -a ALL_CHILDREN
    while read P PP;do
        ALL_CHILDREN[$PP]+=" $P"
    done < <(ps -e -o pid= -o ppid=)

    pstat() {
        local x_pid=$1
        local STATUS=$(2> /dev/null < /proc/$1/status egrep 'Vm|ctxt')

        if [ $? = 0 ]; then
        local  x_vsz=$(echo "$STATUS" | grep VmSize | awk '{print $2}' || echo -n '0')
        local  x_rss=$(echo "$STATUS" | grep VmRSS | awk '{print $2}' || echo -n '0')
        local x_peak=$(echo "$STATUS" | egrep 'VmPeak|VmHWM' | sed 's/^.*:\s*//' | sed 's/[\sa-zA-Z]*$//' | tr '\n' ' ' || echo -n '0 0')
        local x_pmem=$(awk -v rss=$x_rss -v mem_tot=$mem_tot 'BEGIN {printf "%.0f", rss/mem_tot*100*10}' || echo -n '0')
        local vol_ctxt=$(echo "$STATUS" | grep '\bvoluntary_ctxt_switches' | awk '{print $2}' || echo -n '0')
        local inv_ctxt=$(echo "$STATUS" | grep '\bnonvoluntary_ctxt_switches' | awk '{print $2}' || echo -n '0')
        cpu_stat[x_pid]="$x_pid $x_pmem $x_vsz $x_rss $x_peak $vol_ctxt $inv_ctxt"
        fi
    }

    pwalk() {
        pstat $1
        for i in ${ALL_CHILDREN[$1]:=}; do pwalk $i; done
    }

    pwalk $1
}

nxf_stat() {
    cpu_stat=()
    nxf_tree $1

    declare -a sum=(0 0 0 0 0 0 0 0)
    local pid
    local i
    for pid in "${!cpu_stat[@]}"; do
        local row=(${cpu_stat[pid]})
        [ $NXF_DEBUG = 1 ] && echo "++ stat mem=${row[*]}"
        for i in "${!row[@]}"; do
        if [ $i != 0 ]; then
            sum[i]=$((sum[i]+row[i]))
        fi
        done
    done

    [ $NXF_DEBUG = 1 ] && echo -e "++ stat SUM=${sum[*]}"

    for i in {1..7}; do
        if [ ${sum[i]} -lt ${cpu_peak[i]} ]; then
            sum[i]=${cpu_peak[i]}
        else
            cpu_peak[i]=${sum[i]}
        fi
    done

    [ $NXF_DEBUG = 1 ] && echo -e "++ stat PEAK=${sum[*]}\n"
    nxf_stat_ret=(${sum[*]})
}

nxf_mem_watch() {
    set -o pipefail
    local pid=$1
    local trace_file=.command.trace
    local count=0;
    declare -a cpu_stat=(0 0 0 0 0 0 0 0)
    declare -a cpu_peak=(0 0 0 0 0 0 0 0)
    local mem_tot=$(< /proc/meminfo grep MemTotal | awk '{print $2}')
    local timeout
    local DONE
    local STOP=''

    [ $NXF_DEBUG = 1 ] && nxf_sleep 0.2 && ps fx

    while true; do
        nxf_stat $pid
        if [ $count -lt 10 ]; then timeout=1;
        elif [ $count -lt 120 ]; then timeout=5;
        else timeout=30;
        fi
        read -t $timeout -r DONE || true
        [[ $DONE ]] && break
        if [ ! -e /proc/$pid ]; then
            [ ! $STOP ] && STOP=$(nxf_date)
            [ $(($(nxf_date)-STOP)) -gt 10000 ] && break
        fi
        count=$((count+1))
    done

    echo "%mem=${nxf_stat_ret[1]}"      >> $trace_file
    echo "vmem=${nxf_stat_ret[2]}"      >> $trace_file
    echo "rss=${nxf_stat_ret[3]}"       >> $trace_file
    echo "peak_vmem=${nxf_stat_ret[4]}" >> $trace_file
    echo "peak_rss=${nxf_stat_ret[5]}"  >> $trace_file
    echo "vol_ctxt=${nxf_stat_ret[6]}"  >> $trace_file
    echo "inv_ctxt=${nxf_stat_ret[7]}"  >> $trace_file
}

nxf_write_trace() {
    echo "nextflow.trace/v2"           > $trace_file
    echo "realtime=$wall_time"         >> $trace_file
    echo "%cpu=$ucpu"                  >> $trace_file
    echo "rchar=${io_stat1[0]}"        >> $trace_file
    echo "wchar=${io_stat1[1]}"        >> $trace_file
    echo "syscr=${io_stat1[2]}"        >> $trace_file
    echo "syscw=${io_stat1[3]}"        >> $trace_file
    echo "read_bytes=${io_stat1[4]}"   >> $trace_file
    echo "write_bytes=${io_stat1[5]}"  >> $trace_file
}

nxf_trace_mac() {
    local start_millis=$(nxf_date)

    /bin/bash -euo pipefail /some-path/results/BatchXX/work/e4/3dfbede5151f34f8e16c805ca407ae/.command.sh

    local end_millis=$(nxf_date)
    local wall_time=$((end_millis-start_millis))
    local ucpu=''
    local io_stat1=('' '' '' '' '' '')
    nxf_write_trace
}

nxf_fd() {
    local FD=11
    while [ -e /proc/$$/fd/$FD ]; do FD=$((FD+1)); done
    echo $FD
}

nxf_trace_linux() {
    local pid=$$
    command -v ps &>/dev/null || { >&2 echo "Command 'ps' required by nextflow to collect task metrics cannot be found"; exit 1; }
    local num_cpus=$(< /proc/cpuinfo grep '^processor' -c)
    local tot_time0=$(grep '^cpu ' /proc/stat | awk '{sum=$2+$3+$4+$5+$6+$7+$8+$9; printf "%.0f",sum}')
    local cpu_time0=$(2> /dev/null < /proc/$pid/stat awk '{printf "%.0f", ($16+$17)*10 }' || echo -n 'X')
    local io_stat0=($(2> /dev/null < /proc/$pid/io sed 's/^.*:\s*//' | head -n 6 | tr '\n' ' ' || echo -n '0 0 0 0 0 0'))
    local start_millis=$(nxf_date)
    trap 'kill $mem_proc' ERR
    
    /bin/bash -euo pipefail /some-path/results/BatchXX/work/e4/3dfbede5151f34f8e16c805ca407ae/.command.sh &
    local task=$!

    mem_fd=$(nxf_fd)
    eval "exec $mem_fd> >(nxf_mem_watch $task)"
    local mem_proc=$!

    wait $task

    local end_millis=$(nxf_date)
    local tot_time1=$(grep '^cpu ' /proc/stat | awk '{sum=$2+$3+$4+$5+$6+$7+$8+$9; printf "%.0f",sum}')
    local cpu_time1=$(2> /dev/null < /proc/$pid/stat awk '{printf "%.0f", ($16+$17)*10 }' || echo -n 'X')
    local ucpu=$(awk -v p1=$cpu_time1 -v p0=$cpu_time0 -v t1=$tot_time1 -v t0=$tot_time0 -v n=$num_cpus 'BEGIN { pct=(p1-p0)/(t1-t0)*100*n; printf("%.0f", pct>0 ? pct : 0) }' )

    local io_stat1=($(2> /dev/null < /proc/$pid/io sed 's/^.*:\s*//' | head -n 6 | tr '\n' ' ' || echo -n '0 0 0 0 0 0'))
    local i
    for i in {0..5}; do
        io_stat1[i]=$((io_stat1[i]-io_stat0[i]))
    done

    local wall_time=$((end_millis-start_millis))
    [ $NXF_DEBUG = 1 ] && echo "+++ STATS %CPU=$ucpu TIME=$wall_time I/O=${io_stat1[*]}"

    echo "nextflow.trace/v2"           > $trace_file
    echo "realtime=$wall_time"         >> $trace_file
    echo "%cpu=$ucpu"                  >> $trace_file
    echo "rchar=${io_stat1[0]}"        >> $trace_file
    echo "wchar=${io_stat1[1]}"        >> $trace_file
    echo "syscr=${io_stat1[2]}"        >> $trace_file
    echo "syscw=${io_stat1[3]}"        >> $trace_file
    echo "read_bytes=${io_stat1[4]}"   >> $trace_file
    echo "write_bytes=${io_stat1[5]}"  >> $trace_file

    [ -e /proc/$mem_proc ] && eval "echo 'DONE' >&$mem_fd" || true
    wait $mem_proc 2>/dev/null || true
    while [ -e /proc/$mem_proc ]; do nxf_sleep 0.1; done
}

nxf_trace() {
    local trace_file=.command.trace
    touch $trace_file
    if [[ $(uname) = Darwin ]]; then
        nxf_trace_mac
    else
        nxf_trace_linux
    fi
}
nxf_container_env() {
cat << EOF
export PYTHONNOUSERSITE="1"
export R_PROFILE_USER="/.Rprofile"
export R_ENVIRON_USER="/.Renviron"
export JULIA_DEPOT_PATH="/usr/local/share/julia"
export PATH="\$PATH:/home-dir/nextflow/assets/nf-core/viralrecon/bin"
EOF
}

nxf_sleep() {
  sleep $1 2>/dev/null || sleep 1;
}

nxf_date() {
    local ts=$(date +%s%3N);
    if [[ ${#ts} == 10 ]]; then echo ${ts}000
    elif [[ $ts == *%3N ]]; then echo ${ts/\%3N/000}
    elif [[ $ts == *3N ]]; then echo ${ts/3N/000}
    elif [[ ${#ts} == 13 ]]; then echo $ts
    else echo "Unexpected timestamp value: $ts"; exit 1
    fi
}

nxf_env() {
    echo '============= task environment ============='
    env | sort | sed "s/\(.*\)AWS\(.*\)=\(.\{6\}\).*/\1AWS\2=\3xxxxxxxxxxxxx/"
    echo '============= task output =================='
}

nxf_kill() {
    declare -a children
    while read P PP;do
        children[$PP]+=" $P"
    done < <(ps -e -o pid= -o ppid=)

    kill_all() {
        [[ $1 != $$ ]] && kill $1 2>/dev/null || true
        for i in ${children[$1]:=}; do kill_all $i; done
    }

    kill_all $1
}

nxf_mktemp() {
    local base=${1:-/tmp}
    mkdir -p "$base"
    if [[ $(uname) = Darwin ]]; then mktemp -d $base/nxf.XXXXXXXXXX
    else TMPDIR="$base" mktemp -d -t nxf.XXXXXXXXXX
    fi
}

nxf_fs_copy() {
  local source=$1
  local target=$2
  local basedir=$(dirname $1)
  mkdir -p $target/$basedir
  cp -fRL $source $target/$basedir
}

nxf_fs_move() {
  local source=$1
  local target=$2
  local basedir=$(dirname $1)
  mkdir -p $target/$basedir
  mv -f $source $target/$basedir
}

nxf_fs_rsync() {
  rsync -rRl $1 $2
}

on_exit() {
    exit_status=${nxf_main_ret:=$?}
    printf $exit_status > /some-path/results/BatchXX/work/e4/3dfbede5151f34f8e16c805ca407ae/.exitcode
    set +u
    [[ "$tee1" ]] && kill $tee1 2>/dev/null
    [[ "$tee2" ]] && kill $tee2 2>/dev/null
    [[ "$ctmp" ]] && rm -rf $ctmp || true
    rm -rf $NXF_SCRATCH || true
    exit $exit_status
}

on_term() {
    set +e
    [[ "$pid" ]] && kill $pid 2>/dev/null
}

nxf_launch() {
    set +u; env - PATH="$PATH" ${TMP:+SINGULARITYENV_TMP="$TMP"} ${TMPDIR:+SINGULARITYENV_TMPDIR="$TMPDIR"} SINGULARITYENV_NXF_DEBUG=${NXF_DEBUG:=0} singularity exec -B /home -B "$PWD" /home/groups/hpcbio/singularity/depot.galaxyproject.org-singularity-picard-3.0.0--hdfd78af_1.img /bin/bash -c "cd $PWD; eval $(nxf_container_env); /bin/bash /some-path/results/BatchXX/work/e4/3dfbede5151f34f8e16c805ca407ae/.command.run nxf_trace"
}

nxf_stage() {
    true
    # stage input files
    rm -f 4142023023_S11.ivar_trim.sorted.bam
    rm -f 4142023023_S11.ivar_trim.sorted.bam.bai
    rm -f GCA_009858895.3_ASM985889v3_genomic.200409.fna
    ln -s /some-path/results/BatchXX/work/a8/7fb5257241e84b09d89b2d6ef8b03c/4142023023_S11.ivar_trim.sorted.bam 4142023023_S11.ivar_trim.sorted.bam
    ln -s /some-path/results/BatchXX/work/9e/59e431efecadc98b69ec79a3bc4650/4142023023_S11.ivar_trim.sorted.bam.bai 4142023023_S11.ivar_trim.sorted.bam.bai
    ln -s /some-path/results/BatchXX/work/11/ef8c03b00d2930c0a7b945c318cbb3/GCA_009858895.3_ASM985889v3_genomic.200409.fna GCA_009858895.3_ASM985889v3_genomic.200409.fna
}

nxf_unstage() {
    true
    cp .command.out /some-path/results/BatchXX/work/e4/3dfbede5151f34f8e16c805ca407ae/.command.out || true
    cp .command.err /some-path/results/BatchXX/work/e4/3dfbede5151f34f8e16c805ca407ae/.command.err || true
    cp .command.trace /some-path/results/BatchXX/work/e4/3dfbede5151f34f8e16c805ca407ae/.command.trace || true
    [[ ${nxf_main_ret:=0} != 0 ]] && return
    IFS=$'\n'
    for name in $(eval "ls -1d *_metrics *.pdf versions.yml" | sort | uniq); do
        nxf_fs_copy "$name" /some-path/results/BatchXX/work/e4/3dfbede5151f34f8e16c805ca407ae || true
    done
    unset IFS
}

nxf_main() {
    trap on_exit EXIT
    trap on_term TERM INT USR2
    trap '' USR1

    [[ "${NXF_CHDIR:-}" ]] && cd "$NXF_CHDIR"
    export NXF_BOXID="nxf-$(dd bs=18 count=1 if=/dev/urandom 2>/dev/null | base64 | tr +/ 0A)"
    NXF_SCRATCH="$(set +u; nxf_mktemp /scratch)"
    [[ $NXF_DEBUG > 0 ]] && nxf_env
    touch /some-path/results/BatchXX/work/e4/3dfbede5151f34f8e16c805ca407ae/.command.begin
    set +u
    set -u
    [[ $NXF_SCRATCH ]] && cd $NXF_SCRATCH
    nxf_stage

    set +e
    ctmp=$(set +u; nxf_mktemp /dev/shm 2>/dev/null || nxf_mktemp $TMPDIR)
    local cout=$ctmp/.command.out; mkfifo $cout
    local cerr=$ctmp/.command.err; mkfifo $cerr
    tee .command.out < $cout &
    tee1=$!
    tee .command.err < $cerr >&2 &
    tee2=$!
    ( nxf_launch ) >$cout 2>$cerr &
    pid=$!
    wait $pid || nxf_main_ret=$?
    wait $tee1 $tee2
    nxf_unstage
}

$NXF_ENTRY