#! /bin/bash
# Copyright (C) 2020-2024 NVIDIA Corporation. ALL RIGHTS RESERVED.
# Copyright 2017,2018,2019,2020 Cumulus Networks, Inc.  All rights reserved.

# This command can change from release to release without
# any compatibility promises, and is intended for Cumulus Networks
# support use, and is deliberately not documented.

typeset -i suptimeout=600 timeout_multiply timeout_divide
declare -i debug_en=0 onlycoremods=0 timeout_mod=1 SUP_SECURE=1 SUP_VERBOSE=0
declare -i serial_mod=0 mem_limit
typeset -i modlist=0 timeout_time=0
declare -r modules_dir=/usr/share/cumulus/support/
declare -a enables disables defmods optmods modulepgrp modulecmds moduletimeout moduletime
declare -A core_execs defsubs optsubs submods_run=() runmodules
declare -r toptmp=/tmp
declare -r lockf=${toptmp}/cl-support.lock
declare -r perf_cache_dir=/var/cache/cumulus/cl_support
declare -r module_cache=${perf_cache_dir}/modules

json_en=
archive_name=
sprefix=
controlfile=
prog=
do_unlock=0
# We want to be able to have certain warnings give a non-zero exit value
# but not prevent the rest of the script from running, so we have them
# set this variable to something, which is then used as the exit value
# in the cleanup() function if the exit value would otherwise be 0.
delayed_exit_value=0

debug()
{
    [ $debug_en -ne 0 ] && echo DEBUG: "$@" 1>&2
}

warn()
{
    echo -e "${prog}: $*" 1>&2
    [ -n "$controlfile" -a -e "$controlfile" ] && echo -e "$@" >> $controlfile
}

warn_ev()
{
    delayed_exit_value=$1
    shift
    warn "$@"
}

error_ev()
{
    local exit_value=$1
    shift
    warn "$@"
    logger -t "${prog}[$$]" -p err -- "$@"
    cleanup $exit_value
}

error()
{
    error_ev 1 "$@"
}

notify()
{
    echo "$@"
    logger -t "${prog}[$$]" -p notice -- "$@"
}

cp_all_attr_parents() {
    # workaround for bugs.gnu.org/55029
    local dest="${@: -1}"  # Get the last argument as the destination
    local sources=("${@:1:$#-1}")  # Get all but the last argument as sources
    local error_pattern="cp: ‘(boot|proc|var|sys|run).*No such file or directory"

    for src in "${sources[@]}"; do
        cp -a --parents "$src" "$dest" 2> >(grep -Ev "$error_pattern" >&2)
    done
}

export -f cp_all_attr_parents

usagemsg='Usage: [-h (help)] [-cDjlMsv] [-d m1,m2,...] [-e m1,m2,...]\n  [-m mem_limit] [-p prefix] [-r reason] [-S dir] [-T Timeout_seconds] [-t tag]'

usage()
{
    warn "$usagemsg"
    exit 1
}

help()
{
    echo -e "$usagemsg"
    cat << EOF
    -h: Display this help message
    -c: Run only modules matching any core files, if no -e modules
    -D: Display debugging information
    -d: Disable (do not run) modules in this comma separated list
    -e: Enable (only run) modules in this comma separated list; "-e all" runs
        all modules and sub-modules, including all optional modules
    -j: Modules will produce json output files, where supported.
    -l: List available modules, then exit
    -M: Do not timeout modules, use with -T
    -m: Run modules serially and set the module memory limit in MB; "-m 0" runs
        serially without limits
    -p: Use prefix as leading part of archive name
    -r: Reason for running $prog (quoted string)
    -S: Use an alternate output directory (default /var/support)
    -s: Include security sensitive information such as sudoers file
    -T: Timeout in seconds for creating support, 0 disables
    -t: tag string as part of archive name
    -v: Verbose; display status messages
EOF
exit 0
}

cleanup()
{
    killchildren
    [ -n "$SUP_TOPDIR" ] && { { [ $1 -ne 2 ] && rm -rf $SUP_TOPDIR ; } ||
        warn Not removing "$SUP_TOPDIR" ; }
    [ $do_unlock -eq 1 ] && rm -f ${lockf}
    rm -f ${toptmp}/cl-support-${$}*
    [ -n "$SUP_TOPDIR" ] && rm -rf ${SUP_TOPDIR}-etc
    if [ "$1" = "0" ]; then
        exit $delayed_exit_value
    else
        exit $1
    fi
}

support_timeout()
{
    warn Timed out waiting for support modules after $suptimeout seconds:
    showchildren Support modules still running will be terminated:
    cleanup 2
}

killpgrp()
{
    local -i pgrp=-$1

    kill -TERM $pgrp 2>/dev/null
    sleep 0.5
    kill -KILL $pgrp 2>/dev/null
}

killchildren()
{
    local -i pgrp i

    for i in ${!modulepgrp[*]}; do
        pgrp=${modulepgrp[$i]}
        killpgrp $pgrp
    done
}


waitchildren()
{
    local -i i now

    [ $debug_en -ne 0 ] && showchildren Waiting for modules to complete

    while [ ${#modulepgrp[@]} -ne 0 ] ; do
        i=0
        now=$(date +%s)
        [ $timeout_time -ne 0 -a $now -gt $timeout_time ] && support_timeout
        for i in ${!modulepgrp[*]}; do
            local -i pgrp=${modulepgrp[$i]} tmout=${moduletimeout[$i]}
            [ -z "$tmout" ] && tmout=$now
            if [ $timeout_mod -eq 1 -a $now -gt $tmout ]; then
                warn_ev 3 Terminating module after timeout: "${modulecmds[$i]}"
                showpgrp $pgrp
                killpgrp $pgrp
            fi
            kill -0 -$pgrp 2>/dev/null ||  {
                debug Module ${modulecmds[$i]} completed
                debug Copying time data to cache
                cp ${moduletime[$i]} $module_cache
                unset modulepgrp[$i] modulecmds[$i] moduletimeout[$i] moduletime[$i]
                continue 2
            }
        done
        sleep 1.01
    done
}

showpgrp()
{
    ps -Ao pgrp,ppid,pid,tid,stat,pcpu,time,wchan,comm,args | \
        sed -n -e 1p -e "/^ *$1/p" | tee -a $controlfile
}

showchildren()
{
    local -i i

    warn "$@"
    for i in ${!modulepgrp[*]}; do
        warn Module "${modulecmds[$i]}: PGRP: ${modulepgrp[$i]}"
        showpgrp ${modulepgrp[$i]}
    done
}

run_timeout()
{
    local stdout dt modnm mkdir pgrpf
    local -i cnt slp cpgrp  n

    case "$1" in
    [1-9]|[1-9][0-9]|[1-9][0-9][0-9]|[1-9][0-9][0-9][0-9]) slp="$1" ;;
    *) error Invalid timeout value=\""$1"\".
    esac
    shift

    modnm=${1##*/}
    stdout=$SUP_LOGDIR/$modnm

    dt="%F_%T.%N: $*"
    [ $SUP_VERBOSE -eq 1 ] && date +"$dt" 1>&2
    date +"## $dt">> $stdout

    export SUP_MODNAME
    ((tmout = start_time + slp))

    pgrpf=${toptmp}/pgrp-$modnm
    rm -f $pgrpf
    debug 'echo $$ >'" $pgrpf; SUP_MODNAME=$modnm $modules_dir$*" "> $stdout"
    setsid bash -c 'echo $$ >'" $pgrpf; SUP_MODNAME=$modnm /usr/bin/time -v -o $stdout.time -a $modules_dir$*" \
      > $stdout </dev/null &
    cpgrp=$!
    cnt=0
    while ! [ -e $pgrpf -a $cnt -lt 8 ]; do sleep 1.02; ((cnt++)); done
    [ $cnt -gt 1 ] && debug took $cnt seconds for $modnm to start
    [ -e $pgrpf ] || { ((cpgrp++));
        warn No child PGRP for $modnm, assuming $cpgrp ; }
    read cpgrp < $pgrpf
    rm -f $pgrpf
    { [ "$cpgrp" -lt 2 ] &&
        warn Invalid child PGRP $cpgrp for $modnm. Unable to track ; } || {
        modulepgrp=( "${modulepgrp[@]}" $cpgrp )
        n=${#modulepgrp[@]}
        (( n-- ))
        moduletimeout[$n]=$tmout
        modulecmds[$n]="$*"
        moduletime[$n]=$stdout.time
    }
}

run_module_serial()
{
    local stdout dt modnm mkdir pgrpf
    local -i cnt slp cpgrp  n

    case "$1" in
    [1-9]|[1-9][0-9]|[1-9][0-9][0-9]|[1-9][0-9][0-9][0-9]) slp="$1" ;;
    *) error Invalid timeout value=\""$1"\".
    esac
    shift

    modnm=${1##*/}
    stdout=$SUP_LOGDIR/$modnm

    dt="%F_%T.%N: $*"
    [ $SUP_VERBOSE -eq 1 ] && date +"$dt" 1>&2
    date +"## $dt">> $stdout

    last_run=$(get_cached_mem modules/${modnm}.time)
    debug Last execution of module ${modnm} used ${last_run}MB

    if [[ ${mem_limit} -gt 0 && ${last_run} -gt ${mem_limit} ]]; then
        warn_ev 7 Not running module $modnm due to memory limit of ${mem_limit}MB
        return 1
    fi

    export SUP_MODNAME
    ((tmout = start_time + slp))

    pgrpf=${toptmp}/pgrp-$modnm
    rm -f $pgrpf
    debug 'echo $$ >'" $pgrpf; SUP_MODNAME=$modnm $modules_dir$*" "> $stdout"
    setsid bash -c 'echo $$ >'" $pgrpf; SUP_MODNAME=$modnm /usr/bin/time -v -o $stdout.time -a $modules_dir$*" \
      > $stdout </dev/null &
    cpgrp=$!
    cnt=0
    while ! [ -e $pgrpf -a $cnt -lt 8 ]; do sleep 1.02; ((cnt++)); done
    [ $cnt -gt 1 ] && debug took $cnt seconds for $modnm to start
    [ -e $pgrpf ] || { ((cpgrp++));
        warn No child PGRP for $modnm, assuming $cpgrp ; }
    read cpgrp < $pgrpf
    rm -f $pgrpf

    [ $debug_en -ne 0 ] && debug Waiting for module "$*" to complete

    local -i mod_done=1
    while [ ${mod_done} -eq 1 ] ; do
        i=0
        now=$(date +%s)
        [ $timeout_time -ne 0 -a $now -gt $timeout_time ] && support_timeout
        [ -z "$tmout" ] && tmout=$now
        if [ $timeout_mod -eq 1 -a $now -gt $tmout ]; then
            warn_ev 3 Terminating module after timeout: "$*"
            showpgrp $cpgrp
            killpgrp $cpgrp
            mod_done=0
        fi
        kill -0 -$cpgrp 2>/dev/null ||  {
            debug Module $* completed
            debug Copying time data to cache
            cp $stdout.time $module_cache
            mod_done=0
            continue 2
        }
        sleep 1.01
    done
}

get_modlist()
{
    local -a modules dsubs osubs
    local -i i allopt=0
    local sub

    [ ${#defmods[@]} -gt 0 -o ${#optmods[@]} -gt 0 ] && return

    modules=( $(cd $modules_dir &&
        find * -prune -type f -executable -print) )

    for mod in ${modules[*]}; do
        if grep -aq '^#DEFAULT$' $modules_dir$mod
        then defmods+=($mod)
        else optmods+=($mod)
        fi
    done
    for mod in ${defmods[*]} ${optmods[*]}; do
        [ $mod = "${optmods[0]}" ] && allopt=1
        osubs=()
        read -a dsubs <<< $($modules_dir$mod -l)
        for i in "${!dsubs[@]}"; do
            [ "${dsubs[$i]}" = ";" ] && {
                { [ $allopt -eq 1 ] && {
                    optsubs[$mod]="${dsubs[*]:0:$i} ${dsubs[*]:((i+1))}"
                    continue 2; } ; } ||
                    { osubs=(${dsubs[*]:((i+1))}) dsubs=(${dsubs[*]:0:$i}) ; }
                break; }
        done
        for sub in ${dsubs[*]}; do
            [ $allopt -eq 0 ] &&
                defsubs[$mod]="${defsubs[$mod]} $sub" ||
                optsubs[$mod]="${optsubs[$mod]} $sub"
        done
        for sub in ${osubs[*]}; do
            optsubs[$mod]="${optsubs[$mod]} $sub"
        done
    done
}

list_modules()
{
    local mod sub

    get_modlist

    ( echo -n Default  modules: ''
        for mod in ${!defsubs[*]}; do
            for sub in ${defsubs[$mod]}; do
                echo -n $mod.$sub ''
            done
        done
        ) | fmt -t
    ( echo -n Optional  modules: ''
        for mod in ${!optsubs[*]}; do
            for sub in ${optsubs[$mod]}; do
                echo -n $mod.$sub ''
            done
        done
        ) | fmt -t
}

core_modules()
{
    local mod
    for mod in ${defmods[*]} ${optmods[*]}; do
        egrep -aq "^#ONCORE=.*(\<$1\>|\<all\>)" $modules_dir$mod && {
            runmodules[$mod]=$mod
            [ ${#core_execs[$mod]} -eq 0 ] && { core_execs[$mod]="$1"
                continue ; }
            local -a cmds=(${core_execs[$mod]//,/ })
            local cmd
            for cmd in ${cmds[*]}; do
                [ "$1" = "$cmd" ] && continue 2
            done
            core_execs[$mod]="${core_execs[$mod]},$1"
            }
    done
}
# Get LTTNG data and save the current buffers to a dated folder.
check_lttng()
{
    # Stop automated cleanups of saved folders.
    /usr/bin/systemctl stop cumulus-cleanup-lttng_traces.path
    /usr/bin/systemctl stop cumulus-cleanup-lttng_traces.service

    # flush ring buffer to the file
    lttng set-session frr-local-onboot >> /dev/null
    lttng set-session frr-local-zebra-onboot >> /dev/null
    lttng set-session switchd-local-onboot >> /dev/null
    lttng stop >> /dev/null
    lttng start >> /dev/null

    cd /var/log/lttng-traces/ >> /dev/null

    status=$?
    if [ $status -eq 0 ]; then
        if [ -d frr-local-onboot ]; then
           # copy frr babeltrace script to the directory for user convenience
           cp /usr/lib/frr/frr_babeltrace.py  frr-local-onboot/
           cp -p -r frr-local-onboot frr-local-onboot-$(date +"%Y%m%d_%H%M%S")
        fi
        if [ -d frr-local-zebra-onboot ]; then
           # copy frr babeltrace script to the directory for user convenience
           cp /usr/lib/frr/frr_babeltrace.py  frr-local-zebra-onboot/
           cp -p -r frr-local-zebra-onboot frr-local-zebra-onboot-$(date +"%Y%m%d_%H%M%S")
        fi
        if [ -d switchd-local-onboot ]; then
           # copy switchd-babektrace.py script to the directory
           cp /usr/lib/cumulus/switchd-babeltrace.py switchd-local-onboot/
           cp -p -r switchd-local-onboot switchd-local-onboot-$(date +"%Y%m%d_%H%M%S")
        fi
        cd - >> /dev/null
    fi

    # Re-enable automated cleanups
    /usr/bin/systemctl start cumulus-cleanup-lttng_traces.path
    /usr/bin/systemctl start cumulus-cleanup-lttng_traces.service
}

wait_core_finish()
{
    local cores core pid ptmp cmd cprog
    local -i delay=60
    cores=(*)
    for core in ${cores[@]}; do
       [ "$core" = '*' ] && break 2
       cprog=${core%%.*}
       ptmp=${core#*.}
       pid=${ptmp%%.*}
       [ -d /proc/$pid ] || continue
       cmd=$(cat /proc/$pid/comm 2>/dev/null)
       [ "$cmd" != "$cprog" ] && continue
       while [ -e /proc/$pid/comm -a $delay -gt 0 ] ; do
           warn Waiting for pid $pid $cprog coredump to complete
           sleep 1.03
           (( delay-- ))
       done
    done
    [ $delay -eq 0 ] && warn Timed out waiting for coredump to complete
}

check_cores()
{
    local cdir corepat d f cmd

    corepat="$(sysctl kernel.core_pattern)"
    [ -z "$corepat" ] &&
        { warn No core file check, missing sysctl ; return ; }
    pat=${corepat##* }
    cdir=${pat%/*}
    file=${pat##*/}
    [ -z "$cdir" -o -z "$file" ] &&
        { warn No core file check, unexpected core_pattern; return ; }
    case "$file" in
    %e*) ;;
    *) warn kernel.core_pattern \"$file\" does not contain %e, can not \
        determine executable
       return ;;
    esac
    delpat=${file/\%e/EXEC}
    fpat=$(echo "$delpat" | sed -r -e 's/\./\\./' -e 's/%[a-zA-Z%]/[^.]*/g' \
        -e 's/$/$/' -e 's/.core_helper$//' -e 's/EXEC//' )

    d=$PWD
    { [ -d $cdir ] && cd $cdir ; } || { cd $d ; echo "Could not find or cd to $cdir" >> $controlfile ; return ; }

    wait_core_finish
    echo "Core files in $PWD:" * >> $controlfile
    for f in *; do
       [ "$f" = '*' ] &&  continue
       mv -f "$f" $coredir
       [ $? -ne 0 ] && echo "Failed to mv -f \"$f\" $coredir" >> $controlfile
       cmd=$(echo "$f" | sed "s/$fpat//")
       core_modules "$cmd"
    done
    cd $d || exit 1
}

include_in_kdump()
{
    local file=$1
    local kdumpdir=$2
    local witness=$3

    if [ -e "$file" ]; then
        mv "$file" "$kdumpdir"
        echo "Included $file in $(basename "$archive_name")" >> "$witness"
    else
        echo "Skipped, $file not found" >> "$witness"
    fi
}

check_kdump()
{
    local crashdir="/var/crash"

    if ! [ -d "$crashdir" ]; then
	    return
    fi

    tree -hF "$crashdir" > "$kdumpdir"/tree

    while IFS= read -r -d $'\0' dir; do
        local witness="$crashdir/$dir/cl-support.$dir"
        if [ -e "$witness" ]; then
            continue
        fi

        include_in_kdump "$crashdir/$dir/dmesg.$dir" "$kdumpdir" "$witness"
        include_in_kdump "$crashdir/$dir/dump.$dir" "$kdumpdir" "$witness"
    done < <(find "$crashdir" -mindepth 1 -maxdepth 1 -type d -printf "%P\0")

    # include all cl-support.* files
    find "$crashdir" -mindepth 2 -maxdepth 2 -name "cl-support.*" -print0 | \
        xargs -r0 cp -t "$kdumpdir"
}

do_opts()
{
    local -i maxpath maxtag
    local _hn supportdir basedir _base tmp essential archive_dir
    local tag=

    supportdir=/var/support
    cmdline=( "$@" )

    while getopts ":hcDd:e:jlMm:p:r:S:sT:t:v" Option; do
       case $Option in
       D) debug_en=1 ;;
       c) onlycoremods=1 ;;
       d) disables=(${OPTARG//,/ }) ;;
       e) enables=(${OPTARG//,/ }) ;;
       h) help ;;
       j) json_en=-j ;;
       l) modlist=1 ;;
       M) timeout_mod=0 ;;
       m) [[ $OPTARG == ?(-)+([[:digit:]]) ]] || error_ev 1 Memory limit is not an integer ; serial_mod=1 mem_limit=$OPTARG suptimeout=0;;
       p) sprefix="$OPTARG" ;;
       r) reason="$OPTARG" ;;
       S) supportdir=$OPTARG ;;
       s) SUP_SECURE=0 ;;
       T) case $OPTARG in
          [0-9]*) suptimeout=$OPTARG ;;
          *) warn "Timeout value for -T is not a number, ignored: $OPTARG" ;;
          esac ;;
       t) tag=${OPTARG//[[:space:]]/-} ;;
       v) SUP_VERBOSE=1 ;;
       *|?) usage ;;
       esac
    done
    shift $((OPTIND - 1))

    [ $# -ne 0 ] && warn Extra arguments \""$*"\" ignored, did you mean to use '-r ?'

    if [ -z "$sprefix" ]; then
        case "$prog" in
        [^-]*-*) prfx=${prog%%-*} ;;
        *) prfx=cl ;;
        esac
        sprefix="$prfx"
    fi
    # allow only @+_. from [:punct:]
    tmp="${sprefix//[][!\"#$%\&\'()*,:;<=>?^\`{|\}~[:space:][:cntrl:]]/}"
    if [ "$sprefix" != "$tmp" ]; then
      tmp="${sprefix//[[:alnum:]@+_.]}"
      tmp=$(printf "%q" "$tmp"); tmp=${tmp/$/}; tmp=${tmp//\\/\\\\}
      sprefix=$(printf "%q" "$sprefix"); sprefix=${sprefix/$/};
      sprefix=${sprefix//\\/\\\\}
      error "Prefix $sprefix contains illegal characters: $tmp"
    fi

    _hn=$(hostname)
    [ -z "$_hn" ] && _hn=unknown
    [ -n "$tag" ] && _hn="${tag}_$_hn"
    tag=${_hn//[^a-zA-Z0-9\-\.\_]/}
    date=$(date +%Y%m%d_%H%M%S)
    basedir=${sprefix}_support_${tag}_${date}
    _base=$supportdir/$basedir
    tmp=$_base.txz
    [ -d $supportdir ] || error Directory $supportdir does not exist
    maxpath=$(getconf NAME_MAX $supportdir)
    [ ${#tmp} -gt $maxpath ] && {
       essential=${basedir/${tag}/}.txz
        maxtag=$(( maxpath - ${#essential} ))
       _base=${supportdir}/cl_support_${tag:0:$maxtag}_$date
    }

    IFS=' \t%' read used <<< $(df --output=pcent ${supportdir} |
        tail -1)
    [ "$used" ] && [ "$used" -gt 80 ] &&
      error_ev 5 "Aborting because filesystem for ${supportdir} is ${used}% full"

    archive_dir=$_base
    archive_name=$archive_dir.txz
    coredir=$archive_dir/core
    kdumpdir=$archive_dir/kdump

    SUP_TOPDIR=$archive_dir
    SUP_LOGDIR=$SUP_TOPDIR/Support
    export SUP_TOPDIR SUP_LOGDIR SUP_SECURE SUP_VERBOSE
    controlfile=$SUP_TOPDIR/Control
}

set_all_modules()
{
    local mod sub
    for mod in ${defmods[*]} ${optmods[*]}; do
        runmodules[$mod]=$mod
        for sub in ${defsubs[$mod]} ${optsubs[$mod]}; do
            add_submod $mod $sub
        done
    done
}

add_submod()
{
    local s mod=$1 sub=$2
    for s in ${submods_run[$mod]}; do
       [ $s = $sub ] && return
    done
    [ ${#submods_run[$mod]} -eq 0 ] && submods_run[$mod]=$sub ||
        submods_run[$mod]="${submods_run[$mod]} $sub"
}

del_submod()
{
    local s mod=$1 sub=$2 newsubs
    for s in ${submods_run[$mod]}; do
       [ $s = $sub ] && continue
       [ ${#newsubs} -eq 0 ] && newsubs=$s || newsubs="$newsubs $s"
    done
    submods_run[$mod]="$newsubs"
    [ ${#submods_run[$mod]} -eq 0 ] && unset submods_run[$mod] runmodules[$mod]
}

parse_verify_mods()
{
    local m s mod submod mtmp
    local -i i=0 drop drop_s

    [ ${#enables[@]} -eq 0 -a $onlycoremods -eq 0 ] && enables=(${defmods[*]})
    drop=${#enables[@]}
    [ $drop -eq 0 ] && {
        { [ $onlycoremods -eq 1 ] && return ; } ||
        error No default modules and none from cmdline ; }
    (( drop_s=drop + 1 ))
    for m in ${enables[*]} ${disables[*]}; do
        ((i++)); mod=${m%.*} submod=${m#*.}
        [ $m = "$mod" -a "$mod" = "$submod" ] && submod=
        [ -z "$mod" ] && mod=_NOMOD_
        [ $i -le $drop -a "$m" = "all" ] && { set_all_modules; continue; }
        [ $i -gt $drop -a "$m" = "all" ] &&
            error Module $m is not valid with '-d'
        [ ${#defsubs[$mod]} -eq 0 -a ${#optsubs[$mod]} -eq 0 ] &&
            error Module $m is not valid, use -l for list
        [ -n "$submod" ] && { ok=0
            for s in ${defsubs[$mod]} ${optsubs[$mod]} ; do
                [ $s = "$submod" ] && { ok=1 ; break ; }
            done
            [ $ok -eq 0 ] && error Submodule $m is not valid, use -l for list
        }
        [ ${#disables[@]} -gt 0 -a $i -eq $drop_s ] && {
            for mtmp in ${runmodules[*]}; do
                [ ${#submods_run[$mtmp]} -eq 0 ] &&
                    submods_run[$mtmp]=${defsubs[$mtmp]}
            done
        }
        [ $i -le $drop ] && { runmodules[$mod]=$mod ; \
            [ -n "$submod" ] && add_submod $mod $submod
            continue ; }
        { [ $m = $mod ] && unset runmodules[$mod] submods_run[$mod] ; } ||
            del_submod $mod $submod
    done
    [ ${#runmodules[@]} -eq 0 ] && error Empty modules list after -e/-d
    for mod in ${runmodules[*]}; do
        [ "${submods_run[$mod]}" = "${defsubs[$mod]}" ] &&
            unset submods_run[$mod]
    done
}

run_modules()
{
    local mod mod2 corearg

    start_time=$(date +%s)

    get_modlist

    mkdir -p $SUP_TOPDIR $SUP_LOGDIR $coredir $kdumpdir ||
        error_ev 6 Could not create support directories: \
            $coredir $kdumpdir $SUP_LOGDIR

    check_cores
    check_kdump
    check_lttng
    parse_verify_mods

    [ ${#runmodules[@]} -eq 0 ] && error Nothing to do, no modules or cores

    cd $SUP_LOGDIR || cleanup 1

    [ -n "$reason" ] && echo "Reason: $reason" >  $controlfile
    (printf "Command line: %s " "$prog" ;
    printf "%q " "${cmdline[@]}" ;
    printf '\n' ;
    date +"$prog Started collecting data at %FT%T.%N %Z" ;
    echo Support timeout=$suptimeout ) >> $controlfile
    [ $SUP_SECURE -eq 0 ] && echo Include Security files >> $controlfile
    for m in ${!core_execs[*]}; do
        echo Module $m will handle cores: ${core_execs[$m]} >> $controlfile
    done
    echo Modules to be run: >> $controlfile
   [ $SUP_VERBOSE -eq 1 ] && echo Modules to be run:
    for m in ${runmodules[*]}; do
       echo '  '$m: ${submods_run[$m]} >> $controlfile
       [ $SUP_VERBOSE -eq 1 ] && echo "  $m: ${submods_run[$m]}" 1>&2
    done

    for mod in ${runmodules[*]}; do
        local tmout
        tmout=$(sed -n 's/^#TIMEOUT=//p' $modules_dir$mod)
        [ -n "$tmout" -a -n "$timeout_multiply" ] &&
            (( tmout=(tmout*timeout_multiply)/timeout_divide ))
        [ -n "$tmout" -a $suptimeout -ne 0 ] && [ $tmout -gt $suptimeout ] &&
            tmout=$suptimeout
        [ -z "$tmout" -a $suptimeout -ne 0 ] && { [ $suptimeout -gt 60 ] &&
            tmout=$(( suptimeout - 30 )) || tmout=$((suptimeout - 1)) ; }
        [ -z "$tmout" ] && tmout=580

        { [ ${#core_execs[$mod]} -gt 0 ] && corearg="-c \"${core_execs[$mod]}\"" ; } ||
            corearg=

        if [ ${serial_mod} -eq 1 ]; then
            # BUG: #3750538
            # Reset start_time for each module when running in serial mode.
            start_time=$(date +%s)
            debug run_module_serial $tmout $mod $corearg $json_en \
                ${submods_run[$mod]}
            run_module_serial $tmout $mod $corearg $json_en ${submods_run[$mod]}
        else
            debug run_timeout $tmout $mod $corearg $json_en \
                ${submods_run[$mod]}
            run_timeout $tmout $mod $corearg $json_en ${submods_run[$mod]}
            disown -a
        fi
    done
}

create_archive()
{
    local -a excludes dirs
    local transdir extra_dirs d dtmp status pipe

    [ $SUP_VERBOSE -eq 1 ] && notify "Creating $archive_name ..." 1>&2

    cd $SUP_TOPDIR || exit 1
    chmod -R a+r .
    find . -type d -print0 | xargs -0 chmod a+x,u+w

    dirs=($(find * -type l -print | egrep -v '^etc/|^'${SUP_LOGDIR##*/}))
    [ "${#dirs[@]}" -ne 0 ] && {
        for d in ${dirs[*]}; do
            dtmp=$(readlink -s -e $d)
            [ -n "$dtmp" ] && {
                transdir="$transdir --transform s,^$d,${SUP_TOPDIR##*/}/$d,"
                extra_dirs="$extra_dirs $d"
                rm $d
            }
        done

        extra_dirs=" -C / "$extra_dirs
    }

    excludes=(--exclude='etc/nologin'
        --exclude='var/lib/cumulus/installer/*'
        --exclude='var/lib/cumulus/cl-image-upgrade/*'
        --exclude='var/installer/*'
        --exclude='var/images/*'
        --exclude='etc/passwd*'
        --exclude='etc/shadow*'
        --exclude='etc/group*'
        --exclude='etc/gshadow*'
        --exclude='var/log/lastlog'
        --exclude='var/log/journal'
        --exclude='var/log/lttng-traces/switchd-local-onboot'
        --exclude='var/log/lttng-traces/frr-local-onboot'
        --exclude='var/log/lttng-traces/frr-local-zebra-onboot'
        --exclude='etc/ssh/*key*'
        --exclude='etc/nvue/certificates/private')
    [ $SUP_SECURE -eq 1 ] &&
        excludes+=(--exclude='etc/pam*' --exclude='etc/security*'
            --exclude='etc/sudoers*' --exclude='etc/ssl*'
            --exclude='etc/chef' --exclude='etc/puppet/puppet.conf'
            --exclude='etc/hosts*'
            --exclude='etc/nvue.d/*.yaml'
            --exclude 'var/lib/nvue/config/nvue.json'
            --exclude 'var/lib/nvue/config/backup.tar'
            --exclude 'var/lib/nvue/config/.git' )

    [ $SUP_SECURE -eq 1 ] && {
	for SNMPD_CONF_FILE in $SUP_TOPDIR/{etc/snmp,var/lib/nvue/config}/snmpd.conf* $SUP_TOPDIR/{etc/snmp,var/lib/nvue/config}/snmpd.conf.d/* ; do
	    [ -f $SNMPD_CONF_FILE ] && {
		sed -i -e \
		  '/^createUser/s/[^ \t]*[ \t][ \t]*\([AD]ES\).*/AAAAA \1 EEEEE/' \
		  -e '/^rocommunity/s/[ \t][ \t]*[^ \t]*/ CCCC/' \
		  -e '/\(trapsink[ \t]\)\([^ \t]*[ \t]\)*.*/s//\1\2CCCC/' \
		  -e '/\(trap2sink[ \t]\)\([^ \t]*[ \t]\)*.*/s//\1\2CCCC/' \
		  -e '/trapsess.*-A/s/-A[ \t][ \t]*[^ \t]*/-A AAAAA/' \
		  -e '/trapsess.*-X/s/-X[ \t][ \t]*[^ \t]*/-X EEEEE/' \
		  $SNMPD_CONF_FILE
	    }
	done
        for frrconf in $SUP_TOPDIR/{etc/frr,var/lib/nvue/config}/frr.conf* ; do
            [ -f $frrconf ] && {
                sed -i \
                  -e 's/ authentication-key [^ ]*/ authentication-key SSSS/' \
                  -e '/password /s/ [^ ]*$/ SSSS/' \
                  -e '/message-digest/s/md5 [^ ]*/md5 SSSS/' \
                  -e '/authentication string/s/ [^ ]*$/ SSSS/' \
                  -e '/key-string/s/ [^ ]*$/ SSSS/' $frrconf
            }
        done
        for hostapdconf in $SUP_TOPDIR/{etc,var/lib/nvue/config}/hostapd.conf* ; do
            [ -f $hostapdconf ] && {
                sed -i -e '/shared_secret=./s/=.*/=KKKK/' \
                    -e '/radius_das_client=./s/=[^ \t]*/=KKKK/' \
                    $hostapdconf
            }
        done
        for tacplusfile in $SUP_TOPDIR/{etc,var/lib/nvue/config}/etc/tacplus_{servers,nss.conf}* ; do
            [ -f $tacplusfile ] && {
                sed -i -e '/secret=./s/=.*/=SSSS/' $tacplusfile
            }
        done
        for radiusfile in $SUP_TOPDIR/{etc,var/lib/nvue/config}/pam_radius_auth.conf* ; do
            [ -f $radiusfile ] && {
                sed -i -e '/^mapped_priv_user/b' -e '/^#/b' \
                   -e 's/[ \t][^ \t]*/ SSSS/' $radiusfile
            }
        done
    }

    rmdir -p $SUP_TOPDIR/* 2>/dev/null

    XZ_OPT=-2q tar -J -c -b 128 -f $archive_name --warning=no-file-changed \
      --mode=+r -C ${SUP_TOPDIR%/*} ${excludes[*]} $transdir \
      ${SUP_TOPDIR##*/} $extra_dirs
    status=$?
    if [ $status -eq 1 ]; then
        warn "tar returned 1: some files may have changed while being archived"
    elif [ $status -ne 0 ]; then
        error_ev 4 "tar returned fatal error $status while generating $archive_name"
    fi
}

set_exclusive()
{
    exec 17>> $lockf
    flock -e -n 17 || { notify Another instance is already running ; exit 1 ; }
    do_unlock=1
}

main()
{
    local arch
    prog=${0##*/}
    export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/cumulus/bin

    cd /

    arch=$(uname -m 2>/dev/null)
    case "$arch" in
    [aA][rR][mM]*)
        timeout_multiply=20 timeout_divide=10
        (( suptimeout=(suptimeout*timeout_multiply)/timeout_divide ))
        ;;
    esac

    do_opts "$@"

    [ -n "$timeout_multiply" ] &&
        debug Timeout multiplier ${timeout_multiply}/${timeout_divide}

    [ "$(whoami)" != root ] &&
        error_ev 6 "must be run as root (or with sudo)"
    set_exclusive

    renice +5 -p $$ >> /dev/null
    ionice -c 2 -n 5 -p $$ >> /dev/null

    [ $modlist -eq 1 ] && { list_modules && exit 0 ; }

    [ $suptimeout -ne 0 ] && (( timeout_time = $(date +%s) + suptimeout ))

    trap 'warn Terminating on signal; cleanup 254' INT TERM

    run_modules

    debug Waiting for support modules to complete
    waitchildren

    date +"$prog Finished collecting data at %FT%T.%N %Z" >>  $controlfile

    create_archive
    notify Please send $archive_name to Cumulus support.

    cleanup 0
}

exec_cmd()
{
    local json msg tfile nsec
    [ $1 = "-j" ] && { json=.json; shift ; }
    nsec="$(date +%N)"
    ofile=${1}$json
    file=${1}$json${RANDOM}_${nsec}
    tfile=${toptmp}/$$execcmd_time_${file##*/}
    shift
    local TIMEFORMAT='%2R seconds'
    { [ -n "$json" ] && msg='[ { "start" : "'"$*"'"},' ; } ||
        msg="# Start: $*"
    echo "$msg" >> $file
    { time $1 "${@:2}" &>> $file ; } 2>$tfile
    ret=$?
    read secs < $tfile
    { [ -n "$json" ] && msg=', { "complete" : "'" $secs $*"'"}]' ; } ||
        msg="# Completed in $secs: $*"
    echo "$msg" >> $file
    cat $file >> $ofile
    rm $tfile $file
    return $ret
}

# Read a time -v output file and return the Maximum RSS value as MB
# Returns 0 in the file does not exist.
get_cached_mem ()
{
    local time_file=$perf_cache_dir/$1
    local kb mb
    typeset -i kb mb

    kb=$(cat $time_file 2>/dev/null | grep "Maximum resident set size" | awk '{ print $NF }')

    # Set a default value if the above failed
    kb=${kb:-0}
    mb=$(( kb / 1024 ))
    echo $mb
}

export -f exec_cmd
export toptmp

# If we are being sourced, do nothing, otherwise run the main function
(return 0 2>/dev/null) || main "$@"
