#! /bin/bash
# Copyright (C) 2020-2021 NVIDIA Corporation. ALL RIGHTS RESERVED.
# Copyright 2017,2018,2019,2020 Cumulus Networks,  Inc.  All rights reserved.
#
# Record switchd and related state; used by cl-support
#
# Note that cl-support expects the below three flags to be commented
#TIMEOUT=300
#DEFAULT
#ONCORE=switchd sx_sdk

module=${0##*/}

funcs=(mlx debug stack fuse)
nondef_funcs=(verbose)
json=
jexec=
if [ -n "$TMPDIR" ]; then tmpdir="$TMPDIR"; else tmpdir=/tmp; fi

declare -i _switchd_ok=1
switchd_ok()
{
    local cpid z

    { [ $_switchd_ok -eq 0 ] &&  return 1 ; } ||
        { [ $_switchd_ok -eq 2 ] && return 0 ; }
    local -r flagf=$tmpdir/flagf$$ pidf=$tmpdir/pid$$

    touch $flagf
    trap "export -n flagf pidf; rm -f $flagf $pidf" RETURN
    systemctl -q is-active switchd || { echo $module: switchd is not running;
        return 1 ; }
    export flagf pidf
    ( (echo $BASHPID > $pidf ; TMOUT=5 read x < /cumulus/switchd/config/logging \
        > $flagf ; echo $x > $flagf) 2>/dev/null &)
    sleep 1.5
    read cpid < $pidf
    TMOUT=8 read z < $flagf
    [ $? -ne 0 ] && { echo $module: switchd not responding
        kill -KILL $cpid 2>/dev/null; _switchd_ok=0 ; return 1; }
    _switchd_ok=2
    return 0
}

declare -i _debug_state_orig=2
declare -r switchd_dbg_ctrl=/cumulus/switchd/ctrl/debug \
    switchd_dbg_dir=/cumulus/switchd/debug

# enable fuse debug nodes if not already enabled.
enable_debug()
{
	switchd_ok || return
	if [ -e $switchd_dbg_dir/bond ]; then
		_debug_state_orig=1
	elif [ -e $switchd_dbg_ctrl ]; then
		echo 1 > $switchd_dbg_ctrl || {
        echo Unable to enable switchd fuse debug ; return 1; }
		_debug_state_orig=0
	fi
    return 0
}

restore_debug()
{
	[ $_debug_state_orig -eq 0 ] && echo 1 > $switchd_dbg_ctrl
}

func_debug()
{
	switchd_ok || return

	# ACL Dump
	exec_cmd switchd.debug.acl cat ${switchd_dbg_dir}/acl

	# Bond Dump
	switchd_dbg_bond_debug="${switchd_dbg_dir}/bond"
	exec_cmd switchd.debug.bond.info cat ${switchd_dbg_bond_debug}/info
	exec_cmd switchd.debug.bond.sw_bonds cat ${switchd_dbg_bond_debug}/sw_bonds

	# Bridge Dump
	switchd_dbg_bridge_debug="${switchd_dbg_dir}/bridge"
	exec_cmd switchd.debug.bridge.info cat ${switchd_dbg_bridge_debug}/info

	# Vlan Dump
	switchd_dbg_vlan_debug="${switchd_dbg_dir}/vlan"
	exec_cmd switchd.debug.vlan.info cat ${switchd_dbg_vlan_debug}/info

	# Port Dump
	switchd_dbg_misc_debug="${switchd_dbg_dir}/misc"
	exec_cmd switchd.debug.misc.ports cat ${switchd_dbg_misc_debug}/ports

        # VXLAN Dump
	switchd_dbg_vxlan_debug="${switchd_dbg_dir}/vxlan"
	exec_cmd switchd.debug.vxlan.info cat ${switchd_dbg_vxlan_debug}/info

	# Knet Dump
	switchd_dbg_knet_debug="${switchd_dbg_dir}/knet"
	exec_cmd switchd.debug.knet.info cat ${switchd_dbg_knet_debug}/info

	# mroute dump
	switchd_dbg_mroute_debug="${switchd_dbg_dir}/mroute"
	exec_cmd switchd.debug.mroute.info cat ${switchd_dbg_mroute_debug}/info
}

func_stack()
{
	/bin/ps -C switchd -L -o lwp --no-headers | while read p; do exec_cmd switchd.stack.1 /bin/cat /proc/$p/stack; done
	sleep 0.1
	/bin/ps -C switchd -L -o lwp --no-headers | while read p; do exec_cmd switchd.stack.2 /bin/cat /proc/$p/stack; done
	sleep 0.1
	/bin/ps -C switchd -L -o lwp --no-headers | while read p; do exec_cmd switchd.stack.3 /bin/cat /proc/$p/stack; done
}

# get the stacks for all TIDS, with the thread name, for a given command.
# needs to be a function because exec_cmd() can't take a pipeline
getstacks()
{
	/bin/ps -C $1 -L -o lwp --no-headers | \
	   xargs -l -I'##' awk 'BEGIN {getline comm < "/proc/##/comm"}
		{ printf "%9d %14s: %s\n", ##, comm, $0}' /proc/##/stack
}

# get the status file for all TIDS, with the thread name, for a given command.
# needs to be a function because exec_cmd() can't take a pipeline
getstatus()
{
	/bin/ps -C $1 -L -o lwp --no-headers | \
	   xargs -l -I'##' awk 'BEGIN {getline comm < "/proc/##/comm"}
		{ printf "%9d %14s: %s\n", ##, comm, $0}' /proc/##/status
}

# get the stat file for all TIDS, with the thread name, for a given command.
# needs to be a function because exec_cmd() can't take a pipeline
# This doesn't need the IDs in the printf because they are in the stat file
# and it's a oneline file
getstat()
{
	/bin/ps -C $1 -L -o lwp --no-headers | \
	   xargs -l -I'##' cat /proc/##/stat
}

func_verbose() {
	exec_cmd switchd.stacks getstacks switchd
	exec_cmd switchd.status getstatus switchd
	exec_cmd switchd.stat getstat switchd
}

_mlxcmd=/usr/lib/cumulus/mlxcmd
declare -i mlxcmd_ok=1
mlxcmd()
{
   [ $mlxcmd_ok -eq 0 ] && { return 1 ; }
   [ $mlxcmd_ok -eq 1 ] && { (switchd_ok && mlxcmd_ok=2) || {
     mlxcmd_ok=0 ; return 1 ; } ; }
   file=$1
   shift
   exec_cmd $jexec $file $_mlxcmd $json $@
   return 0
}

func_mlx()
{
    #
    # if the SDK is not ready, do not do mlxdump commands
    #
    systemctl --quiet is-active sx_sdk.service
    rc=$?
    if [ $rc -ne 0 ]; then
	echo "# skipped sx_api_dbg_generate_dump due to sdk not ready" > /var/log/sdk_debug_dump.log
        return 0
    fi

    #check if sflow is active
    systemctl --quiet is-active hsflowd.service
    hsflowd=$?

    #
    # if switchd is not ready, do not do mlxdump commands
    #
    if [ ! -f "/run/switchd.ready" ]; then
	echo "# skipped sx_api_dbg_generate_dump due to switchd not ready" > /var/log/sdk_debug_dump.log
	return 0
    fi

    #
    # collect sx kernel RDQ and SDQ state information.
    #
    echo dump_rdq > /proc/mlx_sx/sx_core
    echo dump_sdq > /proc/mlx_sx/sx_core

    [ -x /usr/bin/sx_api_dbg_generate_dump.py ] || return # not mlx
    /usr/bin/sx_api_dbg_generate_dump.py /var/log/sdk_debug_dump.log
    [ -x /usr/bin/sx_api_flex_acl_dump.py ] && { exec_cmd sx_api_flex_acl_dump.txt /usr/bin/sx_api_flex_acl_dump.py; }
    [ -x $_mlxcmd ] && {
    if [ $hsflowd -eq 0  ]; then
        for cmd in sflow sflow_statistics; do
            echo "--out_file port.${cmd}.show ports show $cmd swp\*" >> mlx_cmds.txt
        done
    fi

    for cmd in counters pktdist discards; do
        echo "--out_file port.${cmd}.show ports show $cmd swp\*" >> mlx_cmds.txt
    done
    cat /usr/share/cumulus/support/mlx_cmd_file.txt >> mlx_cmd_file.txt
    cat mlx_cmds.txt >> mlx_cmd_file.txt
    mlxcmd mlx_cmds.log cmds --cmds_file mlx_cmd_file.txt
    rm -f mlx_cmds.log mlx_cmd_file.txt mlx_cmds.txt

    exec_cmd mlxfwversion /usr/lib/cumulus/mlxfwupgrade.sx --fwversion
    exec_cmd cl-route-check.show /usr/cumulus/bin/cl-route-check -V
    exec_cmd cl-mroute-check.show /usr/cumulus/bin/cl-mroute-check -V
    exec_cmd $jexec resources cl-resource-query $json
	# else switchd not running or stuck
    }

    [ -x /usr/bin/mstdump ] && {
	 j=0
         while [ $j -lt 1 ]; do
             /usr/bin/mstdump -full mlnxsw-255 > mstdump_$j.dmp
             xz -2 mstdump_$j.dmp &
             # sleep 1
             (( j++ ))
         done
    }

    [ -x /usr/bin/mlxdump ] && {
         j=0
         while [ $j -lt 1 ]; do
            /usr/bin/mlxdump -d mlnxsw-255 snapshot -m full -o mlxdump_$j.udmp
            xz -2 mlxdump_$j.udmp &
            # sleep 1
            (( j++ ))
         done
    }

    #collect cpld dumps
    [ -x /usr/bin/iorw ] && {
         j=0
         while [ $j -lt 1 ]; do
            /usr/bin/iorw -r -b 0x2500 > cpld_$j.dmp
            xz -2 cpld_$j.dmp &
            # sleep 1
            (( j++ ))
         done
    }

    [ -x /usr/bin/mlxlink ] && {
        local p d
        d=$(mst status | grep pciconf | awk '{print $1}' )
        for p in $(ls -v1 /sys/class/net); do
            [ -d $p/bridge -o -d $p/bonding ] && continue;
            p=${p##*/}
            [[ $p =~ ^(sw[0-9]*p[0-9]+(s[0-7])?$) ]] && {
                p=${p#"swp"}
                p=${p%"s0"}
                p=${p/s1/\/2}
                p=${p/s2/\/3}
                p=${p/s3/\/4}
                p=${p/s4/\/5}
                p=${p/s5/\/6}
                p=${p/s6/\/7}
                p=${p/s7/\/8}
                exec_cmd mlxlink sh -c "mlxlink -d ${d} -c -m -e --show_fec -p ${p} | sed 's/[\x1b\[[0-9;]*m//g'" &
            }
        done
        wait
    }

    [ -x /usr/bin/mlxreg ] && {
        d=$(mst status | grep pci_ | awk '{print $1}' )
        exec_cmd mlxreg sh -c "mlxreg -d ${d} --get --reg_name MGIR" &
    }

    [ -x /usr/bin/hw-management-generate-dump.sh ] && {
        /usr/bin/hw-management-generate-dump.sh
        mv /tmp/hw-mgmt-dump.tar.gz /var/log/
    }

    exec_cmd mlxcables.show /usr/lib/cumulus/dump_mlxcables
}

func_fuse()
{
    switchd_ok || return

    date +"%F_%T.%N: ${module}.${func} Copying switchd fuse filesystem"
    [ $SUP_VERBOSE -eq 1 ] && date +"%F_%T.%N: ${module}.${func} Copying switchd fuse filesystem" 1>&2

    mkdir -p $SUP_TOPDIR/cumulus
    cp -a /cumulus/switchd $SUP_TOPDIR/cumulus/switchd |& egrep -v 'Invalid argument'
}

while getopts "c:jl" Option; do
    case $Option in
    c) [ $SUP_VERBOSE -eq 1 ] && # default submods
        echo ${module}: Invoked for "$OPTARG" core dumps 1>&2 ;;
    j) [ $SUP_VERBOSE -eq 1 ] && echo JSON output not yet implemented 1>&2
       jexec=-j json=--json ;;
    l) echo ${funcs[@]} ';' ${nondef_funcs[@]}
       exit 0 ;;
    *) ;;
    esac
done
shift $((OPTIND - 1))

main()
{
    local -r TIMEFORMAT='%2R seconds' tfile=/run/${module}_funcstime$$
    if [ $# -ne 0 ]; then # only run specified sub-modules
       [ $SUP_VERBOSE -eq 1 ] && echo ${module}: run only submodules: $@ 1>&$stderr
       funcs=($@)
    fi

    case "${funcs[@]}" in
        *debug*|*fuse*) enable_debug ;;
    esac

    for func in ${funcs[@]}; do
        [ $SUP_VERBOSE -eq 1 ] && echo "$module.$func" 1>&2
        date +"### $module.$func Started at %F-%T.%N"
        { time func_$func 2>&$stderr ; } 2>$tfile
        read secs < $tfile
        echo "### $module.$func Completed in $secs"
    done
    rm $tfile
    restore_debug
}

TIMEFORMAT="Module $module Completed in %2R seconds"
exec 42>&2
stderr=42

{ time main $@ 2>&$stderr ; } 2>&1

exit 0
