#! /bin/bash
# Copyright (C) 2020-2023 NVIDIA Corporation. ALL RIGHTS RESERVED.
# Copyright 2017,2018,2019,2020 Cumulus Networks,  Inc.  All rights reserved.
#
# Record switchd and related state; used by cl-support
#
# Note that cl-support expects the below three flags to be commented
#TIMEOUT=400
#DEFAULT
#ONCORE=switchd sx_sdk

module=${0##*/}

funcs=(mlx stack fuse)
nondef_funcs=(verbose)
json=
jexec=
if [ -n "$TMPDIR" ]; then tmpdir="$TMPDIR"; else tmpdir=/tmp; fi

declare -i _switchd_ok=1
switchd_ok()
{
    local cpid z

    { [ $_switchd_ok -eq 0 ] &&  return 1 ; } ||
        { [ $_switchd_ok -eq 2 ] && return 0 ; }
    local -r flagf=$tmpdir/flagf$$ pidf=$tmpdir/pid$$

    touch $flagf
    trap "export -n flagf pidf; rm -f $flagf $pidf" RETURN
    systemctl -q is-active switchd || { echo $module: switchd is not running;
        return 1 ; }
    export flagf pidf
    ( (echo $BASHPID > $pidf ; TMOUT=5 read x < /cumulus/switchd/config/logging \
        > $flagf ; echo $x > $flagf) 2>/dev/null &)
    sleep 1.5
    read cpid < $pidf
    TMOUT=8 read z < $flagf
    [ $? -ne 0 ] && { echo $module: switchd not responding
        kill -KILL $cpid 2>/dev/null; _switchd_ok=0 ; return 1; }
    _switchd_ok=2
    return 0
}

declare -i _debug_state_orig=2
declare -r switchd_dbg_ctrl=/cumulus/switchd/ctrl/debug \
    switchd_dbg_dir=/cumulus/switchd/debug

# enable fuse debug nodes if not already enabled.
enable_debug()
{
	switchd_ok || return
	if [ -e $switchd_dbg_dir/bond ]; then
		_debug_state_orig=1
	elif [ -e $switchd_dbg_ctrl ]; then
		echo 1 > $switchd_dbg_ctrl || {
        echo Unable to enable switchd fuse debug ; return 1; }
		_debug_state_orig=0
	fi
    return 0
}

restore_debug()
{
	[ $_debug_state_orig -eq 0 ] && echo 1 > $switchd_dbg_ctrl
}

func_stack()
{
	/bin/ps -C switchd -L -o lwp --no-headers | while read p; do exec_cmd switchd.stack.1 /bin/cat /proc/$p/stack; done
	sleep 0.1
	/bin/ps -C switchd -L -o lwp --no-headers | while read p; do exec_cmd switchd.stack.2 /bin/cat /proc/$p/stack; done
	sleep 0.1
	/bin/ps -C switchd -L -o lwp --no-headers | while read p; do exec_cmd switchd.stack.3 /bin/cat /proc/$p/stack; done
}

# get the stacks for all TIDS, with the thread name, for a given command.
# needs to be a function because exec_cmd() can't take a pipeline
getstacks()
{
	/bin/ps -C $1 -L -o lwp --no-headers | \
	   xargs -l -I'##' awk 'BEGIN {getline comm < "/proc/##/comm"}
		{ printf "%9d %14s: %s\n", ##, comm, $0}' /proc/##/stack
}

# get the status file for all TIDS, with the thread name, for a given command.
# needs to be a function because exec_cmd() can't take a pipeline
getstatus()
{
	/bin/ps -C $1 -L -o lwp --no-headers | \
	   xargs -l -I'##' awk 'BEGIN {getline comm < "/proc/##/comm"}
		{ printf "%9d %14s: %s\n", ##, comm, $0}' /proc/##/status
}

# get the stat file for all TIDS, with the thread name, for a given command.
# needs to be a function because exec_cmd() can't take a pipeline
# This doesn't need the IDs in the printf because they are in the stat file
# and it's a oneline file
getstat()
{
	/bin/ps -C $1 -L -o lwp --no-headers | \
	   xargs -l -I'##' cat /proc/##/stat
}

func_verbose() {
	exec_cmd switchd.stacks getstacks switchd
	exec_cmd switchd.status getstatus switchd
	exec_cmd switchd.stat getstat switchd
}

_mlxcmd=/usr/lib/cumulus/mlxcmd
declare -i mlxcmd_ok=1
mlxcmd()
{
   [ $mlxcmd_ok -eq 0 ] && { return 1 ; }
   [ $mlxcmd_ok -eq 1 ] && { (switchd_ok && mlxcmd_ok=2) || {
     mlxcmd_ok=0 ; return 1 ; } ; }
   file=$1
   shift
   exec_cmd $jexec $file $_mlxcmd $json $@
   return 0
}

func_mlx()
{
    echo "# Taking CRSpace, cplddump, mlxlink, mlxreg dump " > /var/log/sdk_debug_dump.log

    # collect CRSpace dump in Support dir
    [ -x /usr/bin/sx_api_dbg_generate_dump_ext.py ] || return # not mlx
    [ -d /var/log/CRSpace_dump ] && {
        rm /var/log/CRSpace_dump/*
    } || {
        mkdir -p /var/log/CRSpace_dump
    }
    if [ ! -f /run/reboot-required ]; then
        /usr/bin/sx_api_dbg_generate_dump_ext.py --ircore --amber --path /var/log/CRSpace_dump
    else
        echo "Skipping sx_api_dbg_generate_dump_ext.py due to pending required reboot."
    fi

    # if the SDK is not ready, do not do mlxdump commands
    systemctl --quiet is-active sx_sdk.service
    rc=$?
    if [ $rc -ne 0 ]; then
       echo "# skipped sx_api_dbg_generate_dump due to sdk not ready" > /var/log/sdk_debug_dump.log
       return 0
    fi

    # if switchd is not ready, do not do mlxdump commands
    if [ ! -f "/run/switchd.ready" ]; then
       echo "# skipped sx_api_dbg_generate_dump due to switchd not ready" > /var/log/sdk_debug_dump.log
       return 0
    fi

    # collect sx kernel RDQ and SDQ state information.
    echo dump_rdq > /proc/mlx_sx/sx_core
    echo dump_sdq > /proc/mlx_sx/sx_core


    # collect cpld dumps
    [ -x /usr/bin/iorw ] && {
         j=0
         while [ $j -lt 1 ]; do
            /usr/bin/iorw -r -b 0x2500 > cpld_$j.dmp
            xz -2 cpld_$j.dmp &
            # sleep 1
            (( j++ ))
         done
    }

    [ -x /usr/bin/mlxlink ] && {
        local p d ports ports_count
        d=$(mst status | grep pciconf | awk '{print $1}' )
        ports=$(ls -v1 /sys/class/net)
        ports_count=$(echo $ports | wc -w)
        for p in $ports; do
            [ -d $p/bridge -o -d $p/bonding ] && continue;
            p=${p##*/}
            [[ $p =~ ^(sw[0-9]*p[0-9]+(s[0-7])?$) ]] && {
                p=${p#"swp"}
                p=${p%"s0"}
                p=${p/s1/\/2}
                p=${p/s2/\/3}
                p=${p/s3/\/4}
                p=${p/s4/\/5}
                p=${p/s5/\/6}
                p=${p/s6/\/7}
                p=${p/s7/\/8}
                exec_cmd mlxlink sh -c "mlxlink -d ${d} -c -m -e --show_fec -p ${p} | sed 's/\x1b\[[0-9;]*m//g'" &
                if [ $ports_count -gt 127 ]; then
                    sleep 0.3
                fi
            }
        done
        wait
    }

    [ -x /usr/bin/mlxreg ] && {
        d=$(mst status | grep pci_ | awk '{print $1}' )
        exec_cmd mlxreg sh -c "mlxreg -d ${d} --get --reg_name MGIR" &
    }

    [ -x /usr/bin/hw-management-generate-dump.sh ] && {
        /usr/bin/hw-management-generate-dump.sh
        mv /tmp/hw-mgmt-dump.tar.gz /var/log/
    }

    exec_cmd mlxcables.show /usr/lib/cumulus/dump_mlxcables


    #check if sflow is active
    systemctl --quiet is-active hsflowd.service
    hsflowd=$?

    [ -x /usr/bin/sx_api_dbg_generate_dump.py ] || return # not mlx
    /usr/bin/sx_api_dbg_generate_dump.py /var/log/sdk_debug_dump.log
    [ -x /usr/bin/sx_api_flex_acl_dump.py ] && { exec_cmd sx_api_flex_acl_dump.txt /usr/bin/sx_api_flex_acl_dump.py; }
    [ -x $_mlxcmd ] && {
        if [ $hsflowd -eq 0  ]; then
            for cmd in sflow sflow_statistics; do
                echo "--out_file port.${cmd}.show ports show $cmd swp*" >> mlx_cmds.txt
            done
        fi

        for cmd in counters pktdist discards; do
            echo "--out_file port.${cmd}.show ports show $cmd swp*" >> mlx_cmds.txt
        done
        cat /usr/share/cumulus/support/mlx_cmd_file.txt >> mlx_cmd_file.txt
        cat mlx_cmds.txt >> mlx_cmd_file.txt
        mlxcmd mlx_cmds.log cmds --cmds_file mlx_cmd_file.txt
        rm -f mlx_cmds.log mlx_cmd_file.txt mlx_cmds.txt

        exec_cmd mlxfwversion /usr/lib/cumulus/mlxfwupgrade.sx --fwversion
        exec_cmd cl-route-check.show /usr/cumulus/bin/cl-route-check -V
        exec_cmd cl-mroute-check.show /usr/cumulus/bin/cl-mroute-check -V
        exec_cmd $jexec resources cl-resource-query $json
        # else switchd not running or stuck
    }
}

func_fuse()
{
    switchd_ok || return

    date +"%F_%T.%N: ${module}.${func} Copying switchd fuse filesystem"
    [ $SUP_VERBOSE -eq 1 ] && date +"%F_%T.%N: ${module}.${func} Copying switchd fuse filesystem" 1>&2

    mkdir -p $SUP_TOPDIR/cumulus
    cp -a /cumulus/switchd $SUP_TOPDIR/cumulus/switchd |& egrep -v 'Invalid argument'
}

while getopts "c:jl" Option; do
    case $Option in
    c) [ $SUP_VERBOSE -eq 1 ] && # default submods
        echo ${module}: Invoked for "$OPTARG" core dumps 1>&2 ;;
    j) [ $SUP_VERBOSE -eq 1 ] && echo JSON output not yet implemented 1>&2
       jexec=-j json=--json ;;
    l) echo ${funcs[@]} ';' ${nondef_funcs[@]}
       exit 0 ;;
    *) ;;
    esac
done
shift $((OPTIND - 1))

main()
{
    local -r TIMEFORMAT='%2R seconds' tfile=/run/${module}_funcstime$$
    if [ $# -ne 0 ]; then # only run specified sub-modules
       [ $SUP_VERBOSE -eq 1 ] && echo ${module}: run only submodules: $@ 1>&$stderr
       funcs=($@)
    fi

    case "${funcs[@]}" in
        *debug*|*fuse*) enable_debug ;;
    esac

    for func in ${funcs[@]}; do
        [ $SUP_VERBOSE -eq 1 ] && echo "$module.$func" 1>&2
        date +"### $module.$func Started at %F-%T.%N"
        { time func_$func 2>&$stderr ; } 2>$tfile
        read secs < $tfile
        echo "### $module.$func Completed in $secs"
    done
    rm $tfile
    restore_debug
}

TIMEFORMAT="Module $module Completed in %2R seconds"
exec 42>&2
stderr=42

{ time main $@ 2>&$stderr ; } 2>&1

exit 0
