#!/usr/share/venvs/netq-agent/bin/python
#
# Copyright (c) 2017-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# LicenseRef-NvidiaProprietary
#
# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual
# property and proprietary rights in and to this material, related
# documentation and any modifications thereto. Any use, reproduction,
# disclosure or distribution of this material and related documentation
# without an express license agreement from NVIDIA CORPORATION or
# its affiliates is strictly prohibited
#

import json
import subprocess
import os
import sys
import socket
from time import localtime, sleep
from netq_lib.common.utils import subprocess_wrapper, is_sonic, is_cl3, is_cl4, is_bcm
from concurrent import futures


DATE = '%d%d%d' % (localtime().tm_year, localtime().tm_mon, localtime().tm_mday)
TIME = '%d%d%d' % (localtime().tm_hour, localtime().tm_min, localtime().tm_sec)

SUPPORT_PATH = '/var/support/'
EXTENSION = '.txz' if not is_sonic() else '.tar.gz'
TAR_OPTIONS = '-cJvf' if not is_sonic() else '-czvf'
OPTAPREFIX = 'opta'
NETQPREFIX = 'netq'
BASENAME = 'support_%s_%s_%s' % (socket.gethostname(), DATE, TIME)
NETQ_BASENAME = '%s_%s_%s' % (socket.gethostname(), DATE, TIME)
BASENAME_COMBINED = 'support_combined_%s_%s_%s' % (socket.gethostname(), DATE, TIME)
UNIFIED_BASENAME = 'unified_support_%s_%s_%s' % (socket.gethostname(), DATE, TIME)
NETQ_AGENT_PY3_4_VENV_PATH = '/usr/share/venvs/netq-agent/lib/python3.4/site-packages/netq_agent'
NETQ_AGENT_PY3_6_VENV_PATH = '/usr/share/venvs/netq-agent/lib/python3.6/site-packages/netq_agent'
NETQ_AGENT_PY3_7_VENV_PATH = '/usr/share/venvs/netq-agent/lib/python3.7/site-packages/netq_agent'
NETQ_AGENT_PY3_8_VENV_PATH = '/usr/share/venvs/netq-agent/lib/python3.8/site-packages/netq_agent'
NETQ_AGENT_PY3_10_VENV_PATH = '/usr/share/venvs/netq-agent/lib/python3.10/site-packages/netq_agent'
NETQ_AGENT_PY3_11_VENV_PATH = '/usr/share/venvs/netq-agent/lib/python3.11/site-packages/netq_agent'
NETQ_AGENT_PY3_12_VENV_PATH = '/usr/share/venvs/netq-agent/lib/python3.12/site-packages/netq_agent'
NETQ_CLI_CLIENT_PATH = '/tests/netq_agent/agent_cli_client.py'
NETQ_AGENT_VENV_PATH = ''
NETQ_CLIENT_CLI = ''
NETQ_CMD_SCRIPT = 'netq_script.sh'
OPTA_FILE = '/etc/app-release'
COPY_DB_LOGS = 'db_logs'
MASTER_IP_FILE = '/mnt/admin/master_ip'
MASTER_IPV6_FILE = '/mnt/admin/master_ipv6'
WORKER_IP_FILE = '/mnt/admin/worker_ip'
SSH_DIR = '/mnt/admin/ssh'
MASTER_LOCAL_SUPPORT_PATH = '/var/support/'

RC_SUCCESS = 0
RC_FAIL = 1

CMD_TIMEOUT = 60
FAILED_COMMANDS_FILE = 'failed-commands.txt'
KAFKA_CONNECT_SERVICE_IP_COMMAND = ['kubectl', 'get', 'service/kafka-connect', '-n', 'netq-infra', '-o', 'jsonpath={.spec.clusterIP}']
KAFKA_CONNECT_SERVICE_IP = ''

OPTA_FILES = {
    '/var/log/': ['syslog', 'auth', 'netq-agent', 'netqd', 'boot', 'kern', 'containers', 'dpkg', 'kubelet', 'apt'],
    '/etc/': ['os-release', 'app-release', 'resolv.conf', 'netq', 'kubernetes', 'hosts', 'netplan'],
    '/home/nvidia/': ['kube'],
    '/var/run/': ['netq'],
}

PODS_FILES = {
    'cassandra': 'var/log/cassandra'
}

CUMULUS_FILES = {
    '/var/log/': ['syslog', 'netq-agent', 'netqd', 'netq-model', 'boot', 'dpkg', 'auth'],
    '/etc/': ['os-release', 'netq'],
    '/var/run/': ['netq'],
}

OPTA_CMDS = {
    'kubectl-describe.txt': ['kubectl', 'describe', 'pods', '--all-namespaces'],
    'kubectl-getall.txt': ['kubectl', 'get', 'all', '--all-namespaces'],
    'kubectl-describe-node.txt': ['kubectl', 'describe', 'node'],
    'kubectl-top-node.txt': ['kubectl', 'top', 'nodes'],
    'kubectl-top-pod.txt': ['kubectl', 'top', 'pods'],
    'kubectl-get-netqkafka.txt': ['kubectl', 'get', 'netkafka', '-n', 'netq-infra', '-o', 'yaml'],
    'kubectl-get-netqinstaller.txt': ['kubectl', 'get', 'netqinstaller', '-n', 'netq-eth', '-o', 'yaml'],
    'kubectl-get-netqedge.txt': ['kubectl', 'get', 'netqedge', '-n', 'netq-eth', '-o', 'yaml'],
    'kubectl-get-netqclustermanager.txt': ['kubectl', 'get', 'netqclustermanager', '-n', 'netq-eth', '-o', 'yaml'],
    'kubectl-get-netqcentral.txt': ['kubectl', 'get', 'netqcentral', '-n', 'netq-eth', '-o', 'yaml'],
    'kubectl-get-netqapps.txt': ['kubectl', 'get', 'netqapps', '-n', 'netq-eth', '-o', 'yaml'],
    'kubelet-pod-volume-usage.txt': ['bash', '-c', 'du -sh /var/lib/kubelet/pods/*/volumes/*/* 2>/dev/null | sort -hr'],
    'prometheus-volume-stats.txt': [
    'bash', '-c', r'''
PROM_IP=$(kubectl get svc prometheus-k8s -n monitoring -o jsonpath="{.spec.clusterIP}") && \
curl -G "http://$PROM_IP:9090/api/v1/query" \
  --data-urlencode "query=sum by (namespace,persistentvolumeclaim) (kubelet_volume_stats_used_bytes) / 1024 / 1024 / 1024" \
  | jq -r ".data.result[] | \"\\(.metric.namespace)\t\\(.metric.persistentvolumeclaim)\t\\(.value[1]) GB\"" \
  | column -t
'''
],
    "vmstorage-disk-usage.txt": [
        "bash",
        "-c",
        "kubectl -n netq-infra get pods -o name | grep vmstorage-victoria-metrics "
        "| xargs -I{} kubectl -n netq-infra exec {} -- sh -c \"echo -n '{} '; df -h /var/log | awk 'NR==2 {print \\$3, \\$4, \\$5}'\""
    ],
    'crictl-images.txt': ['crictl', 'image', 'ls'],
    'crictl-info.txt': ['crictl', 'info'],
    'crictl-ps.txt': ['crictl', 'ps', '-a'],
    'opta-check.txt': ['opta-check'],
    'netq-show-agents.txt': ['netq', 'show', 'agents'],
    'netq-show-lldp.txt': ['netq', 'show', 'lldp'],
    'netq-show-inv-os.txt': ['netq', 'show', 'inventory', 'os'],
    'netq-show-clag.txt': ['netq', 'show', 'clag'],
    'netq-show-bgp.txt': ['netq', 'show', 'bgp'],
    'netq-show-evpn.txt': ['netq', 'show', 'evpn'],
    'netq-show-ospf.txt': ['netq', 'show', 'ospf'],
    'netq-show-opta-health.txt': ['netq', 'show', 'opta-health'],
    'netq-show-status.txt': ['netq', 'show', 'status', 'verbose'],
    'netq-show-events.txt': ['netq', 'show', 'events', 'between', 'now', 'and', '24h'],
    'netq-show-otlp-endpoints.txt': ['netq', 'show', 'otlp', 'endpoints'],
    'netq-show-otlp-health-verbose.txt': ['netq', 'show', 'otlp', 'health', 'verbose'],
    'netq-show-otlp-security-mode.txt': ['netq', 'show', 'otlp', 'security-mode'],
    'netq-show-otlp-tls-cert.txt': ['netq', 'show', 'otlp', 'tls-cert'],
    'netq-show-otlp-tls-ca-cert.txt': ['netq', 'show', 'otlp', 'tls-ca-cert'],
    'netq-app-admin.txt': "crictl ps -aq label=io.kubernetes.container.name=netq-app-admin",
    'topology-json.txt': ['topology-json'],
    'dmesg.txt': ['dmesg'],
    'lspci.txt': ['lspci'],
    'ls-mnt-r.txt': ['ls', '-lR', '/mnt/'],
    'chrony.txt': ['chronyc', 'tracking'],
    'kubectl-describe-configmap.txt': ['kubectl', 'describe', 'configmap', '--all-namespaces'],
    'kubectl-describe-ingress.txt': ['kubectl', 'describe', 'ingress', '--all-namespaces'],
    'kubectl-describe-clusterrole.txt': ['kubectl', 'describe', 'clusterrole'],
    'kubectl-describe-role.txt': ['kubectl', 'describe', 'role', '--all-namespaces'],
    'kubectl-describe-serviceaccount.txt': ['kubectl', 'describe', 'serviceaccount', '--all-namespaces'],
    'kubectl-get-serviceaccount.txt': ['kubectl', 'get', 'serviceaccount', '--all-namespaces'],
    'kubectl-get-role.txt': ['kubectl', 'get', 'role', '--all-namespaces'],
    'kubectl-get-clusterrole.txt': ['kubectl', 'get', 'clusterrole'],
    'kubectl-get-crd.txt': ['kubectl', 'get', 'crd', '--all-namespaces'],
    'kubectl-get-secret.txt': ['kubectl', 'get', 'secret', '--all-namespaces'],
    'kubectl-get-configmap.txt': ['kubectl', 'get', 'configmap', '--all-namespaces'],
    'kubectl-get-ingress.txt': ['kubectl', 'get', 'ingress', '--all-namespaces'],
    'kafka-connector-status.txt': ['curl', 'http://{service_ip}:8083/connectors?expand=status'],
    'opta-info.txt': ['/usr/sbin/opta-info.py'],
    'cassandra-tables-disk-usage.txt': ['du', '-h', '/mnt/cassandra'],
    'cassandra-describe-cluster.txt': ['bash', '/usr/sbin/netq-cassandra-information-dump.sh'],
    'netqd-file-descriptors.txt': ['sh', '-c', 'true'],
    'kubectl-get-strimzi.txt': ['kubectl', 'get', 'kafkas,kafkaconnects,kafkatopics,kafkausers', '--all-namespaces', '-o', 'wide'],
    'kubectl-describe-strimzi.txt': ['kubectl', 'describe', 'kafkas,kafkaconnects', '--all-namespaces'],
    'kubectl-get-connectors.txt': ['kubectl', 'get', 'kafkaconnectors.kafka.strimzi.io', '--all-namespaces', '-o', 'wide'],
    'kubectl-describe-connectors.txt': ['kubectl', 'describe', 'kafkaconnectors.kafka.strimzi.io', '--all-namespaces'],
}

LINUX_CMDS = {
    'df-h.txt': ['df', '-h'],
    'lsmod.txt': ['lsmod'],
    'top-n1.txt': ['top', '-b', '-n1'],
    'ss-pan.txt': ['ss', '-pan'],
    'free-lm.txt': ['free', '-lm'],
    'ps-aux.txt': ['ps', 'aux'],
    'systemctl.txt': ['systemctl', '-l', '--type', 'service', '--all'],
    'ip-link-show.txt': ['ip', '-d', 'link', 'show'],
    'ip-addr-show.txt': ['ip', 'addr', 'show'],
    'ip-route-show-table-all.txt': ['ip', 'route', 'show', 'table', 'all'],
    'ip-neighbor-show.txt': ['ip', 'neighbor', 'show'],
    'brctl-show.txt': ['brctl', 'show'],
    'bridge-fdb-show.txt': ['bridge', 'fdb', 'show'],
    'bridge-vlan-show.txt': ['bridge', 'vlan', 'show'],
    'ntpq.txt': ['ntpq', '-pn'],
    'timedatectl.txt': ['timedatectl'],
    'netq-agent-cli.txt': ['python', NETQ_CLIENT_CLI],
    'proc_meminfo.txt': ['cat', '/proc/meminfo'],
    'proc_cpuinfo.txt': ['cat', '/proc/cpuinfo'],
    'journalctl_containerd.txt': ['journalctl', '--no-pager', '-u', 'containerd'],
    'env_nvidia.txt': ['sudo', 'su', '-l', 'nvidia', '-c', 'printenv'],
    'env_root.txt': ['sudo', 'su', '-l', 'root', '-c', 'printenv'],
    'ansible_log.txt': ['cat', '/mnt/admin/tmp/ansible.log'],
    'ulimit.txt': ['bash', '-c', 'ulimit -a'],
    'threads-max.txt': ['cat', '/proc/sys/kernel/threads-max'],
    'vm_backuprestore_log.txt': ['cat', '/var/log/vm-backuprestore.log'],
    'netqd-file-descriptors.txt': ['sh', '-c', 'true'],
}

DEBIAN_UBUNTU_FILES = {
    '/var/log/': ['syslog', 'netq-agent', 'netqd', 'netq-model'],
    '/etc/': ['os-release', 'netq'],
    '/var/run/': ['netq'],
}
DEBIAN_UBUNTU_CMDS = {
    'dpkg.txt': ['dpkg', '-l'],
}
CENTOS_RHEL_FILES = {
    '/var/log/': ['messages', 'netq-agent', 'netqd', 'netq-model'],
    '/etc/': ['os-release', 'netq'],
    '/var/run/': ['netq'],
}
CENTOS_RHEL_CMDS = {
    'rpm-qa.txt': ['rpm', '-qa']
}

SONIC_FILES = {
    '/var/log/': ['syslog', 'netq-agent', 'netqd', 'netq-model', 'boot', 'dpkg', 'auth'],
    '/etc/': ['os-release', 'netq'],
    '/etc/sonic/': ['sonic_version.yml'],
    '/var/run/': ['netq'],
}

SONIC_CMDS = {
    'config-db-json.txt': ['/usr/local/bin/sonic-cfggen', '-d', '--print-data'],
}

def get_netqd_fd_count():
    """Get file descriptor count for netqd process using lsof."""
    try:
        # Find netqd process PID
        ps_cmd = ['ps', 'aux']
        result = subprocess.run(ps_cmd, capture_output=True, text=True, timeout=10)
        if result.returncode != 0:
            return "Error: Could not get process list"

        netqd_pid = None
        for line in result.stdout.split('\n'):
            if 'netqd' in line and 'grep' not in line:
                parts = line.split()
                if len(parts) > 1:
                    netqd_pid = parts[1]
                    break

        if not netqd_pid:
            return "netqd process not found"

        # Get file descriptor count using lsof
        lsof_cmd = ['lsof', '-p', netqd_pid]
        lsof_result = subprocess.run(lsof_cmd, capture_output=True, text=True, timeout=30)

        if lsof_result.returncode != 0:
            return f"Error: lsof failed for PID {netqd_pid} - {lsof_result.stderr}"

        # Count lines (each line represents an open file descriptor)
        fd_count = len(lsof_result.stdout.strip().split('\n')) - 1  # Subtract header line

        # Also get system limits for context
        try:
            import resource
            soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
            return f"netqd PID: {netqd_pid}\nFile descriptors in use: {fd_count}\nSystem soft limit: {soft_limit}\nSystem hard limit: {hard_limit}\n\nDetailed lsof output:\n{lsof_result.stdout}"
        except ImportError:
            return f"netqd PID: {netqd_pid}\nFile descriptors in use: {fd_count}\n\nDetailed lsof output:\n{lsof_result.stdout}"

    except subprocess.TimeoutExpired:
        return "Error: Command timed out while getting file descriptor information"
    except Exception as e:
        return f"Error: {str(e)}"

def is_opta():
    return os.path.isfile(OPTA_FILE)


def is_cumulus():
    return (is_cl3() or is_cl4())


def platform():
    sysname, nodename, release, version, machine = os.uname()

    if os.path.isdir(NETQ_AGENT_PY3_4_VENV_PATH):
        NETQ_AGENT_VENV_PATH = NETQ_AGENT_PY3_4_VENV_PATH
    elif os.path.isdir(NETQ_AGENT_PY3_6_VENV_PATH):
        NETQ_AGENT_VENV_PATH = NETQ_AGENT_PY3_6_VENV_PATH
    elif os.path.isdir(NETQ_AGENT_PY3_7_VENV_PATH):
        NETQ_AGENT_VENV_PATH = NETQ_AGENT_PY3_7_VENV_PATH
    elif os.path.isdir(NETQ_AGENT_PY3_8_VENV_PATH):
        NETQ_AGENT_VENV_PATH = NETQ_AGENT_PY3_8_VENV_PATH
    elif os.path.isdir(NETQ_AGENT_PY3_10_VENV_PATH):
        NETQ_AGENT_VENV_PATH = NETQ_AGENT_PY3_10_VENV_PATH
    elif os.path.isdir(NETQ_AGENT_PY3_11_VENV_PATH):
        NETQ_AGENT_VENV_PATH = NETQ_AGENT_PY3_11_VENV_PATH
    elif os.path.isdir(NETQ_AGENT_PY3_12_VENV_PATH):
        NETQ_AGENT_VENV_PATH = NETQ_AGENT_PY3_12_VENV_PATH
    else:
        NETQ_AGENT_VENV_PATH = ''
        major = sys.version_info.major
        minor = sys.version_info.minor
        print('skipping model dump - python version {}.{} - not supported'.format(major, minor))

    if len(NETQ_AGENT_VENV_PATH):
        NETQ_CLIENT_CLI = '%s%s' % (NETQ_AGENT_VENV_PATH, NETQ_CLI_CLIENT_PATH)
        LINUX_CMDS.update({'netq-agent-cli.txt': ['python3', NETQ_CLIENT_CLI]})

    if is_opta():
        OPTA_CMDS.update(LINUX_CMDS)
        OPTA_CMDS.update(DEBIAN_UBUNTU_CMDS)
        return OPTA_FILES, OPTA_CMDS
    elif 'Cumulus' in version:
        LINUX_CMDS.update(DEBIAN_UBUNTU_CMDS)
        return CUMULUS_FILES, LINUX_CMDS
    elif 'Ubuntu' in version or 'Debian' in version:
        DEBIAN_UBUNTU_CMDS.update(LINUX_CMDS)
        if is_sonic():
            DEBIAN_UBUNTU_CMDS.update(SONIC_CMDS)
            return SONIC_FILES, DEBIAN_UBUNTU_CMDS
        return DEBIAN_UBUNTU_FILES, DEBIAN_UBUNTU_CMDS
    elif 'el7' in release:
        try:
            # brctl is not installed by default
            subprocess_wrapper(['yum', 'install', 'bridge-utils.x86_64', '-y'])
        except subprocess.CalledProcessError as ex:
            print('ERROR: {}'.format(ex.cmd))
        CENTOS_RHEL_CMDS.update(LINUX_CMDS)
        return CENTOS_RHEL_FILES, CENTOS_RHEL_CMDS

    else: # rhel
        try:
            # brctl is not installed by default
            subprocess_wrapper(['yum', 'install', 'bridge-utils.x86_64', '-y'])
        except subprocess.CalledProcessError as ex:
            print('ERROR: {}'.format(ex.cmd))
        CENTOS_RHEL_CMDS.update(LINUX_CMDS)
        return CENTOS_RHEL_FILES, CENTOS_RHEL_CMDS


def copy_files(file_dict, support_path):
    cp_files = []
    for path in file_dict:
        if not os.path.exists(path):
            continue
        for file in os.listdir(path):
            for file_pattern in file_dict[path]:
                if file_pattern in file:
                    cp_files.append('%s%s' % (path, file))

    try:
        for file in cp_files:
            # Avoid dereferencing symlinks in /var/log/containers to prevent cp failures on broken links
            if file == '/var/log/containers' or file.startswith('/var/log/containers/'):
                subprocess_wrapper(['cp', '-r', '-L', '--parents', file, support_path + '/Logs'])
            else:
                subprocess_wrapper(['cp', '-r', '-L', '--parents', file, support_path + '/Logs'])
    except subprocess.CalledProcessError as ex:
        print('ERROR: failed to {}'.format(ex.cmd))
        return RC_FAIL
    else:
        return RC_SUCCESS


def copy_pods_files(support_path):
    try:
        for pod_name in PODS_FILES:
            copy_pod_files(pod_name, PODS_FILES[pod_name], support_path)
    except Exception as ex:
        print('ERROR: failed to {}'.format(ex.cmd))
        return RC_FAIL
    else:
        return RC_SUCCESS


def copy_pod_files(pod_name, directory_path, support_path):
    full_pod_names = get_full_pod_name(pod_name)
    for full_pod_name in full_pod_names:
        namespace, pod_name_only = full_pod_name.split('/', 1)
        kubectl_exec_cassandra_command = f'kubectl cp -n {namespace} {pod_name_only}:{directory_path} {support_path}/Logs/{full_pod_name}'
        shell_process = subprocess.Popen(kubectl_exec_cassandra_command, stdin=subprocess.PIPE,
                stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)


def get_full_pod_name(pod_name):
    kubectl_get_pod_command = f"kubectl get pods --all-namespaces | grep {pod_name}"
    output = subprocess.check_output(kubectl_get_pod_command, shell=True)
    lines = output.decode("utf-8").split("\n")
    full_name_pods = []
    for line in lines:
        if pod_name in line:
            parts = line.split()
            if len(parts) >= 2:
                namespace = parts[0]
                pod_name_full = parts[1]
                full_name_pods.append(f"{namespace}/{pod_name_full}")
    return full_name_pods


def cmd_outputs(cmd_dict, support_path):
    devnull = open(os.devnull, 'w')
    failed_cmds = ''
    try:
        for file in cmd_dict:
            if file == 'systemctl.txt':
                output = subprocess_wrapper(cmd_dict[file]).split()
                services = set()
                for word in output:
                    if 'service' in word:
                        services.add(word)

                properties = 'Names,ActiveState,UnitFilePreset,MainPID,ActiveEnterTimestamp'
                run_cmd = ['systemctl', 'show', '-p', properties] + list(services)
            elif file == 'netq-app-admin.txt':
                output = subprocess_wrapper(cmd_dict[file].split())
                with open(NETQ_CMD_SCRIPT, 'w') as fd:
                    os.chmod("netq_script.sh", 0o744)
                    fd.write("#!/bin/sh\n")
                    for container in output.splitlines():
                        fd.write("echo %%%%% Logs of {} %%%%%\n".format(container))
                        fd.write("crictl logs {}\n".format(container))
                    run_cmd = './' + NETQ_CMD_SCRIPT
            elif type(cmd_dict[file]) == str:
                # call this in a temp bash script
                with open(NETQ_CMD_SCRIPT, 'w') as fd:
                    os.chmod("netq_script.sh", 0o744)
                    fd.write("#!/bin/sh\n")
                    fd.write(cmd_dict[file] + "\n")
                    run_cmd = './' + NETQ_CMD_SCRIPT
            elif file == 'kafka-connector-status.txt' :
                try:
                    KAFKA_CONNECT_SERVICE_IP=subprocess.check_output(KAFKA_CONNECT_SERVICE_IP_COMMAND,text=True, timeout=CMD_TIMEOUT,stderr=devnull)
                    run_cmd = cmd_dict[file]
                    run_cmd[1] = run_cmd[1].format(service_ip=KAFKA_CONNECT_SERVICE_IP)
                except Exception:
                    pass
            elif file == 'netqd-file-descriptors.txt':
                # Special handling for file descriptor count
                fd_info = get_netqd_fd_count()
                with open(file, 'w') as fp:
                    fp.write(fd_info)
                subprocess_wrapper(['mv', file, support_path + '/Support'])
                continue
            else:
                run_cmd = cmd_dict[file]

            with open(file, 'w') as fp:
                try:
                    subprocess.check_call(run_cmd, stdout=fp, stderr=devnull, timeout=CMD_TIMEOUT)
                    subprocess_wrapper(['mv', file, support_path + '/Support'])
                except Exception as ex:
                    reattempt_status = RC_FAIL
                    if is_opta() and ' '.join(run_cmd) == "netq show agents":
                        with open(file, 'r') as fp1:
                            line = fp1.readline()
                            if "Login Failed" in line or "Access key is not found" in line:
                                print("Access key is not found. Please check the access key entered or generate a fresh access_key,secret_key pair and add it to the CLI configuration")
                                print("Proceeding with opta-support generation without netq show outputs")
                    subprocess_wrapper(['rm', '-f', file])
                    if reattempt_status != RC_SUCCESS:
                        failed_cmds += "Command: %s, Error Output: %s\n" % (' '.join(run_cmd), str(ex))

        if failed_cmds:
            fd_failed_cmds = open(FAILED_COMMANDS_FILE, 'a')
            fd_failed_cmds.write(failed_cmds)
            fd_failed_cmds.close()
            subprocess_wrapper(['mv', FAILED_COMMANDS_FILE, support_path + '/Support'])

            try:
                os.remove(NETQ_CMD_SCRIPT)
            except Exception:
                pass

    except subprocess.CalledProcessError as ex:
        print('ERROR: failed to {}'.format(ex.cmd))
        return RC_FAIL
    else:
        return RC_SUCCESS


def _read_config_from_file(filename):
    """ Read config from file.

    Args:
        filename(str): Name of config file.

    Returns:
        string: Content of config file if file exists else an empty string.
    """
    content = None
    if os.path.exists(filename):
        with open(filename, "r") as fd:
            content = fd.readline()
    return content


def _read_worker_ip_config_from_file(filename):
    """
    Read multi line config from file.
    Args:
        filename(str): Name of config file.

    Returns:
        string: Content of config file if file exists else an empty string.
    """
    worker_ips = None
    if os.path.exists(filename):
        with open(filename, "r") as fd:
            worker_ips = [line.strip() for line in fd]
    return worker_ips


def _run_on_worker(worker_ip, command):
    """ Method to run a command on worker node.

    Args:
        worker_ip: Ip of worker node.
        command: Command to run.

    Returns: None

    Raises: None
    """
    ssh_cmd = 'ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=5 -qi {}/master ' \
                          'netq-admin@{} sudo {}'.format(SSH_DIR, worker_ip, command)

    if is_bcm():
        ssh_cmd = 'ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=5 ' \
                          'root@{} sudo {}'.format(worker_ip, command)
    return subprocess.check_output(ssh_cmd, shell=True)


def _copy_from_worker_to_master(worker_ip, remote_file_path, local_path):
    """ Method to run a command on worker node.

    Args:
        worker_ip: Ip of worker node.
        command: Command to run.

    Returns: None

    Raises: None
    """
    ssh_cmd = 'scp -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=5 -qi {}/master ' \
                          'netq-admin@{}:{} {}'.format(SSH_DIR, worker_ip, remote_file_path , local_path)

    if is_bcm():
        ssh_cmd = 'scp -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=5 ' \
                          'root@{}:{} {}'.format(worker_ip, remote_file_path , local_path)
    return subprocess.check_output(ssh_cmd, shell=True)


def combine_tarballs(WORKER_SUPPORT_FILE_LIST):
    if is_opta():
        COMBINED_SUPPORT_NAME = OPTAPREFIX + '_' + BASENAME_COMBINED
        MASTER_SUPPORT_NAME = OPTAPREFIX + '_' + BASENAME
    else:
        COMBINED_SUPPORT_NAME = NETQPREFIX + '_' + BASENAME_COMBINED
        MASTER_SUPPORT_NAME = NETQPREFIX + '_' + BASENAME
    COMBINED_SUPPORT_FILE = '%s%s%s' % (SUPPORT_PATH, COMBINED_SUPPORT_NAME, EXTENSION)
    MASTER_SUPPORT_FILE = '%s%s' % (MASTER_SUPPORT_NAME, EXTENSION)
    COMBINED_SUPPORT_FILE_LIST = WORKER_SUPPORT_FILE_LIST + [MASTER_SUPPORT_FILE]
    if not COMBINED_SUPPORT_FILE_LIST:
        print("No tar files found in the directory.")
        return
    try:
        subprocess_wrapper(['/bin/tar', '--directory=/var/support', TAR_OPTIONS, COMBINED_SUPPORT_FILE] + COMBINED_SUPPORT_FILE_LIST)
        print('Please send {} to Nvidia support.'.format(COMBINED_SUPPORT_FILE))
    except subprocess.CalledProcessError as ex:
        print('ERROR: could not create combined support file ({})'.format(ex.cmd))
        pass


def is_node_up(host):
    try:
        response = subprocess.run(
            ["ping", "-c", "1", host],
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            timeout=5
        )
        return response.returncode == 0
    except subprocess.TimeoutExpired:
        return False


def create_opta_support_on_worker(worker_ip_list):
    command = 'is_worker=True opta-support'
    with futures.ThreadPoolExecutor() as executor:
        future_to_ip = {executor.submit(_run_on_worker, worker_ip, command): worker_ip for worker_ip in worker_ip_list}
        for future in futures.as_completed(future_to_ip):
            ip = future_to_ip[future]
            msg = f'opta-support failed on {ip}.'
            try:
                output = future.result().decode('utf8').strip()
                if "ERROR" in output:
                    raise RuntimeError(msg + output)
            except Exception as e:
                raise RuntimeError(msg + output)


def create_unified_support_structure(is_worker):
    """Create unified support directory structure"""
    unified_support_path = f'/tmp/{BASENAME_COMBINED}'
    eth_logs_path = f'{unified_support_path}/eth_logs'
    observability_path = f'{unified_support_path}/observability_data'

    # Create directory structure
    subprocess_wrapper(['mkdir', '-p', unified_support_path])
    if not is_worker:
        subprocess_wrapper(['mkdir', '-p', eth_logs_path])
        subprocess_wrapper(['mkdir', '-p', observability_path])

    return unified_support_path, eth_logs_path, observability_path


def collect_eth_logs_to_directory(output_dir, is_worker):
    if is_worker:
        opta_support_to_directory(output_dir, is_cluster=True)
    else:
        master_ip = _read_config_from_file(MASTER_IP_FILE)
        master_ip_ipv6 = _read_config_from_file(MASTER_IPV6_FILE)
        worker_ip_list = _read_worker_ip_config_from_file(WORKER_IP_FILE)
        is_cluster = bool(worker_ip_list and [ip for ip in worker_ip_list if ip.strip()])
        opta_support_to_directory(output_dir, is_cluster)

        if is_cluster:
            # Handle cluster scenario - collect from workers and copy to output_dir
            WORKER_SUPPORT_FILE_LIST = []
            active_worker_ip_list = []
            for worker_ip in worker_ip_list:
                if not is_node_up(worker_ip):
                    print("ERROR : node ip {} is unreachable. Skipping opta-support for this node".format(worker_ip))
                    continue
                active_worker_ip_list.append(worker_ip)
            try:
                create_opta_support_on_worker(active_worker_ip_list)
            except Exception as e:
                print("Failed to collect opta-support from worker nodes. Please check if worker nodes are initialized and reachable and try again.")
                sys.exit(1)
            for worker_ip in active_worker_ip_list:
                msg = f'Failed to get latest opta-support file on {worker_ip}.'
                command = '"ls -t /var/support/*.txz | head -n 1"'
                latest_file = ""
                try:
                    latest_file = _run_on_worker(worker_ip, command).decode('utf8').strip()
                    if "ERROR" in latest_file:
                        raise RuntimeError(msg + latest_file)
                except subprocess.CalledProcessError as e:
                    raise RuntimeError(msg + latest_file)

                if latest_file:
                    msg = f'Failed to copy latest opta-support file on {worker_ip} to eth_logs directory.'
                    remote_file_path = latest_file
                    local_path = output_dir
                    try:
                        output = _copy_from_worker_to_master(worker_ip, remote_file_path, local_path).decode('utf8').strip()
                        if "ERROR" in output:
                            raise RuntimeError(msg + output)
                    except subprocess.CalledProcessError as e:
                        raise RuntimeError(msg + output)
    return RC_SUCCESS


def opta_support_to_directory(output_dir, is_cluster):
    prog_name = os.path.basename(sys.argv[0])
    if len(sys.argv) > 1:
        copy_db_logs = COPY_DB_LOGS in sys.argv

    if os.getuid():
        print('Must run {} as root'.format(prog_name))
        sys.exit(1)

    if is_opta():
        print('Generating opta-support archive. Process takes few minutes to complete...')
    file_dict, cmd_dict = platform()

    if is_opta():
        SUPPORT_NAME = OPTAPREFIX + '_' + BASENAME
    else:
        SUPPORT_NAME = NETQPREFIX + '_' + BASENAME

    TMP_SUPPORT_PATH = f'{output_dir}/{SUPPORT_NAME}'

    subprocess_wrapper(['mkdir', '-p', TMP_SUPPORT_PATH])
    subprocess_wrapper(['mkdir', TMP_SUPPORT_PATH + '/Logs'])
    subprocess_wrapper(['mkdir', TMP_SUPPORT_PATH + '/Support'])

    if is_cumulus():
        CUMULUS_SUPPORT_NAME = '/cl-support'
        CL_SUPPORT_PATH = TMP_SUPPORT_PATH + (CUMULUS_SUPPORT_NAME)
        subprocess_wrapper(['mkdir', CL_SUPPORT_PATH])
        print('Collecting cl-support...')
        try:
            subprocess_wrapper(['cl-support', '-s', '-S', CL_SUPPORT_PATH, '-p', \
                                'netq-cl', '-r', '\"triggered from netq-support\"', '-T', '300', '-j'])
            print('Collecting netq-support...')
        except subprocess.CalledProcessError as ex:
            print('cl-support collection did not succeed. collecting netq-support alone')
            pass

        nvue_client = None
        try:
            # Use NVUE API client to get platform information
            from netq_agent.cmd.nvue_api_client import create_nvue_client
            nvue_client = create_nvue_client()
            json_output = nvue_client.get_platform_hardware()

            if json_output and json_output.get("asic-model") not in ["Spectrum", "n/a"]:
                NETQ_SPICE_SUPPORT = 'python /sbin/netq-spice-support'
                subprocess_wrapper(NETQ_SPICE_SUPPORT.split())

        except Exception as e:
            print("Error getting platform information via NVUE API:", e)
            pass
        finally:
            # Clean up NVUE client connection
            if nvue_client:
                nvue_client.close()

    cmd_outputs(cmd_dict, TMP_SUPPORT_PATH)
    sleep(3)
    copy_files(file_dict, TMP_SUPPORT_PATH)
    if 'copy_db_logs' in locals() and copy_db_logs:
        copy_pods_files(TMP_SUPPORT_PATH)


def collect_nvl_observability_data_to_directory(output_dir, img_build_version):
    try:
        # Change to the output directory before running the script
        original_cwd = os.getcwd()
        os.chdir(output_dir)

        if not img_build_version:
            raise ValueError("ERROR: Build version is not recognized, cannot collect NVL observability data")

        # Run the observability data extraction script
        result = subprocess.run(["bash", "/opt/netq-admin/nvl/scripts/extract-observability-data.sh", str(is_bcm()).lower(), img_build_version])

        # Change back to original directory
        os.chdir(original_cwd)

        if result.returncode != 0:
            print(f"WARNING: extract-observability-data.sh failed with return code {result.returncode}")
            return RC_FAIL

        return RC_SUCCESS

    except Exception as e:
        print(f"ERROR: Failed to collect observability data: {e}")
        os.chdir(original_cwd)
        return RC_FAIL


def get_installation_mode():
    try:
        netq_status = subprocess.run(["netq", "show", "status", "json"], capture_output=True, text=True, check=True)
        netq_status_json = json.loads(netq_status.stdout)
        installation_mode = netq_status_json.get("installation_mode", "").lower()
        img_build_version = netq_status_json.get("installer_version", "")

        modes = {
            "combined": (True, True),
            "nvlink": (False, True),
            "ethernet": (True, False),
        }

        # in case the installation mode isn't available, run the default OPTA-support - Ethernet.
        is_eth_installed, is_nvl_installed = modes.get(installation_mode, (True, False))
        return is_eth_installed, is_nvl_installed, img_build_version

    except Exception as e:
        print(f"Failed to get installation status, proceeding with default values.")
        # Return default values: Ethernet mode enabled, NVL disabled, empty version
        # This ensures the function always returns a tuple for unpacking
        return True, False, ""


def check_collection_results(is_eth, eth_result, is_nvl, nvl_result):
    if is_eth and eth_result != RC_SUCCESS:
        print("ERROR: Ethernet log collection failed")
    if is_nvl and nvl_result != RC_SUCCESS:
        print("ERROR: NVL observability data collection failed")


def cleanup_temp_dir(path):
    try:
        if os.path.exists(path):
            subprocess_wrapper(['rm', '-rf', path])
    except Exception as e:
        print(f"WARNING: Failed to clean up temporary directory {path}: {e}")


def create_unified_archive(unified_path):
    unified_basename = os.path.basename(unified_path)
    unified_tar_file = f'{SUPPORT_PATH}{unified_basename}{EXTENSION}'

    try:
        if not os.path.isdir(SUPPORT_PATH):
            subprocess_wrapper(['mkdir', SUPPORT_PATH])
        subprocess_wrapper(['/bin/tar', '--directory=/tmp', TAR_OPTIONS, unified_tar_file, unified_basename])
        print(f'Please send {unified_tar_file} to Nvidia support.')
    except subprocess.CalledProcessError as ex:
        print(f'ERROR: could not create unified support file ({ex.cmd})')


def main():
    is_worker = os.getenv('is_worker')

    if is_cumulus() and not is_opta():
        prog_name = os.path.basename(sys.argv[0])
        if len(sys.argv) > 1:
            copy_db_logs = COPY_DB_LOGS in sys.argv

        if os.getuid():
            print('Must run {} as root'.format(prog_name))
            sys.exit(1)

        file_dict, cmd_dict = platform()
        SUPPORT_NAME = NETQPREFIX + '_support_' + NETQ_BASENAME
        SUPPORT_FILE = '%s%s%s' % (SUPPORT_PATH, SUPPORT_NAME, EXTENSION)
        TMP_SUPPORT_PATH = '/tmp/' + SUPPORT_NAME

        subprocess_wrapper(['mkdir', '-p', TMP_SUPPORT_PATH])
        subprocess_wrapper(['mkdir', TMP_SUPPORT_PATH + '/Logs'])
        subprocess_wrapper(['mkdir', TMP_SUPPORT_PATH + '/Support'])

        # Cumulus-specific collection
        CUMULUS_SUPPORT_NAME = '/cl-support'
        CL_SUPPORT_PATH = TMP_SUPPORT_PATH + (CUMULUS_SUPPORT_NAME)
        subprocess_wrapper(['mkdir', CL_SUPPORT_PATH])
        print('Collecting cl-support...')
        try:
            subprocess_wrapper(['cl-support', '-s', '-S', CL_SUPPORT_PATH, '-p', \
                                'netq-cl', '-r', '\"triggered from netq-support\"', '-T', '300'])
            print('Collecting netq-support...')
        except subprocess.CalledProcessError as ex:
            print('cl-support collection did not succeed. collecting netq-support alone')
            pass

        nvue_client = None
        try:
            # Use NVUE API client to get platform information
            from netq_agent.cmd.nvue_api_client import create_nvue_client
            nvue_client = create_nvue_client()
            json_output = nvue_client.get_platform_hardware()

            if json_output and json_output.get("asic-model") not in ["Spectrum", "n/a"]:
                NETQ_SPICE_SUPPORT = 'python /sbin/netq-spice-support'
                subprocess_wrapper(NETQ_SPICE_SUPPORT.split())

        except Exception as e:
            print("Error getting platform information via NVUE API:", e)
            pass
        finally:
            # Clean up NVUE client connection
            if nvue_client:
                nvue_client.close()

        cmd_outputs(cmd_dict, TMP_SUPPORT_PATH)
        sleep(3)
        copy_files(file_dict, TMP_SUPPORT_PATH)
        if 'copy_db_logs' in locals() and copy_db_logs:
            copy_pods_files(TMP_SUPPORT_PATH)

        try:
            if not os.path.isdir(SUPPORT_PATH):
                subprocess_wrapper(['mkdir', SUPPORT_PATH])
            subprocess_wrapper(['/bin/tar', '--directory=/tmp', TAR_OPTIONS, SUPPORT_FILE, SUPPORT_NAME])
            print('Please send {} to Nvidia support.'.format(SUPPORT_FILE))
        except subprocess.CalledProcessError as ex:
            print('ERROR: could not create support file ({})'.format(ex.cmd))

        # Clean up temporary directory
        try:
            subprocess_wrapper(['rm', '-rf', TMP_SUPPORT_PATH])
        except Exception as e:
            print(f"WARNING: Failed to clean up temporary directory {TMP_SUPPORT_PATH}: {e}")
        return

    # For OPTA systems, use unified support structure
    eth_success = RC_SUCCESS
    observability_success = RC_SUCCESS

    unified_path, eth_logs_dir, observability_dir = create_unified_support_structure(is_worker)

    try:
        is_eth_installed, is_nvl_installed, img_build_version = get_installation_mode()

        if is_eth_installed:
            eth_success = collect_eth_logs_to_directory(eth_logs_dir, is_worker)

        if is_nvl_installed and not is_worker:
            observability_success = collect_nvl_observability_data_to_directory(observability_dir, img_build_version)

        # Indicate if either collection failed
        check_collection_results(is_eth_installed, eth_success, is_nvl_installed, observability_success)

        # Create unified tar file
        if eth_success == RC_SUCCESS or observability_success == RC_SUCCESS:
            create_unified_archive(unified_path)
        else:
            print("ERROR: Both eth logs and observability data collection failed")

    except Exception as e:
        print(f"ERROR: logs collection failed: {e}")

    finally:
        cleanup_temp_dir(unified_path)


if __name__ == '__main__':
    main()
