#!/bin/bash

<<COMMENT
This script is troubleshoot billing-related issues.
Author: zhaochangming
Created: 2023-03-24 11:39:00
Version: 1.0.0
COMMENT

# Set environment
export PATH=/bin/:/sbin:/usr/bin/:/usr/sbin:/usr/local/sbin/:/usr/local/bin/:$PATH
CURTIME=$(date +%Y%m%d%H%M%S)
. /etc/init.d/functions

# Set variable
ne_url='pg-monitor.onething.net'
miner_url='report-miner.onethingpcs.com'
zqbpc_url='api-zqbpc.onethingpcs.com'
netdev_route=$(ip route get 223.5.5.5 | awk 'NR==1{print$5}')
netdev_pr=$(ip route ls table pr_table_admin | grep "^default" | awk '{print$5}')
dns_nameserver=$(awk 'NR==1 && /nameserver/{print$2}' /etc/resolv.conf)  


# Check network connectivity using ping
check_ping() {
    local host=$1
    ping -c 8 -w 8 $host -q &>/dev/null && return 0 || return 1
}

# Switch the reporting line to the management line 
func_switch_management_line(){
    local interface=$1
    if check_ping "-I ${interface} ${ne_url}";then
        ip rule add from all to ${dns_nameserver} lookup pr_table_admin pref 30
        ne_server_ip=$(nslookup ${ne_url} ${dns_nameserver} | grep 'Address:' | grep -v '#' | awk '{print $2}') 
        miner_server_ip=$(nslookup ${miner_url} ${dns_nameserver} | grep 'Address:' | grep -v '#' | awk '{print $2}') 
        zqbpc_server_ip=$(nslookup ${zqbpc_url} ${dns_nameserver} | grep 'Address:' | grep -v '#' | awk '{print $2}') 
        ip rule add from all to ${ne_server_ip} lookup pr_table_admin pref 30
        ip rule add from all to ${miner_server_ip} lookup pr_table_admin pref 30
        ip rule add from all to ${zqbpc_server_ip} lookup pr_table_admin pref 30
        cat >> /etc/rc.local <<EOF
ip rule add from all to "${ne_server_ip}" lookup pr_table_admin pref 30
ip rule add from all to "${miner_server_ip}" lookup pr_table_admin pref 30
ip rule add from all to "${zqbpc_server_ip}" lookup pr_table_admin pref 30
EOF
        check_ping ${ne_url}
    else
        echo "基础检查:出口网络不通,业务线路异常,切换管理线路,管理线路异常,请联系基础运维处理."
    fi

}

# 判断当前上网线路
func_cur_internet_line(){
    [ "${netdev_route}" != ${netdev_pr} ] && return 0 || return 1
}


# 判断DNS是否劫持 
func_dns_hijack(){
    ns_ip=$(nslookup  tw15b0178.onething.net | grep 'Address:' | grep -v '#' | awk '{print $2}')
    [ $(echo ${ns_ip} | grep -c 123.182.50.3) -eq 0 ] && return 0 || return 1 
}


# 1.1 Check network environment
func_basic_env_network_check(){
    
    # 检查主机上网线路连通性
    if check_ping 223.5.5.5;then
        action "基础检查:主机出口网络连通性." true
    else
        action "基础检查:主机出口网络连通性." false
        if func_cur_internet_line;then       
            echo "基础检查:出口网络不通,业务线路离线,正尝试切换到管理线路..." 
            func_switch_management_line ${netdev_route} && action "基础检查:出口网络不通,业务线路异常,切管理线路成功." true \
            || action "基础检查:出口网络不通,业务线路异常,切管理线路失败,请联系基础运维处理." false
        fi
    fi
    

    # 检查DNS域名解析
    if check_ping xunlei.com;then        
        # 检查DNS域名是否被解劫持
        if func_dns_hijack;then
             echo "基础检查:DNS解析异常IP疑似劫持."
             if func_cur_internet_line;then
                 echo "基础检查:DNS解析异常IP疑似劫持,正尝试修复...".
                 manage_gw=$(ip route ls table pr_table_admin | grep "^default" | awk '{print$3}') 
                 ip route replace ${dns_nameserver} via ${manage_gw} && echo "ip route replace ${dns_nameserver} via ${manage_gw}" >> /etc/rc.local
                 func_dns_hijack && action "基础检查:DNS解析异常IP劫持,修复失败.请联系基础运维处理." false || action "result:基础环境检查,DNS解析异常IP劫持,修复成功." true
             fi
        else
             action "基础检查:DNS解析." true
        fi
    else
        action "基础检查:DNS解析." false
        echo "基础检查:DNS解析异常,正尝试修复..."
        cp -raf /etc/{resolv.conf,resolv.conf.bak.${CURTIME}}
        echo -e "nameserver 223.5.5.5\nnameserver 114.114.114.114" > /etc/resolv.conf
        check_ping xunlei.com && action "基础检查:DNS解析异常,修复成功." true || action "基础检查:DNS解析异常,修复失败.请联系基础运维处理." false
    fi

    # 检查主机丢包情况
    domains=( "www.aliyun.com" "www.baidu.com" "www.qq.com" "www.jd.com" ) 
    total_loss=0
    count=0 

    for domain in "${domains[@]}"; do
        result=$(ping -c 10 $domain | grep 'packet loss' | awk '{print $6}' | tr -d '%')
        if [ "$result" != "" ]; then
            total_loss=$(echo "$total_loss + $result" | bc)
            ((count++))
        fi
    done

    if [ "$count" -gt 0 ]; then
        average_loss=$(echo "scale=2; $total_loss / $count" | bc)
        awk -v n1="${average_loss}" 'BEGIN { if (int(n1) < 5 ) exit 0; else exit 1 }'
        [ $? -eq 0 ] && action "基础检查:当前丢包率:$average_loss%" true \
        || action "基础基础:当前丢包率:$average_loss%,请联系基础运维处理." false
    else
        action "基础检查:PING无有效数据." false
    fi 
}


# 1.2 检查磁盘使用情况
func_basic_env_disk_check(){
    
     # 检查磁盘使用情况
     check_usage(){
         local usage=$1
         local item=$2

        if [ $usage -ge 100 ]; then
            action "基础检查:/目录使用率:${item}:${usage}%,请联系业务运维处理." false
        else
            action "基础检查:/目录使用率:${item}:${usage}%" true
        fi
    }
    read disk_used_capacity disk_used_inode <<< $(df -h / | awk 'NR==2 {sub(/%/,""); print $5}')$(echo " ")$(df -i / | awk 'NR==2 {sub(/%/,""); print $5}')
    check_usage ${disk_used_capacity} "容量"
    check_usage ${disk_used_inode} "inode"
}

# 1.3 检查系统负载情况
func_basic_env_load_check(){
   
    # 检查负载 
    cpu_core=$(lscpu | grep '^CPU(s)' | awk '{print$2}')
    read one_minute_load <<<  $(awk '{print$1}' /proc/loadavg)
    awk -v n1="${cpu_core}" -v n2="${one_minute_load}" 'BEGIN { if (n1 > n2 ) exit 0; else exit 1 }'
    [ $? -eq 0 ] && action "基础检查:1分钟负载:${one_minute_load}" true \
    || action "基础检查:1分钟负载:${one_minute_load},请联系业务运维处理." false

    # 检查CPU使用情况
    cpu_usage=$(top -bn1 | grep "Cpu(s)")
    idle_cpu=$(echo $cpu_usage | awk '{print $8}')
    cpu_used=$(echo "scale=2; 100 - ${idle_cpu}" | bc)
    awk -v n1="${cpu_used}" 'BEGIN { if (n1 < 70 ) exit 0; else exit 1 }'
    [ $? -eq 0 ] && action "基础检查:CPU使用情况:${cpu_used}%" true \
    || action "基础检查:CPU使用情况:${cpu_used}%,请联系业务运维处理." false
       
}

# 2.1 nodeexporter上报网络检查 
func_ne_network_check(){

    # 上报域名网络是否可达
    check_ping ${ne_url} && action "ne插件检查:上报网络联通性检查." true \
    || action "ne插件检查:上报网络不通,请联系基础运维处理." false

    # 上报server端口是否可达
    nc -zv ${ne_url} 80 >/dev/null 2>&1 && action "ne插件检查:上报端口联通性检查." true \
    || action "ne插件检查:上报端口不通,请联系基础运维处理." false 
}


# 2.2 nodeexporter版本检查
func_ne_ver_check(){
   
    # 要求1.7.2以上版本，建议更新到最新版本
    require_ver='1.7.2'    
    func_compare_ver(){
        local_ver=$(xycall list | jq -r '.ipklist[] | select(.package=="plugin-nodeexporter") | .local_version')
        latest_ver=$(xycall list | jq -r '.ipklist[] | select(.package=="plugin-nodeexporter") | .latest_version')         
    }
   
    # 判断版本
    func_compare_ver

    if [ "$(printf '%s\n' "$local_ver" "$require_ver" | sort -V | tail -n 1)" == "${require_ver}" ]; then
        if [ "${local_ver}" == "${latest_ver}" ];then
            action "ne插件检查:版本检查,要求>=1.7.2,联系基础运维处理." false  
        else
            echo "ne插件检查:版本检查,要求>=1.7.2,正尝试更新..."
            xycall update plugin-nodeexporter >/dev/null 2>&1
            sleep 10
            func_compare_ver
            [ "${local_ver}" == "${latest_ver}" ] && action "ne插件检查:版本检查,更新成功.当前版本:${local_ver}" true \
            || action "ne插件检查:版本检查,更新失败,请联系基础运维处理." false
        fi 
    else
        action "ne插件检查:版本检查,当前版本:${local_ver}." true
    fi
}


# 2.3 nodeexporter服务检查
func_ne_server_check(){

    # 进程检查
    pro_count=$(ps auxf | grep [n]ode_exporter -c)
    if [ ${pro_count} -ne 0 ];then
        action "ne插件检查:服务进程检查." true
    else
       echo "ne插件检查:服务进程异常,正尝试重启服务..."
       /usr/local/xyvod/xyapp/system/miner.plugin-nodeexporter.ipk/bin/control restart >/dev/null 2>&1
       [ $(ps auxf | grep [n]ode_exporter -c) -ne 0 ] && action "ne插件检查:服务进程已重启." true \
       || action "ne插件检查:服务进程重启失败，请联系资源运维处理." false
    fi

    # 端口检查
    lis_port=$(netstat -anltp | awk -F '[: ]+' '/node_exporter/ && /LISTEN/ {print$5}')
    [ ${lis_port} -eq 6280 ] &&  action "ne插件检查:服务端口检查." true \
    || action "ne插件检查:服务端口6280被占用,请联系业务运维处理."  false
}

# 2.4  nodeexporter日志检查
func_ne_log_check(){
     
    # 检索错误日志:/var/log/node_exporter/node_exporter.log
    fun_ne_log_error(){
        local error_info=$1
        [ $(tail -n 100 /var/log/node_exporter/node_exporter.log | grep error | grep -c "${error_info}") -eq 0 ] && return 0 || return 1
    }
   
    # 常见错误1:上报数据源格式错误，基础硬件信息上报数据格式错误
    local disk_fix_ver='2024.3.20'
    if fun_ne_log_error 'hardware_disk.prom';then
        action "ne插件检查:日志检查,检查错误关键字hardware_disk.prom." true
    else
        lasttime=$(cat /var/log/node_exporter/node_exporter.log | grep hardware_disk.prom. | tail -n 1 | awk -F'[= ]+' '{print$2}' | xargs -I{} date -u -d {} +"%s") 
        if [ ! -e /usr/local/inst_base_monitor.pid ];then         
            action "ne插件检查:存在错误日志hardware_disk.prom." false
            if [ $(head -n 2 /usr/local/op-base/hardware_disk_report.sh | grep ${disk_fix_ver} -c) -eq 1 ];then
                action "ne插件检查:存在错误日志hardware_disk.prom,当前修复脚本已是最新,请联系李振鹏处理." false
            else
                echo "ne插件检查:存在错误日志hardware_disk.prom,正尝试修复..."
                wget http://thunder:xunlei@apt.sandai.net/basic_setup/dkj/inst_base_monitor.sh -O /tmp/inst_base_monitor.sh  >/dev/null 2>&1 && sh /tmp/inst_base_monitor.sh >/dev/null 2>&1 
                if [ $(head -n 2 /usr/local/op-base/hardware_disk_report.sh | grep ${disk_fix_ver} -c) -eq 1 ];then
                    action "ne插件检查:存在错误日志hardware_disk.prom,修复成功." true
                    date +%s > /usr/local/inst_base_monitor.pid
                    #/usr/local/xyvod/xyapp/system/miner.plugin-nodeexporter.ipk/bin/control restart >/dev/null 2>&1
                else
                    action "ne插件检查:存在错误日志hardware_disk.prom,修复失败,请联系李振鹏处理." false
                fi
            fi
        else
            fix_datetime=$(cat /usr/local/inst_base_monitor.pid)
            latest_time=$(cat /var/log/node_exporter/node_exporter.log | grep hardware_disk.prom. | tail -n 1 | awk -F'[= ]+' '{print$2}' | xargs -I{} date -u -d {} +"%s")
            if [ ${fix_datetime} -gt ${latest_time}];then
                action "ne插件检查:存在错误日志hardware_disk.prom,请联系李振鹏处理." false
            else
                action "ne插件检查:日志检查,检查错误关键字hardware_disk.prom." true
            fi
        fi
    fi

    # 常见错误2:metrics上报数据超时，网卡数量过多
    fun_ne_log_error "metric family: write tcp 127.0.0.1:6280"
    if [ $? -eq 0 ];then
        action "ne插件检查:node_exporter.log日志检查,检索错误关键字metric." true 
    else
        key_count=$(ps axuf | grep [n]odeexporter | grep -c ppp)
        if [ ${key_count} -ne 0 ];then       
            action "ne插件检查:node_exporter.log日志检查,检索到错误关键字write tcp 127.0.0.1:6280.已是最新版本,问题仍未解决,请联系资源运维处理." false
        else
            echo "ne插件检查:node_exporter.log日志检查,检索到错误关键字write tcp 127.0.0.1:6280,过滤掉ppp类型网卡数据源,正在修复..."
            sed -i '/^NODE_EXPORTER_NETWORK_DEVICE_EXCLUDE_REGEX="^(cali|veth|cni|tunl|flannel|lan|vm0n).*|vxlan.calico$"/c\NODE_EXPORTER_NETWORK_DEVICE_EXCLUDE_REGEX="^(cali|veth|cni|tunl|flannel|lan|vm0n|ppp).*|vxlan.calico$"'  /usr/local/xyvod/xyapp/system/miner.plugin-nodeexporter.ipk/bin/exporter
            /usr/local/xyvod/xyapp/system/miner.plugin-nodeexporter.ipk/bin/control restart >/dev/null 2>&1
            key_verify_count=$(ps axuf | grep [n]odeexporter | grep -c ppp)
            if [ ${key_count} -ne 0 ];then
                action "ne插件检查:node_exporter.log日志检查,检索到错误关键字write tcp 127.0.0.1:6280,过滤掉ppp类型网卡数据源,修复成功."
            else
                action "ne插件检查:node_exporter.log日志检查,检索到错误关键字write tcp 127.0.0.1:6280,过滤掉ppp类型网卡数据源,修复失败,请联系资源运维处理."
            fi
        fi
    fi

    # 常见错误3:无网卡设备 could not get net class info
    fun_ne_log_error "could not get net class info" &&  action "ne插件检查:日志检查,检查错误关键字not get class info." true \
    || action "ne插件检查:存在错误日志not get class info,存在线路离线,请联系资源运维处理." false

    
    # 检索上报日志:/var/log/node_exporter/reporterd.log
    fun_ne_log_report(){
        local report_info=$1
        [ $(tail -n 50 /var/log/node_exporter/reporterd.log | grep -c "${report_info}") -eq 0 ] && return 0 || return 1
    }
     
    # 常见错误1:上报超时
    fun_ne_log_report "status code: 100" &&  action "ne插件检查:日志检查,检查上报日志关键字status code: 100." true \
    ||  action "ne插件检查:reporterd.log日志检查,存在错误日志:status code: 100.请联系资源运维处理." false

    # 常见错误2:格式问题
    fun_ne_log_report "text format parsing error in line 1" &&  action "ne插件检查:日志检查,检查上报日志关键字parsing error." true \
    || action "ne插件检查:reporterd.log日志检查,存在错误日志:text format parsing error in line 1,请联系资源运维处理." false
    
}


# 1.Basic environment check
func_basic_env_check(){
    
    # 1.1 检查网络
    func_basic_env_network_check

    # 1.2 检查硬盘
    func_basic_env_disk_check
 
    # 1.3 检查负载
    func_basic_env_load_check
}

# 2.nodeexporter插件检查
func_ne_check(){
   
    # 2.1 nodeexporter上报网络检查
    func_ne_network_check   
    
    # 2.2 nodeexporter版本检查
    func_ne_ver_check

    # 2.3 nodeexporter服务检查
    func_ne_server_check

    # 2.4 nodeexporter日志检查
    func_ne_log_check

    # 2.5 metrics数据提交检查

}


# Main function
main(){
    
    # 1.基础环境检查
    func_basic_env_check

    # 2.NE插件检查
    func_ne_check

}

# Program entrance
main

