#!/bin/bash
# Don't warn about unreachable commands in this file
# shellcheck disable=SC2317
set -e

# measure time of status check
start_seconds=$(cut -d . -f 1 /proc/uptime) || true

#####################################################################
# Defaults (can be redefined via /etc/ngcp-status/ngcp-status.conf)

# Program defaults:

RC=0

FORMAT=${NGCP_STATUS_FORMAT:-text}
COLOR=${COLOR:-true}

LOG_FILE="/var/log/ngcp/ngcp-status.log"

# System defaults:

NGCP_TYPE=spce

MAINTENANCE_MODE=false
PRODUCTION_MODE=true

CARRIER_EDITION=false
PRO_EDITION=false
CE_EDITION=false
NGCP_LICENSE=false

LOAD_AVERAGE_WARN=5
LOAD_AVERAGE_CRIT=10
RAM_USED_CRIT=85       # %
RAM_USED_WARN=75       # %
DISK_USED_CRIT=85      # %
DISK_USED_WARN=75      # %
DISK_INODES_CRIT=85    # %
DISK_INODES_WARN=75    # %
SWAP_USED_WARN=50      # %
SWAP_USED_CRIT=75      # %

SSD_TRIM="no"

COREDUMP_EXPIRATION=5  # days

# Service defaults:

INIT_SYSTEM=systemd

NTP_BACKEND=timesyncd

APPROX_RW_PORT=9999
APPROX_RO_PORT=9998

NGINX_SSL_CIPHERS=''
NGINX_SSL_CIPHERS_AUTOPROV=''
NGINX_SSL_CIPHERS_INTERCEPT=''

MYSQL_PAIR_HOST="localhost"
MYSQL_PAIR_PORT=3306
#MYSQL_CENTRAL_HOST="db01"
#MYSQL_CENTRAL_PORT=3306
MYSQL_LOCAL_HOST="127.0.0.1"
MYSQL_LOCAL_PORT=3308
MYSQL_ACC_MAX=50000    # rows
MYSQL_CDR_MAX=5000000  # rows
MYSQL_CONN_CRIT=85     # %
MYSQL_CONN_WARN=50     # %
MYSQL_LAG_MAX=600      # seconds
MYSQL_QUERIES_CRIT=1000
MYSQL_QUERIES_WARN=500
MYSQL_SIZE_MAX=81920   # MB (80G)
MYSQL_SLOW_CRIT=600    # seconds
MYSQL_SLOW_TIMEOUT=60  # seconds
MYSQL_SLOW_WARN=3600   # seconds

# Comms defaults:

BILLING_ENABLE='no'
CALL_RECORDING_ENABLE='no'
EXTERNAL_LNP_ENABLE='no'
KAMAILIO_PROXY_ALLOW_REFER_METHOD="no"
KAMAILIO_PROXY_ENABLE="no"
KAMAILIO_PROXY_PRESENCE_ENABLE="no"
NGCP_FAXSERVER_ENABLE='no'
PBX_ENABLE="no"
PREPAID_INEWRATE_ENABLE='no'
PREPAID_SWRATE_ENABLE='no'
PUSHD_ENABLE='no'
SMS_ENABLE='no'
VOISNIFF_LI_ENABLE='no'
VOISNIFF_MYSQL_DUMP_ENABLE='no'
XMPP_ENABLE='no'

# Copy of all STDOUT and STDERR output to logs
if [ ! -f "${LOG_FILE}" ]; then
  touch "${LOG_FILE}"
  chmod 0640 "${LOG_FILE}"
  chgrp adm "${LOG_FILE}"
fi
exec  > >(tee -a "${LOG_FILE}"    )
exec 2> >(tee -a "${LOG_FILE}" >&2)

STATE_DIR=$(mktemp --tmpdir -d ngcp-status-state-XXXXXXXXXX)

cleanup() {
  rm -rf "${STATE_DIR}"
}

trap cleanup EXIT ERR HUP INT QUIT TERM

# node information
declare -a ngcp_roles=()

declare -a worker_pid=()

# results arrays
# shellcheck disable=SC2034
{
declare -a self_warnings=()
declare -a apt_errors=()
declare -a apt_warnings=()
declare -a bios_errors=()
declare -a bios_info=()
declare -a cdr_errors=()
declare -a chassis_info=()
declare -a collective_check_errors=()
declare -a core_errors=()
declare -a core_warnings=()
declare -a disks_info=()
declare -a fsck_warnings=()
declare -a glusterfs_errors=()
declare -a ha_errors=()
declare -a ha_warnings=()
declare -a hw_info=()
declare -a io_errors=()
declare -a io_inodes_errors=()
declare -a io_inodes_info=()
declare -a io_inodes_warnings=()
declare -a io_space_errors=()
declare -a io_space_info=()
declare -a io_space_warnings=()
declare -a io_warnings=()
declare -a la_errors=()
declare -a la_info=()
declare -a la_warnings=()
declare -a lan_errors=()
declare -a lan_warnings=()
declare -a service_errors=()
declare -a service_warnings=()
declare -a monitoring_errors=()
declare -a mysql_errors=()
declare -a mysql_info=()
declare -a mysql_warnings=()
declare -a ngcp_errors=()
declare -a ngcp_info=()
declare -a ngcp_warnings=()
declare -a ngcpcfg_errors=()
declare -a ngcpcfg_info=()
declare -a tls_info=()
declare -a ntp_errors=()
declare -a ntp_warnings=()
declare -a fstrim_warnings=()
declare -a ppa_warnings=()
declare -a ram_errors=()
declare -a ram_info=()
declare -a ram_warnings=()
declare -a swap_errors=()
declare -a swap_warnings=()
declare -a systemd_errors=()
declare -a systemd_warnings=()
declare -a tmpfs_errors=()
declare -a tz_errors=()
declare -a wan_warnings=()
declare -a license_errors=()
declare -a null=()
}

#####################################################################
# Functions

warn() {
    self_warnings+=("$*")
}

error() {
    echo "ERROR: $*" >&2
    exit 1
}

state() {
  local area="$1"
  local type="$2"
  local mesg="$3"

  mkdir -p "${STATE_DIR}/${area}/${type}"
  echo "${mesg}" >>"${STATE_DIR}/${area}/${type}/$$"
}

load_mesg() {
  local area="$1"
  local type="$2"

  if [ -d "${STATE_DIR}/${area}/${type}" ]; then
    mapfile -t "${area}_${type}" <"${STATE_DIR}/${area}/${type}"/*
  fi
}

run_func() {
  local start_func_time_ms
  local func_duration_ms

  start_func_time_ms="$(awk '{print $1}' /proc/uptime)"
  "$@" || RC=$?
  func_duration_ms="$(echo "$(awk '{print $1}' /proc/uptime) ${start_func_time_ms}" | awk '{ printf "%-.3f", $1 - $2 }' )"

  printf "Executed: %-50s in %6.3f seconds, exit code %s\n" "$@" "${func_duration_ms}" "${RC}" >> "${LOG_FILE}"
}

spawn() {
  "$@" &
  worker_pid+=($!)
}

spawn_func() {
  spawn run_func "$@"
}

is_package_installed() {
  local package="$1"

  if [ -z "${package}" ] ; then
    error "package is not specified, cannot continue"
  fi

  if [ "$(dpkg-query -f "\${db:Status-Status} \${db:Status-Eflag}" -W "${package}" 2>/dev/null)" = "installed ok" ]; then
    return 0
  else
    return 1
  fi
}

# float number safe comparison of two values
is_equal_or_greater() {
  if awk -- 'BEGIN { exit !(ARGV[1] >= ARGV[2]) }' "${1}" "${2}" ; then
    return 0
  else
    return 1
  fi
}

check_own_dependencies() {
  if ! [ -r /etc/default/ngcp-roles ] ; then
    warn "cannot read /etc/default/ngcp-roles, continue anyway"
  else
    # shellcheck disable=SC1091
    . /etc/default/ngcp-roles
  fi

  if [[ ! -r /etc/mysql/sipwise_extra.cnf ]] ; then
    warn "cannot read /etc/mysql/sipwise_extra.cnf, continue anyway"
  fi
}

check_root_privileges() {
  #Check if I'm root
  if [[ ${EUID} -ne 0 ]]; then
    i_am_root=false
    ping_flood=""
  else
    i_am_root=true
    ping_flood="-f"
  fi
}

load_config() {
  if ! [ -r /etc/ngcp-status/ngcp-status.conf ] ; then
    warn "cannot read /etc/ngcp-status/ngcp-status.conf, using default values"
    return
  fi
  # shellcheck disable=SC1091
  . /etc/ngcp-status/ngcp-status.conf
}

set_colors() {
  if [[ "${COLOR}" = "true" ]] ; then
    x='\e[0m'          # Reset color
    r='\e[1;31m'       # Red
    g='\e[1;32m'       # Green
    y='\e[1;33m'       # Yellow
    b='\e[1;34m'       # Blue
#   p='\e[1;35m'       # Purple
    c='\e[1;36m'       # Cyan
    w='\e[1;37m'       # White
    ha="${r}"          # HA status specific
  fi
}

get_production_label() {
  if "${PRODUCTION_MODE}" ; then
    production_label="PRODUCTION"
  else
    production_label="LAB"
  fi
}

get_mysql_replication_errors() {
  local _tmp="$1"
  local _errno=""
  local _error=""

  _errno="$(awk '/^\s+Last_Errno:/ { print $NF }' "${_tmp}")"
  _error="$(awk -F"_Error: " '/^\s+Last_Error:/ { print $2 }' "${_tmp}")"

  if [ "${_errno}" != "0" ] ; then
    state mysql errors "[#${_errno}] ${_error}"
  fi

  _errno="$(awk '/^\s+Last_SQL_Errno:/ { print $NF }' "${_tmp}")"
  _error="$(awk -F"_Error: " '/^\s+Last_SQL_Error:/ { print $2 }' "${_tmp}")"

  if [ "${_errno}" != "0" ] ; then
    state mysql errors "[SQL #${_errno}] ${_error}"
  fi

  _errno="$(awk '/^\s+Last_IO_Errno:/ { print $NF }' "${_tmp}")"
  _error="$(awk -F"_Error: " '/^\s+Last_IO_Error:/ { print $2 }' "${_tmp}")"

  if [ "${_errno}" != "0" ] ; then
    state mysql errors "[IO #${_errno}] ${_error}"
  fi
}

check_mysql_replication() {
  local _host="$1"
  local _port="$2"
  local _connection="$3"
  declare -a _opts=()
  _opts+=("--defaults-extra-file=/etc/mysql/sipwise_extra.cnf")
  _opts+=("-h${_host}")
  _opts+=("-P${_port}")
  _opts+=("-Bs")

  if "${CE_EDITION}" ; then
    return
  fi

  if ! timeout 2 mysql "${_opts[@]}" -e "SELECT 1 FROM DUAL" >/dev/null 2>&1 ; then
    state mysql errors "Is MariaDB up on ${_host}:${_port}? Failed to check replication!"
    return
  fi

  local _tmp
  _tmp=$(mktemp -t ngcp-mysql-repl-XXXXXXXXXX)

  timeout 2 mysql "${_opts[@]}" -e "SHOW SLAVE '${_connection}' STATUS\G" > "${_tmp}" 2>/dev/null || true

  local _Using_Gtid
  _Using_Gtid=$(awk '/^\s+Using_Gtid:/ { print $NF }' "${_tmp}")

  if [ "${_Using_Gtid:-No}" = 'No' ] ; then
    state mysql errors "Replication without GTDI on ${_host}:${_port}!"
  fi

  local _Master_Host
  _Master_Host=$(awk '/^\s+Master_Host/ { print $NF }' "${_tmp}")
  local _Slave_IO_Running
  _Slave_IO_Running=$(awk '/^\s+Slave_IO_Running:/ { print $NF }' "${_tmp}")
  local _Slave_SQL_Running
  _Slave_SQL_Running=$(awk '/^\s+Slave_SQL_Running:/ { print $NF }' "${_tmp}")
  local _Seconds_Behind_Master
  _Seconds_Behind_Master=$(awk '/^\s+Seconds_Behind_Master:/ { print $NF }' "${_tmp}")

  if [ -z "${_Slave_IO_Running}" ] || [ -z "${_Slave_SQL_Running}" ] ; then
    state mysql errors "Is replication configured on ${_host}:${_port} from ${_Master_Host}?"

  elif [ "${_Slave_SQL_Running}" != 'Yes' ] ; then
    state mysql errors "Slave_SQL_Running is stopped on ${_host}:${_port} from ${_Master_Host}!"
    get_mysql_replication_errors "${_tmp}"

  elif [ "${_Slave_IO_Running}" != 'Yes' ] ; then
    state mysql errors "Slave_IO_Running is stopped on ${_host}:${_port} from ${_Master_Host}!"
    get_mysql_replication_errors "${_tmp}"

  elif [ "${_Seconds_Behind_Master}" -ge "${MYSQL_LAG_MAX}" ] ; then
    state mysql warnings "MariaDB lag is too big: ${_Seconds_Behind_Master} seconds on ${_host}:${_port} from ${_Master_Host}"

  fi

  rm -f "${_tmp}"
}

check_mysql_slowlog() {
  local _host="$1"
  local _port="$2"
  declare -a _opts=()
  _opts+=("--defaults-extra-file=/etc/mysql/sipwise_extra.cnf")
  _opts+=("-h${_host}")
  _opts+=("-P${_port}")
  _opts+=("-Bs")
  _opts+=("-N")

  if ! timeout 2 mysql "${_opts[@]}" -e "SELECT 1 FROM DUAL" >/dev/null 2>&1 ; then
    state mysql errors "Is MariaDB up on ${_host}:${_port}? Failed to check slow queries log!"
    return
  fi

  local _query="SELECT variable_value FROM information_schema.global_variables WHERE variable_name='slow_query_log'
  UNION
  SELECT variable_value FROM information_schema.global_variables WHERE variable_name='slow_query_log_file';"
  declare -a _result
  mapfile -t _result < <(timeout 2 mysql "${_opts[@]}" -e "${_query}" || true)

  local _slow_log_status="${_result[0]}"
  if [ "${_slow_log_status}" = "OFF" ] ; then
    state mysql info "MariaDB slow log disabled on ${_host}:${_port}, skipping DB check"
    return
  fi

  local _slow_log="${_result[1]}"
  if ! [ -f "${_slow_log}" ] ; then
    state mysql warnings "Missing ${_slow_log} on ${_host}:${_port}"
    return
  fi

  local _last_rec
  _last_rec=$(tail -50 "${_slow_log}" | awk '/^# Time: /{print $3" "$4}' | tail -1) || true

  if [ "${_last_rec}" = "" ] ; then
    return
  fi

  local _date_rec
  _date_rec=$(date -d "${_last_rec}" +"%Y%m%d%H%M%S")
  local _now
  _now=$(date +"%Y%m%d%H%M%S")

  if [ $(( _date_rec + MYSQL_SLOW_CRIT )) -ge "${_now}" ] ; then
    state mysql errors "fresh slow queries on ${_host}:${_port}, see ${_slow_log}"
  elif [ $(( _date_rec + MYSQL_SLOW_WARN )) -ge "${_now}" ] ; then
    state mysql warnings "found slow queries on ${_host}:${_port}, see ${_slow_log}"
  fi
}

check_mysql_processlist() {
  local _host="$1"
  local _port="$2"
  declare -a _opts_dump=()
  _opts_dump+=("--defaults-extra-file=/etc/mysql/sipwise_extra.cnf")
  _opts_dump+=("-h${_host}")
  _opts_dump+=("-P${_port}")
  declare -a _opts=()
  _opts+=("${_opts_dump[@]}")
  _opts+=("-Bs")
  _opts+=("-N")

  local _tmp="/var/tmp/ngcp-slow-query-${_host}-${_port}"

  if ! timeout 2 mysql "${_opts[@]}" -e "SELECT 1 FROM DUAL" >/dev/null 2>&1 ; then
    state mysql errors "Is MariaDB up on ${_host}:${_port}? Failed to check processlist!"
    return
  fi

  local _query="SELECT COUNT(*) FROM information_schema.processlist"
  local _count
  _count=$(timeout 2 mysql "${_opts[@]}" -e "${_query}" || true)

  if [ "${_count}" -ge "${MYSQL_QUERIES_CRIT}" ] ; then
    state mysql errors "huge processlist amount '${_count}' on ${_host}:${_port}"
  elif [ "${_count}" -ge "${MYSQL_QUERIES_WARN}" ] ; then
    state mysql warnings "high processlist amount '${_count}' on ${_host}:${_port}"
  fi

  _query="SELECT * FROM information_schema.processlist WHERE user NOT IN ('replicator','system user') AND time > ${MYSQL_SLOW_TIMEOUT} ORDER BY TIME DESC"
  timeout 2 mysql "${_opts_dump[@]}" -e "${_query}" > "${_tmp}" 2>/dev/null || true

  if [ "$(wc -l < "${_tmp}")" -gt 0 ] ; then
    state mysql errors "slow query in progress on ${_host}:${_port}, see ${_tmp}"
  fi
}

check_mysql_connections() {
  local _host="$1"
  local _port="$2"
  declare -a _opts=()
  _opts+=("--defaults-extra-file=/etc/mysql/sipwise_extra.cnf")
  _opts+=("-h${_host}")
  _opts+=("-P${_port}")
  _opts+=("-Bs")
  _opts+=("-N")

  if ! timeout 2 mysql "${_opts[@]}" -e "SELECT 1 FROM DUAL" >/dev/null 2>&1 ; then
    state mysql errors "Is MariaDB up on ${_host}:${_port}? Failed to check connections!"
    return
  fi

  local _query="SELECT variable_value FROM information_schema.global_variables WHERE variable_name = 'max_connections'
  UNION
  SELECT variable_value FROM information_schema.session_status WHERE variable_name = 'Threads_connected'
  UNION
  SELECT variable_value FROM information_schema.global_status WHERE variable_name = 'max_used_connections';"

  declare _result=()
  mapfile -t _result < <(timeout 2 mysql "${_opts[@]}" -e "${_query}" || true)

  local _max="${_result[0]}"
  local _used="${_result[1]}"
  local _used_ever="${_result[2]}"

  local _rel
  _rel=$(( _used * 100 / ${_max:-1} ))

  if [ "${_rel}" -ge "${MYSQL_CONN_CRIT}" ] ; then
    state mysql errors "huge connection usage '${_used}[${_rel}%]' on ${_host}:${_port}"
  elif [ "${_rel}" -ge "${MYSQL_CONN_WARN}" ] ; then
    state mysql warnings "high connection usage '${_used}[${_rel}%]' on ${_host}:${_port}"
  fi

  if [ "${_max}" = "${_used_ever}" ] ; then
    state mysql warnings "hit 'max_connections' recently, run 'flush status' to clean"
  fi
}

check_mysql_deadlock() {
  local _host="$1"
  local _port="$2"
  declare -a _opts=()
  _opts+=("--defaults-extra-file=/etc/mysql/sipwise_extra.cnf")
  _opts+=("-h${_host}")
  _opts+=("-P${_port}")
  _opts+=("-Bs")
  _opts+=("-N")

  if ! timeout 2 mysql "${_opts[@]}" -e "SELECT 1 FROM DUAL" >/dev/null 2>&1 ; then
    state mysql errors "Is MariaDB up on ${_host}:${_port}? Failed to check deadlocks!"
    return
  fi

  if timeout 2 mysql "${_opts[@]}" -e "SHOW ENGINE INNODB STATUS\G" | \
     grep -i -A10 "LATEST DETECTED DEADLOCK" >>"${LOG_FILE}" 2>&1 ; then
    state mysql errors "detected InnoDB engine DEADLOCKs on ${_host}:${_port}"
  fi
}

check_mysql_equal_schemes() {
  if ! timeout 5 ngcp-mysql-compare-dbs &>/dev/null ; then
    state mysql warnings "detected divergence in mysql schemes, run ngcp-mysql-compare-dbs to check details"
  fi
}

check_service_status() {
  local _tmp
  _tmp="$(mktemp -t ngcp-service-summary-XXXXXXXXXX)"

  ngcp-service summary > "${_tmp}" 2>&1 || true

  if grep -q "^~" "${_tmp}" ; then
    declare -a _service_unmonitored_services
    mapfile -t _service_unmonitored_services < <(awk '/^~/ { print $2 }' "${_tmp}")
    state service warnings "found ${#_service_unmonitored_services[@]} unmonitored service(s): ${_service_unmonitored_services[*]}"
  fi

  if grep -q "^\?" "${_tmp}" ; then
    declare -a _service_unknown_services
    mapfile -t _service_unknown_services < <(awk '/^\?/ { print $2 }' "${_tmp}")
    state service warnings "found ${#_service_unknown_services[@]} unknown state service(s): ${_service_unknown_services[*]}"
  fi

  if grep -q "^!" "${_tmp}" ; then
    declare -a _service_error_services
    mapfile -t _service_error_services < <(awk '/^!/ { print $2 }' "${_tmp}")
    state service errors "found ${#_service_error_services[@]} failed service(s): ${_service_error_services[*]}"
  fi

  rm -f "${_tmp}"
}

check_ngcpcfg_api_service() {
  if systemctl --quiet is-active ngcpcfg-api ; then
    state service warnings "found ngcpcfg-api service running, supposed to be running only during deployments"
  fi
}

check_systemd_failed_units() {
  if ! [ -d /run/systemd/system ] ; then
    return 0
  fi

  local _tmp
  _tmp="$(mktemp -t ngcp-systemd-status-XXXXXXXXXX)"

  if ! systemctl --failed &> "${_tmp}" ; then
    state systemd errors "error running \`systemctl --failed\`"
    rm -f "${_tmp}"
    return
  fi

  local _systemd_service_failures
  mapfile -t _systemd_service_failures < <(awk '/ failed / {print $2}' "${_tmp}" | xargs echo -n)

  if [ "${#_systemd_service_failures[@]}" != "0" ] ; then
    state systemd errors "found ${#_systemd_service_failures[@]} failing systemd services: ${_systemd_service_failures[*]}"
  fi

  rm -f "${_tmp}"
}

check_systemd_missing_reload() {
  if ! [ -d /run/systemd/system ] ; then
    return 0
  fi

  local _tmp
  _tmp="$(mktemp -t ngcp-systemd-reload-XXXXXXXXXX)"

  # use whitelist instead of just `systemctl cat '*'` to avoid:
  # 1) `No files found for init.scope.`
  # 2) `No files found for sys-devices-pci0000:00-0000:00:03.0-net-eth0.device.`
  # 3) `No files found for -.slice.`
  # 4) `No files found for run-user-10000.mount.`
  # 5) `No files found for vagrant.mount.`
  if ! systemctl cat '*.automount' '*.path' '*.service' '*.socket' '*.swap' '*.target' '*.timer' --state=active >/dev/null 2> "${_tmp}" ; then
    state systemd errors "problem during \`systemctl cat …\` run identified: $(cat "${_tmp}")"
  else
    local _systemctl_daemon_reload_count
    _systemctl_daemon_reload_count=$(grep -c 'changed on disk, the version systemd has loaded is outdated' "${_tmp}" || true)
    if [[ "${_systemctl_daemon_reload_count}" -gt 0 ]]; then
      state systemd warnings "found ${_systemctl_daemon_reload_count} systemd unit files changed on disk, run \`systemctl daemon-reload\` to reload configuration"
    fi
  fi

  rm -f "${_tmp}"
}

check_apt_list() {
  local _debian_file="/etc/apt/sources.list.d/debian.list"
  local _sipwise_file="/etc/apt/sources.list.d/sipwise.list"

  if ! [ -r "${_sipwise_file}" ] ; then
    state apt errors "${_sipwise_file} not found, skipped check_apt_list"
    return
  fi

  if ! [ -r "${_debian_file}" ] ; then
    state apt errors "${_debian_file} not found, skipped check_apt_list"
    return
  fi

  if "${PRO_EDITION}" || "${CARRIER_EDITION}" ; then
    if grep -q "spce" "${_sipwise_file}" ; then
      state apt errors "CE in PRO file ${_sipwise_file}"
    fi
  else
    if grep -q "sppro" "${_sipwise_file}" ; then
      state apt errors "PRO in CE file ${_sipwise_file}"
    fi
  fi

  if ! grep -q "${os_codename}-debug" "${_debian_file}" ; then
    state apt errors "Missing apt repo '${os_codename}-debug' in ${_debian_file}"
  fi

  if "${CE_EDITION}" ; then
    if grep -Eq "^deb http://debian.sipwise.com" "${_debian_file}" ; then
      state apt warnings "Switch to https in ${_debian_file}"
    fi
    if grep -Eq "^deb http://deb.sipwise.com" "${_sipwise_file}" ; then
      state apt warnings "Switch to https in ${_sipwise_file}"
    fi
  fi

  local tempfile
  tempfile=$(mktemp -t ngcp-status-apt-warns-XXXXXXXXXX)

  grep -E "^deb " "${_debian_file}" > "${tempfile}" 2>/dev/null || true
  if [ "$(wc -l < "${tempfile}")" != "5" ]; then
    state apt warnings "NGCP should have 5 apt repos in ${_debian_file}"
  fi

  grep -E "^deb " "${_sipwise_file}" > "${tempfile}" 2>/dev/null || true
  if [ "$(wc -l < "${tempfile}")" != "1" ]; then
    state apt warnings "NGCP should have 1 apt repo in ${_sipwise_file}"
  fi

  rm -f "${tempfile}"
}

check_approx_config() {
  if ! "${CARRIER_EDITION}" ; then
    return
  fi

  if grep -qs "${APPROX_RW_PORT}" /etc/apt/sources.list /etc/apt/sources.list.d/*.list ; then
    state apt errors "Approx RW port '${APPROX_RW_PORT}' in apt source list! Use RO port '${APPROX_RO_PORT}'"
  fi
}

check_approx_cache() {
  if "${CE_EDITION}" ; then
    return
  fi

  if ! ngcp-approx-cache --clean-check &>/dev/null ; then
    state apt warnings "clean approx cache: ngcp-approx-cache --clean"
  fi
}

check_customtt_files() {
  local _path="/etc/ngcp-config"

  if ! [ -d "${_path}" ] ; then
    warn "${_path} not found, skipped check_customtt_files"
    return
  fi

  local _tmp
  _tmp=$(mktemp -t ngcp-customtt-XXXXXXXXXX)

  local a="\.sp[1-9]?"
  local b="\.(web|db|prx|lb|slb)[0-9]+[a-i]?"
  local awk_regexp=".*customtt\.tt2(${a}|${b})?$"
  local sed_regexp="s/\.customtt//;s/(${a}|${b})//"
  find "${_path}" -regextype awk -iregex  "${awk_regexp}" > "${_tmp}"

  local _customtt_count
  _customtt_count=$(wc -l < "${_tmp}")

  local _plus=0
  local _minus=0
  local _tmp_diff _orig_tt2
  _tmp_diff=$(mktemp -t ngcp-customtt-diff-XXXXXXXXXX)

  if [ "${_customtt_count}" != "0" ] ; then
    while read -r _file ; do
      _orig_tt2=$(echo "${_file}" | sed -r "${sed_regexp}")
      diff -u "${_orig_tt2}" "${_file}" > "${_tmp_diff}" 2>&1 || true
      _plus=$((  _plus + $(grep -E -c '^\+' "${_tmp_diff}" || true) ))
      _minus=$((_minus + $(grep -E -c '^-'  "${_tmp_diff}" || true) ))
    done < "${_tmp}"
    state ngcp warnings "found ${_customtt_count} customtt files (+${_plus}/-${_minus} LoC)"
  fi

  rm -f "${_tmp}" "${_tmp_diff}"
}

check_ngcp_files() {
  local _files=(/etc/ngcp_version /etc/sipwise_ngcp_version /etc/default/ngcp-roles)
  _files+=(/etc/ngcp_hostname)
  _files+=(/etc/ngcp_nodename)

  if "${PRO_EDITION}" || "${CARRIER_EDITION}" ; then
    _files+=(/etc/ngcp_mgmt_node)
  fi

  if "${CARRIER_EDITION}" ; then
    _files+=(/etc/ngcp_ha_role)
  fi

  for _file in "${_files[@]}" ; do
    if ! [ -f "${_file}" ] ; then
      state ngcp errors "missing NGCP file ${_file}"
    fi
  done

  if [ -f /run/reboot-required ] ; then
    state ngcp errors "pending reboot detected, please reboot the node!"
  fi
}

get_ngcp_version() {
  if [ -r /etc/ngcp_version ]; then
    ngcp_version="$(cat /etc/ngcp_version)"
  else
    warn "/etc/ngcp_version not found, continue anyway"
  fi
  ngcp_version=${ngcp_version:-unknown}
}

get_ngcp_type() {
  case "${NGCP_TYPE}" in
    carrier)
      ngcp_type="CARRIER"
      CARRIER_EDITION=true
      ;;
    sppro)
      ngcp_type="PRO"
      PRO_EDITION=true
      ;;
    spce)
      ngcp_type="CE"
      CE_EDITION=true
      ;;
  esac
}

get_ngcp_roles() {
  if [ "${NGCP_IS_MGMT}" = "yes" ] ; then
    ngcp_roles+=("MGMT")
  fi
  if [ "${NGCP_IS_DB}" = "yes" ] ; then
    ngcp_roles+=("DB")
  fi
  if [ "${NGCP_IS_LB}" = "yes" ] ; then
    ngcp_roles+=("LB")
  fi
  if [ "${NGCP_IS_PROXY}" = "yes" ] ; then
    ngcp_roles+=("PRX")
  fi
  if [ "${NGCP_IS_RTP}" = "yes" ] ; then
    ngcp_roles+=("RTP")
  fi
  if [ "${NGCP_IS_LI}" = "yes" ] ; then
    ngcp_roles+=("LI")
  fi
  if [ "${NGCP_IS_LI_DIST}" = "yes" ] ; then
    ngcp_roles+=("LI_DIST")
  fi
  if [ "${NGCP_IS_STORAGE}" = "yes" ] ; then
    ngcp_roles+=("STOR")
  fi
}

get_ngcp_nodes() {
  if "${CE_EDITION}" ; then
    # nothing to do here on CE
    state ngcp nodes_count 1
    return
  fi

  if [ -z "${NGCP_HOSTS}" ] ; then
    state lan warnings "\$NGCP_HOSTS is empty, check /etc/default/ngcp-roles"
    return
  fi

  local see_more_details=false

  # shellcheck disable=SC2206
  ngcp_nodes_count="$(arr=(${NGCP_HOSTS}) && echo ${#arr[@]})"
  state ngcp nodes_count "${ngcp_nodes_count}"

  if ! ngcp-check-cluster-ssh neighbours >> "${LOG_FILE}" 2>&1 ; then
    state lan errors "SSH issue, run: ngcp-check-cluster-ssh neighbours"
    see_more_details=true
  fi

  if ! ngcp-check-cluster-ping neighbours >> "${LOG_FILE}" 2>&1 ; then
    state lan errors "PING issue, run: ngcp-check-cluster-ping neighbours'"
    see_more_details=true
  fi

  if "${see_more_details}" ; then
    state lan errors "see more details in log '${LOG_FILE}'"
  fi
}

get_ha_status() {
  if ! [ -x "$(which ngcp-check-active)" ] ; then
    warn "ngcp-check-active not found, continue anyway"
    ngcp_ha_status="UNKNOWN"
    ha=${x}
    return
  fi

  case $(ngcp-check-active -v) in
    active)
      ngcp_ha_status="ACTIVE"
      ha=${g}
      ;;
    standby)
      ngcp_ha_status="inactive"
      ha=${c}
      ;;
    unknown)
      ngcp_ha_status="HA ERROR!"
      ;;
    *)
      ngcp_ha_status=""
      ;;
  esac
}

check_ha_status() {
  if [ "${ngcp_ha_status}" != "ACTIVE" ] && [ "${ngcp_ha_status}" != "inactive" ] ; then
    state ha errors "HA failed state detected, run 'ngcp-check-active'"
  fi
}

check_ngcp_collective_check() {
  local collcheck_opts=()

  case "${NGCP_STATUS_IGNORE:-}" in
    *load*)
      collcheck_opts+=("--skip-checks=load")
      ;;
  esac

  if ! ngcp-collective-check "${collcheck_opts[@]}" >>"${LOG_FILE}" 2>&1 ; then
    state collective_check errors "reports errors, run 'ngcp-collective-check' manually!"
  fi
}

get_os_stats() {
  os_version=$(uname -r)
  os_release=$(cat /etc/debian_version)
  if [ -x "$(which lsb_release)" ] ; then
    os_codename=$(lsb_release -c -s 2>/dev/null)
  else
    os_codename=$(uname -s)
  fi
  os_smp=$(uname -v | grep -s " SMP " >/dev/null 2>&1 && echo "SMP" || echo "non-SMP")
}

check_timezone() {
  if [ ! -L /etc/localtime ]; then
    state tz errors "/etc/localtime is not a symlink, run 'dpkg-reconfigure tzdata'"
    return
  fi

  if [ ! -e /etc/timezone ]; then
    state tz errors "/etc/timezone does not exist, run 'dpkg-reconfigure tzdata'"
    return
  fi

  local _localtime
  local _timezone

  _localtime=$(readlink /etc/localtime)
  _localtime="${_localtime#/usr/share/zoneinfo/}"

  _timezone=$(cat /etc/timezone)

  if [ "${_localtime}" != "${_timezone}" ]; then
    state tz errors "/etc/localtime symlink does not match /etc/timezone contents, run 'dpkg-reconfigure tzdata'"
  fi
}

check_debian_release() {
  local _os_codename=${os_codename:-undefined}

  case "${_os_codename,,}" in
    bookworm|trixie)
      ;;
    *)
      state ngcp errors "NGCP requires Debian bookworm or trixie, while detected '${_os_codename,,}'"
      ;;
  esac
}

get_server_stats() {
  server_ip=$(timeout 2 perl -e 'use IO::Socket::INET; print IO::Socket::INET->new(PeerAddr=>"debian.sipwise.com", PeerPort=>"80", Proto=>"tcp")->sockhost;' || true)
  server_ip=${server_ip:-0.0.0.0}

  server_hostname=$(hostname 2>/dev/null || echo "hostname-unknown")
  server_fqdn=$(hostname -f 2>/dev/null || echo "hostname-fqdn-unknown")
  if [ "${server_hostname}" = "${server_fqdn}" ] ; then
    state lan warnings "misconfiguration? 'hostname' equal to FQDN 'hostname -f'"
  fi
  if [[ "${server_hostname}" = "hostname-unknown" ]] ; then
    state lan errors "'hostname' executed with error (hostname is ${server_hostname})"
  fi
  if [[ "${server_fqdn}" = "hostname-fqdn-unknown" ]] ; then
    state lan errors "'hostname -f' executed with error (FQDN is '${server_fqdn}')"
  fi
}

get_cpu_stats() {
  cpu_count=$(grep 'physical id' /proc/cpuinfo | sort -u | wc -l)
  cpu_cores=$(grep -c 'core id' /proc/cpuinfo)
  cpu_cores_count=$(grep 'cpu cores' /proc/cpuinfo | awk '{print NF; exit}')
  cpu_siblings=$(grep 'siblings' /proc/cpuinfo | awk '{print NF; exit}')
  cpu_info=$(grep -m1 "model name" /proc/cpuinfo | awk -F: '{print $2}')

  if [ "${cpu_cores_count:-1}" != "${cpu_siblings:-1}" ] ; then
    cpu_hyperthreading="(HT)"
  fi
}

get_mem_stats() {
  local _tmp
  _tmp=$(mktemp -t ngcp-mem-stats-XXXXXXXXXX)
  local _mem_eaters=""

  if free -h >>"${LOG_FILE}" 2>&1 ; then
    local _mem_total_short
    _mem_total_short=$(free -h -g | grep Mem: | awk -F" " '{print $2}')
  else
    local _mem_total_short
    _mem_total_short="$(free -g | grep Mem: | awk -F" " '{print $2}')Gb"
  fi

  local _mem_total
  local _mem_used
  local _mem_used_relative
  local _mem_cached
  local _mem_cached_relative
  _mem_total=$(free | grep Mem: | awk -F" " '{print $2}')
  _mem_used=$(free | grep Mem: | awk -F" " '{print $3}')
  _mem_used_relative=$(( _mem_used * 100 / _mem_total ))
  _mem_cached=$(free | grep Mem: | awk '{print $7}')
  _mem_cached_relative=$(( _mem_cached * 100 / _mem_total ))

  ps axo pmem,comm --sort -pmem | head -n 3 | tail -2 > "${_tmp}" 2>&1

  while read -r _line ; do
    local _used
    local _name
    _used=$(echo "${_line}" | awk '{print $1}')
    _name=$(echo "${_line}" | awk '{print $2}')
    _mem_eaters+="${_name}[${_used}%] "
  done < "${_tmp}"

  if [ $(( _mem_used_relative - _mem_cached_relative )) -ge "${RAM_USED_CRIT}" ] ; then
    state ram errors "${_mem_total_short} (used ${_mem_used_relative}%/cache ${_mem_cached_relative}%) ${_mem_eaters}"
  elif [ $(( _mem_used_relative - _mem_cached_relative ))  -ge "${RAM_USED_WARN}" ] ; then
    state ram warnings "${_mem_total_short} (used ${_mem_used_relative}%/cache ${_mem_cached_relative}%) ${_mem_eaters}"
  else
    state ram info "${_mem_total_short} (used ${_mem_used_relative}%/cache ${_mem_cached_relative}%) ${_mem_eaters}"
  fi

  rm -f "${_tmp}"
}

get_ngcpcfg_commit_id() {
  local _path="/etc/ngcp-config/"

  if ! [ -d "${_path}" ] ; then
    warn "folder ${_path} not found"
    return
  fi

  pushd "${_path}" >/dev/null

  local _id
  local _subj
  local _date

  _id=$(git log -1 --pretty=%h)
  _subj=$(git log -1 --pretty=%s | sed 's/\[.*\]//')
  _date=$(git log -1 --pretty=%cr)

  state ngcpcfg info "[${_id}] ${_subj} (${_date})"

  popd >/dev/null
}

get_tls_config_info() {
  declare -a tmp=()

  [ -z "${NGINX_SSL_CIPHERS}" ] || tmp+=("nginx.ssl_ciphers")
  [ -z "${NGINX_SSL_CIPHERS_AUTOPROV}" ] || tmp+=("nginx.ssl_ciphers_autoprov")
  [ -z "${NGINX_SSL_CIPHERS_INTERCEPT}" ] || tmp+=("nginx.ssl_ciphers_intercept")

  if [ -n "${tmp[*]}" ]; then
    state tls info "Custom TLS ciphers: ${tmp[*]}"
  fi
}

check_fsck_repairs() {
  local _fsck_log
  _fsck_log="/run/initramfs/fsck.log"

  if [ -f "${_fsck_log}" ] && (grep -e '^/dev/' "${_fsck_log}" | grep -v ': clean,' -q) ; then
    state fsck warnings "Errors detected during 'fsck' at boot, see:"
    state fsck warnings "  ${_fsck_log}"
    state fsck warnings "(remove/rename file to avoid this warning)"
    return
  fi
}

check_io_stats() {
  if ! ngcp-io-scheduler status 2>&1 | grep -q 'cheduler setting for disk' ; then
    state io errors "Missing scheduler in 'ngcp-io-scheduler status'"
    return
  fi
}

check_grafana_config() {
  local db="/var/lib/grafana/grafana.db"

  if [ "${NGCP_IS_MGMT:-yes}" != "yes" ] ; then
    return
  fi

  if [ ! -e "${db}" ]; then
    state monitoring errors "grafana sqlite database does not exist"
    return
  fi
  if [ ! -s "${db}" ]; then
    state monitoring errors "grafana sqlite database is an empty file, remove it ('rm ${db}')"
    return
  fi

  local _tmp
  _tmp=$(mktemp -t ngcp-grafana-config-out-XXXXXXXXXX)

  if ! sqlite3 "${db}" "select name from data_source" >"${_tmp}" 2>/dev/null ; then
    state monitoring errors "grafana sqlite database cannot be accessed"
    rm -f "${_tmp}"
    return
  fi

  declare -A grafana_db

  while read -r db_source ; do
    grafana_db["${db_source}"]=have
  done <"${_tmp}"

  # shellcheck disable=SC2043
  for db_source in Prometheus ; do
    if [ "${grafana_db[${db_source}]}" != have ]; then
      state monitoring errors "grafana database source ${db_source} not configured"
    fi
  done

  rm -f "${_tmp}"
}

check_mysql_config() {
  local cfg="/etc/mysql/my.cnf"

  if ! grep -cqE "^(\s)?innodb_file_per_table" "${cfg}" ; then
    state mysql warnings "Missing DB option innodb_file_per_table in ${cfg}"
  fi
}

check_mysql_acc() {
  local _host="$1"
  local _port="$2"
  declare -a _opts=()
  _opts+=("--defaults-extra-file=/etc/mysql/sipwise_extra.cnf")
  _opts+=("-h${_host}")
  _opts+=("-P${_port}")
  _opts+=("-Bs")
  _opts+=("-N")
  local _acc_size="0"

  if ! timeout 2 mysql "${_opts[@]}" -e "SELECT 1 FROM DUAL" >/dev/null 2>&1 ; then
    state mysql errors "Is MariaDB up on ${_host}:${_port}? Failed to check kamailio.acc!"
    return
  fi

  local _query="SELECT table_rows FROM information_schema.tables WHERE table_schema = 'kamailio' AND table_name = 'acc';"
  _acc_size=$(timeout 2 mysql "${_opts[@]}" -e "${_query}" 2>/dev/null || true)

  if [ -z "${_acc_size}" ] ; then
    state mysql errors "Failed to get records count in kamailio.acc on ${_host}:${_port}"
    return
  fi

  if [ "${_acc_size}" -ge "${MYSQL_ACC_MAX}" ] ; then
    state mysql warnings "kamailio.acc is too big on ${_host}:${_port} for an upgrade: ${_acc_size} rows"
  fi
}

check_mysql_cdrs() {
  local _host="$1"
  local _port="$2"
  declare -a _opts=()
  _opts+=("--defaults-extra-file=/etc/mysql/sipwise_extra.cnf")
  _opts+=("-h${_host}")
  _opts+=("-P${_port}")
  _opts+=("-Bs")
  _opts+=("-N")
  local _cdr_size="0"

  if ! timeout 2 mysql "${_opts[@]}" -e "SELECT 1 FROM DUAL" >/dev/null 2>&1 ; then
    state mysql errors "Is MariaDB up on ${_host}:${_port}? Failed to check accounting.cdr!"
    return
  fi

  local _query="SELECT table_rows FROM information_schema.tables WHERE table_schema = 'accounting' AND table_name = 'cdr';"
  _cdr_size=$(timeout 2 mysql "${_opts[@]}" -e "${_query}" 2>/dev/null || true)

  if [ -z "${_cdr_size}" ] ; then
    state mysql errors "Failed to get records count in cdr on ${_host}:${_port}"
    return
  fi

  if [ "${_cdr_size}" -ge "${MYSQL_CDR_MAX}" ] ; then
    state mysql warnings "accounting.cdr is too big on ${_host}:${_port} for an upgrade: ${_cdr_size} rows"
  fi
}

check_mysql_size() {
  local _mysql_size="0"
  local _mysql_data="/ngcp-data/mysql/"

  _mysql_size=$(du -ms "${_mysql_data}" | awk '{print $1}' 2>/dev/null)

  if [ -z "${_mysql_size}" ] ; then
    state mysql errors "Failed to get mysql size ${_mysql_data}"
    return
  fi

  if [ "${_mysql_size}" -ge "${MYSQL_SIZE_MAX}" ] ; then
    state mysql warnings "DB ${_mysql_data} is too big for an upgrade: ${_mysql_size} MB"
  fi
}

check_mysql_up_script() {
  local _not_repl_list=""
  local _file="/var/tmp/ngcp-status-up-script-errors"

  if "${CE_EDITION}" ; then
    # nothing to do here on CE
    return
  fi

  if "${MAINTENANCE_MODE:-false}" ; then
    # skip the test during the upgrade to do not confuse users
    return
  fi

  rm -f "${_file}"

  if ! which ngcp-check-rev-applied >/dev/null 2>&1 ; then
    warn "ngcp-check-rev-applied not found, skipped check_mysql_up_script"
    return
  fi

  mapfile -t _not_repl_list < <(find /usr/share/ngcp-db-schema/db_scripts/ -type f -name '*_not_replicated.up' -printf "%f\n" | awk -F_ '{print $1}' | sed 's/^0*//')
  for hostname in $(ngcpcfg get 'hosts.keys.sort'); do
    nodename=$(ngcpcfg get "hosts.${hostname}.nodename")
    ngcp-check-rev-applied --schema db_schema --revision "${_not_repl_list[@]}" --node "${nodename}" | grep "No match" >> "${_file}" 2>&1 || true
  done

  if [ -s "${_file}" ] ; then
    state mysql errors "up script not applied properly, see ${_file}"
    echo -e "\nPlease, fix the issue executing 'ngcp-update-db-schema' manually on the node with the missing up scripts" >> "${_file}"
  fi
}

check_mysql_innodb() {
  local _host="$1"
  local _port="$2"
  declare -a _opts=()
  _opts+=("--defaults-extra-file=/etc/mysql/sipwise_extra.cnf")
  _opts+=("-h${_host}")
  _opts+=("-P${_port}")
  _opts+=("-Bs")
  _opts+=("-N")

  local _file="/var/tmp/ngcp-status-mysql-wrong-engines"
  rm -f "${_file}"

  if ! timeout 2 mysql "${_opts[@]}" -e "SELECT 1 FROM DUAL" >/dev/null 2>&1 ; then
    state mysql errors "Is MariaDB up on ${_host}:${_port}? Failed to check DB engines!"
    return
  fi

  # Remove 'syslog' here when TT#71211 is resolved
  local _query="SELECT TABLE_SCHEMA as DbName ,TABLE_NAME as TableName ,ENGINE as Engine
    FROM information_schema.TABLES
    WHERE ENGINE != 'InnoDB' AND
    TABLE_SCHEMA NOT IN('sys', 'syslog','mysql','information_schema','performance_schema');"

  timeout 2 mysql "${_opts[@]}" -e "${_query}" >"${_file}" 2>&1 || true

  if [ -s "${_file}" ] ; then
    state mysql errors "wrong DB Engines on ${_host}:${_port}, see ${_file}"
    echo -e "\nNote: Sipwise NGCP uses 'InnoDB' DB Engines only." >> "${_file}"
  fi
}

check_mysql() {
  if ! is_package_installed mariadb-server ; then
    warn "MariaDB server is not installed, skipped check_mysql"
    return
  fi

  spawn check_mysql_config
  spawn check_mysql_replication "${MYSQL_PAIR_HOST}" "${MYSQL_PAIR_PORT}" ""
  spawn check_mysql_slowlog     "${MYSQL_PAIR_HOST}" "${MYSQL_PAIR_PORT}"
  spawn check_mysql_processlist "${MYSQL_PAIR_HOST}" "${MYSQL_PAIR_PORT}"
  spawn check_mysql_connections "${MYSQL_PAIR_HOST}" "${MYSQL_PAIR_PORT}"
  spawn check_mysql_deadlock    "${MYSQL_PAIR_HOST}" "${MYSQL_PAIR_PORT}"
  spawn check_mysql_acc         "${MYSQL_PAIR_HOST}" "${MYSQL_PAIR_PORT}"
  spawn check_mysql_cdrs        "${MYSQL_PAIR_HOST}" "${MYSQL_PAIR_PORT}"
  spawn check_mysql_size        "${MYSQL_PAIR_HOST}" "${MYSQL_PAIR_PORT}"
  spawn check_mysql_admin_pass  "${MYSQL_PAIR_HOST}" "${MYSQL_PAIR_PORT}"
  spawn check_mysql_client_sslcert "${MYSQL_PAIR_HOST}" "${MYSQL_PAIR_PORT}"
  spawn check_mysql_users_without_password "${MYSQL_PAIR_HOST}" "${MYSQL_PAIR_PORT}"
  spawn check_mysql_up_script   "${MYSQL_PAIR_HOST}" "${MYSQL_PAIR_PORT}"
  spawn check_mysql_innodb      "${MYSQL_PAIR_HOST}" "${MYSQL_PAIR_PORT}"
  spawn check_mysql_equal_schemes

  if "${CARRIER_EDITION}" && [[ "${NGCP_IS_PROXY}" = "yes" || ( "${NGCP_IS_LI}" = "yes" && "${NGCP_IS_LB}" == "yes" ) ]] ; then
    spawn check_mysql_replication "${MYSQL_LOCAL_HOST}" "${MYSQL_LOCAL_PORT}" ""
    spawn check_mysql_replication "${MYSQL_LOCAL_HOST}" "${MYSQL_LOCAL_PORT}" "db01b"
    spawn check_mysql_slowlog     "${MYSQL_LOCAL_HOST}" "${MYSQL_LOCAL_PORT}"
    spawn check_mysql_processlist "${MYSQL_LOCAL_HOST}" "${MYSQL_LOCAL_PORT}"
    spawn check_mysql_connections "${MYSQL_LOCAL_HOST}" "${MYSQL_LOCAL_PORT}"
    spawn check_mysql_deadlock    "${MYSQL_LOCAL_HOST}" "${MYSQL_LOCAL_PORT}"
    spawn check_mysql_size        "${MYSQL_LOCAL_HOST}" "${MYSQL_LOCAL_PORT}"
    spawn check_mysql_users_without_password "${MYSQL_LOCAL_HOST}" "${MYSQL_LOCAL_PORT}"
  fi
}

check_glusterfs_status() {
  if ! timeout 2 ls /mnt/glusterfs/mgmt-share/ >/dev/null 2>&1 ; then
    state glusterfs errors "glusterfs error, no data in /mnt/glusterfs/mgmt-share/"
  fi
}

check_ngcpcfg_status() {
  if ! which ngcpcfg >/dev/null 2>&1 ; then
    warn "ngcpcfg not found, skipped check_ngcpcfg_status"
    return
  fi

  if ngcpcfg status --local-only 2>&1 | grep ACTION_NEEDED >>"${LOG_FILE}" 2>&1 ; then
    state ngcpcfg errors "ACTION_NEEDED, run 'ngcpcfg status --local-only' manually!"
  fi

  if ! ngcpcfg check >>"${LOG_FILE}" 2>&1 ; then
    state ngcpcfg errors "found YML issue(s), run 'ngcpcfg check' manually!"
  fi
}

check_sipwise_repos_ping() {
  if [ "${NGCP_IS_MGMT:-yes}" = "yes" ] ; then
    if ! timeout 3 ping "${ping_flood}" -c 3 "debian.sipwise.com" >>"${LOG_FILE}" 2>&1 ; then
      state wan warnings "failed to ping 'debian.sipwise.com'"
    elif ! timeout 3 ping "${ping_flood}" -c 3 "deb.sipwise.com" >>"${LOG_FILE}" 2>&1 ; then
      state wan warnings "failed to ping 'deb.sipwise.com'"
    fi
  fi
}

check_sipwise_repos_fetch() {
  local _url="$1"

  if ! timeout 2 wget -4 -O /dev/null "${_url}" >>"${LOG_FILE}" 2>&1 </dev/null ; then
    state wan warnings "failed to wget '${_url}'"
  fi
}

check_sipwise_hosts() {
  spawn check_sipwise_repos_ping

  if [ -r /etc/ngcp_mgmt_node ] ; then
    local mgmt_node
    mgmt_node=$(cat /etc/ngcp_mgmt_node)
  else
    local mgmt_node="db01"
  fi

  local debian_flavour=''
  if command -v lsb_release &>/dev/null ; then
    debian_flavour="$(lsb_release -c -s 2>/dev/null)"
  else
    state wan warnings "failed to identify Debian version, missing lsb-release?"
    return 0
  fi

  local ngcp_version
  ngcp_version="$(cat /etc/ngcp_version)"

  local sipwise_repo="spce/${ngcp_version}"
  local sipwise_url='https://debian.sipwise.com'
  if "${CARRIER_EDITION}"; then
    sipwise_repo="sppro/${ngcp_version}"
    sipwise_url="http://${mgmt_node}:${APPROX_RO_PORT}"
  elif "${PRO_EDITION}"; then
    sipwise_repo="sppro/${ngcp_version}"
  fi

  local trunk_prefix=''
  if [[ "${ngcp_version}" == 'trunk' ]]; then
    sipwise_repo="autobuild"
    trunk_prefix='release-trunk-'
  elif [[ "${ngcp_version}" == 'trunk-weekly' ]]; then
    sipwise_repo="autobuild/release/release-trunk-weekly"
    trunk_prefix="release-trunk-weekly"
    debian_flavour=''
  fi

  local _url="${sipwise_url}/${sipwise_repo}/dists/${trunk_prefix}${debian_flavour}/InRelease"

  check_sipwise_repos_fetch "${_url}"
}

check_resolvconf() {
  if "${CE_EDITION}" ; then
    return
  fi

  if is_package_installed resolvconf ; then
    state lan warnings "resolvconf is installed, it's NOT recommended in PRO/Carrier"
  fi
}

check_dns_servers() {
  local _list=""
  local _dns_servers

  _dns_servers=$(grep -E '^nameserver' /etc/resolv.conf | awk '{print $2}') || true

  if [ "${_dns_servers}" = "" ] ; then
    state lan errors "cannot detect DNS servers, check /etc/resolv.conf !"
    return
  fi

  for _dns_server in ${_dns_servers} ; do
    if ! timeout 2 host sipwise.com "${_dns_server}" >>"${LOG_FILE}" 2>&1 ; then
      _list+="${_dns_server} "
    fi
  done

  if [ "${_list}" != "" ] ; then
    state lan errors "check DNS server(s): ${_list}"
    return
  fi

  # system should have 2+ DNS records
  local _dns_servers_count
  _dns_servers_count=$(grep -Ec '^nameserver' /etc/resolv.conf) || true
  if [ "${_dns_servers_count:-0}" -lt 2 ] ; then
    state lan errors "detected ${_dns_servers_count} DNS server only, required 2+ DNS servers!"
  fi
}

check_ping_nodes() {
  IFS=" " read -r -a _pingnodes <<< "$(crm resource param p_ping show host_list)"
  _count=${#_pingnodes[@]}

  if [ "${_count}" -lt 2 ] ; then
    state ha errors "missing HA pingnodes, detected ${_count}, required 2+"
  elif [ "${_count}" -lt 3 ] ; then
    state ha warnings "recommended HA pingnodes count 3+, found ${_count}"
  fi

  for pingnode in "${_pingnodes[@]}"; do
    if [[ "${pingnode}" =~ 8.8.(8.8|4.4) ]] ; then
      state ha warnings "You have unreliable pingnodes 8.8.8.8/8.8.4.4"
    fi
  done
}

check_ntp_offset()
{
  local offset_unit="$1"
  local offset_time="$2"
  local offset_max="$3"

  # Absolute and unitless value of offset.
  local offset_norm=${offset_time#-}
  offset_norm=${offset_norm%"${offset_unit}"}

  echo "$offset_norm $offset_max" | awk '{print ($1 > $2) ? "failed" : "ok"}'
}

check_ntp_status() {
  # Configuration:
  #  Values below maxoffset are OK; units depend on backend, see below.
  local maxoffset
  local minservers

  local ntpcmd ntpopts
  if [ "${NTP_BACKEND}" = 'ntpd' ]; then
    minservers=3
    # maxoffset in milliseconds.
    maxoffset=1000
    ntpcmd='ntpq'
    ntpopts='-pn'
    ntpmatch='^[*+]'
  else
    minservers=1
    # maxoffset in seconds.
    maxoffset=1
    ntpcmd='timedatectl'
    ntpopts='timesync-status'
    ntpmatch='Server:'
  fi

  if ! which "${ntpcmd}" >/dev/null 2>&1 ; then
    state ntp errors "${ntpcmd} executable not available."
    return
  fi

  local ntp_stdout
  local ntp_stderr
  ntp_stdout=$(mktemp -t ngcp-ntp-stdout-XXXXXXXXXX)
  ntp_stderr=$(mktemp -t ngcp-ntp-stderr-XXXXXXXXXX)

  if ! "${ntpcmd}" "${ntpopts}" >"${ntp_stdout}" 2>"${ntp_stderr}" ; then
    state ntp errors "${ntpcmd} ${ntpopts} returned with error: $(cat "${ntp_stderr}")"
    rm -f "${ntp_stderr}" "${ntp_stdout}"
    return
  fi

  local host offset
  if grep -q "${ntpmatch}" "${ntp_stdout}" ; then # we have a valid current time source
    if [ "${NTP_BACKEND}" = 'ntpd' ]; then
      # ntpd
      host=$(grep "${ntpmatch}" "${ntp_stdout}" | awk '{print $1}' | sed "s/${ntpmatch}//")
      offset=$(grep "${ntpmatch}" "${ntp_stdout}" | awk '{print $9}')
    else
      # timesyncd
      host=$(sed -ne 's/^[[:space:]]*Server:[[:space:]]\([^[:space:]]\+\)[[:space:]]\+(.*)$/\1/p' < "${ntp_stdout}")
      offset=$(sed -ne 's/^[[:space:]]*Offset:[[:space:]]//p' < "${ntp_stdout}")
    fi
 else # no current time source available
    state ntp errors "no current time source for NTP [check ${ntpcmd} ${ntpopts}]"
    rm -f "$ntp_stderr" "$ntp_stdout"
    return
  fi

  local ntpservers
  if [[ "${NTP_BACKEND}" = 'ntpd' ]]; then
    mapfile -t ntpservers <<< "$host"
  else
    mapfile -t ntpservers < <(timedatectl show-timesync --value \
      -pSystemNTPServers -pFallbackNTPServers)
  fi
  if [[ ${#ntpservers[@]} -lt $minservers ]]; then
    state ntp errors "configured ${#ntpservers[@]} NTP servers, required at last ${minservers} NTP servers!"
  fi

  if [ -z "$offset" ] ; then
    state ntp errors "cannot identify NTP offset with time server ${host} [check: ${ntpcmd} ${ntpopts}]"
    rm -f "${ntp_stderr}" "${ntp_stdout}"
    return
  fi

  # Ensure that the offset is within the expected range
  local offset_check
  if [ "${NTP_BACKEND}" = 'ntpd' ]; then
    # ntpd
    local normalized_offset=${offset#-} # absolute value of offset
    offset_check=$(echo "$normalized_offset $maxoffset" | awk '{if ($1 > $2) print "failed"; else print "ok"}')
  else
    # timesyncd
    if [ "${offset}" != "${offset%us}" ]; then
      # Less than 1ms.
      offset_check=ok
    elif [ "${offset}" != "${offset%ms}" ]; then
      # Less than 1s.
      offset_check=ok
    elif [ "${offset}" != "${offset%s}" ]; then
      # Between 1s and 1min.
      offset_check=$(check_ntp_offset s "$offset" "$maxoffset")
    else
      # More than 1min (h, d, w, month), so we unconditionally fail.
      offset_check=failed
    fi
  fi

  if [ "$offset_check" = "failed" ] ; then
    state ntp warnings "offset ${offset} of time server ${host} too large"
    rm -f "${ntp_stderr}" "${ntp_stdout}"
    return
  fi

  rm -f "${ntp_stderr}" "${ntp_stdout}"
}

check_fstrim_status() {
  if ! lsblk -P -o NAME,TYPE,RO,DISC-GRAN 2>/dev/null | grep 'TYPE="disk" RO="0"' | grep -q -v 'DISC-GRAN="0'; then
    # no SSD devices present
    return
  fi

  if [ "${SSD_TRIM}" = "yes" ]; then
    return
  fi

  state fstrim warnings 'SSD detected: Enable FSTRIM service via option fstrim.enable'
}

check_free_space() {
  local _disk_space_critical=""
  local _disk_space_warning=""
  local _disk_space_current=""
  local _file

  _file=$(mktemp -t ngcp-free-space-XXXXXXXXXX)

  df -H --exclude-type=iso9660 --exclude-type=tmpfs --exclude=devtmpfs --exclude=efivarfs 2>/dev/null | awk 'NR>1 {print $5 " " $6}' | sort -u > "${_file}"

  while read -r line ; do
    local _used
    _used="${line%%% *}"
    local _part
    _part="${line##* }"

    if [ "${_used}" -ge "${DISK_USED_CRIT}" ] ; then
      _disk_space_critical+="${_part}[${_used}%] "
    elif [ "${_used}" -ge "${DISK_USED_WARN}" ] ; then
      _disk_space_warning+="${_part}[${_used}%] "
    else
      _disk_space_current+="${_part}[${_used}%] "
    fi

  done <"${_file}"

  if [ "${_disk_space_critical}" != "" ] ; then
    state io_space errors "space used ${_disk_space_critical}"
  fi
  if [ "${_disk_space_warning}" != "" ] ; then
    state io_space warnings "space used ${_disk_space_warning}"
  fi
  state io_space info "space used ${_disk_space_current}"

  rm -f "${_file}"
}

check_disk_inodes() {
  local _disk_inodes_critical=""
  local _disk_inodes_warning=""
  local _disk_inodes_current=""
  local _file
  _file=$(mktemp -t ngcp-disk-inodes-XXXXXXXXXX)

  df -P -H -i 2>/dev/null | grep -vE "^Filesystem|^tmpfs|^cdrom|^udev" | awk '{print $5 " " $6}' | sort -u > "${_file}"

  while read -r line ; do
    local _used
    _used="${line%%% *}"
    local _part
    _part="${line##* }"

    if ! [[ ${_used} =~ [0-9]+ ]] ; then
      continue
    fi

    if [ "${_used}" -ge "${DISK_INODES_CRIT}" ] ; then
      _disk_inodes_critical+="${_part}[${_used}%] "
    elif [ "${_used}" -ge "${DISK_INODES_WARN}" ] ; then
      _disk_inodes_warning+="${_part}[${_used}%] "
    fi

    _disk_inodes_current+="${_part}[${_used}%] "

  done <"${_file}"


  if [ "${_disk_inodes_critical}" != "" ] ; then
    state io_inodes errors "inodes in use ${_disk_inodes_critical}"
  fi
  if [ "${_disk_inodes_warning}" != "" ] ; then
    state io_inodes warnings "inodes in use ${_disk_inodes_warning}"
  fi
  state io_inodes info "inodes in use ${_disk_inodes_current}"

  rm -f "${_file}"
}

get_disks_types() {
  local tmp
  # shellcheck disable=SC1004
  tmp=$(lsblk -d -o name,rota,type | awk '\
{ \
  if ($2 == "1" && $3 == "disk") { \
    s=s "/dev/" $1 " [HDD] "; \
    next ; \
  }; \
  if ($2 == "1" && $3 == "rom") { \
    s=s "/dev/" $1 " [CDROM] "; \
    next ; \
  }; \
  if ($2 == "1") { \
    s=s "/dev/" $1 " [UNKNOWN] "; \
    next ; \
  }; \
  if ($2 == "0") { \
    s=s "/dev/" $1 " [SSD] "; \
    next ; \
  }; \
} \
END \
{ \
  print s;\
}')

  state disks info "${tmp}"
}

check_mount_options() {
  if ! grep " / " /proc/mounts | grep -q noatime >>"${LOG_FILE}" 2>&1 ; then
    state io errors "missing 'noatime' for / , check /etc/fstab"
  fi
}

check_filesystem_type() {
  os_root_fs_type=$(df -T / | tail -1 | awk '{print $2}')
  if ! [[ "${os_root_fs_type}" =~ ext[3,4] ]] ; then
    state io warnings "wrong filesystem type '${os_root_fs_type}' for / (expected ext3/ext4)"
  fi
}

check_tmpfs() {
  if [ "$(stat -f -c '%T' /tmp)" != "tmpfs" ] ; then
    return 0
  fi

  state tmpfs errors "/tmp is running on tmpfs"
}

get_swap_usage() {
  local _tmp
  local _tmp2
  local _swap_used_by=""
  local _total_swap_used
  local _rel_swap_used
  local _pid
  local _proc
  local _pid_swap_used

  _total_swap_used=$(awk '/partition|file/{print $4; exit}' /proc/swaps) || true
  _rel_swap_used=$(awk '/partition|file/{printf "%.0f", $4 / $3 * 100; exit}' /proc/swaps) || true

  if [ "${_total_swap_used}" = "0" ] ; then
    return
  fi

  if ! "${i_am_root}" ; then
    echo "used $(( _total_swap_used / 1000 )) MiB [${_rel_swap_used}%]"
    return
  fi

  _tmp=$(mktemp -t ngcp-swap-tmp-XXXXXXXXXX)
  _tmp2=$(mktemp -t ngcp-swap-tmp2-XXXXXXXXXX)

  # find all processes which uses swap and calculate used swap size
  for _file in /proc/[0-9]*/smaps ; do
    if [ -r "${_file}" ] ; then
      if grep -q '^Swap:' "${_file}" >>"${LOG_FILE}" 2>&1 ; then
        grep '^Swap:' "${_file}" | awk '{sum+=$2} END {printf sum}' >> "${_tmp}"
      else
        echo -n "0" >> "${_tmp}"
      fi
      echo -e "\t${_file} " >> "${_tmp}"
    fi
  done

  # remove processes without swap usage
  grep -E -v '^0\s' "${_tmp}" > "${_tmp2}"

  if ! [ -s "${_tmp2}" ] ; then
    _swap_used_by="no top consumers identified"
  else
    local _top_swappers
    _top_swappers=$(sort -n -r "${_tmp2}" | head -3 | awk '{print $2}')

    for _file in ${_top_swappers} ; do
      _pid=$(echo "${_file}" | cut -d / -f 3)      # /proc/12345/smaps
      _proc=$(ps -o comm --no-headers -p "${_pid}")  # process name for pid 12345
      _pid_swap_used=$(grep "/proc/${_pid}/smaps" "${_tmp}" | awk '{print $1}')
      if [ "${_total_swap_used}" -gt 0 ] ; then
        _pid_swap_relative=$(( _pid_swap_used * 100 / _total_swap_used ))
      else
        _pid_swap_relative=0
      fi
      _swap_used_by+="${_proc}[${_pid_swap_relative}%] "
    done
  fi

  echo "used $(( _total_swap_used / 1000 )) MiB [${_rel_swap_used}%]: ${_swap_used_by}"

  rm -f "${_tmp}" "${_tmp2}"
}

check_swap_usage() {
  local _swap_used
  _swap_used=$(awk '/partition|file/{printf "%.0f", $4 / $3 * 100; exit}' /proc/swaps) || true

  if [ -z "$_swap_used" ] ; then
    state swap errors "SWAP is not found, check and enable it"
  else
    if [ "${_swap_used}" -ge "${SWAP_USED_CRIT}" ] ; then
      state swap errors "$(get_swap_usage)"
    elif [ "${_swap_used}" -ge "${SWAP_USED_WARN}" ] ; then
      state swap warnings "$(get_swap_usage)"
    fi
  fi
}

check_lvm() {
  local _rootdevice
  _rootdevice=$(grep " / " /proc/mounts 2>&1 | grep -v rootfs | awk '{print $1}')

  if [ -z "${_rootdevice}" ] ; then
    state ngcp errors "check lvm: missed device for root mount point '/'"
    return
  fi

  local _rootdevicetype
  _rootdevicetype=$(lsblk -a -r -o TYPE "${_rootdevice}" 2>&1 | tail -1) || true

  if [ -z "${_rootdevicetype}" ] ; then
    state ngcp errors "check lvm: missed root device type for device '${_rootdevice}'"
    return
  fi

  if [ "${_rootdevicetype}" != "lvm" ] ; then
    state ngcp warnings "NGCP on '${_rootdevice}' uses '${_rootdevicetype}'! LVM is mandatory!"
  fi
}

check_load_average() {
  local _la5
  local _la10
  local _la15

  _la5=$(cut -d " " -f1 /proc/loadavg)
  _la10=$(cut -d " " -f2 /proc/loadavg)
  _la15=$(cut -d " " -f3 /proc/loadavg)

  local _la5_int
  local _la10_int
  local _la15_int

  _la5_int=$(echo "${_la5}" | cut -d. -f1)
  _la10_int=$(echo "${_la10}" | cut -d. -f1)
  _la15_int=$(echo "${_la15}" | cut -d. -f1)

  # shellcheck disable=SC2194
  case "load" in
    "${NGCP_STATUS_IGNORE:-}")
      state la info "system load (ignoring): ${_la5} ${_la10} ${_la15}"
      return
    ;;
  esac

  if is_equal_or_greater "${_la5_int}" "${LOAD_AVERAGE_CRIT}" || \
    is_equal_or_greater "${_la10_int}" "${LOAD_AVERAGE_CRIT}" || \
    is_equal_or_greater "${_la15_int}" "${LOAD_AVERAGE_CRIT}"
  then
    state la errors "system overloaded: ${_la5} ${_la10} ${_la15}"
  elif is_equal_or_greater "${_la5_int}" "${LOAD_AVERAGE_WARN}" || \
    is_equal_or_greater "${_la10_int}" "${LOAD_AVERAGE_WARN}" || \
    is_equal_or_greater "${_la15_int}" "${LOAD_AVERAGE_WARN}"
  then
    state la warnings "system heavily loaded: ${_la5} ${_la10} ${_la15}"
  fi

  state la info "system load OK: ${_la5} ${_la10} ${_la15}"
}

check_zombies() {
  local _zombies_parent=""
  local _tmp

  _tmp=$(mktemp -t ngcp-zombies-XXXXXXXXXX)

  # shellcheck disable=SC2009
  ps --no-headers -Ao state,pid,ppid,comm | grep Z > "${_tmp}" || true
  local _count
  _count=$(wc -l < "${_tmp}")

  if [ "${_count}" != "0" ] ; then
    _zombies_parent="${_count} zombies: "

    while read -r line ; do
      _pid=$(echo "${line}" | awk '{print $2}')
      _proc=$(echo "${line}" | awk '{print $4}' | sed 's/ <defunct>//g')
      _ppid=$(echo "${line}" | awk '{print $3}')
      _zombies_parent+="${_proc}/${_pid}[${_ppid}] "
    done < "${_tmp}"
  fi

  if [ "${_count}" -gt 0 ] ; then
    _zombies_parent+="..."
  fi

  if [ -n "${_zombies_parent}" ] ; then
    state ngcp warnings "${_zombies_parent}"
  fi

  rm "${_tmp}"
}

check_rtpengine_kernel_mode() {
  if [ "${ngcp_ha_status}" != "ACTIVE" ] || [ "${NGCP_IS_RTP}" != "yes" ] ; then
    return
  fi

  if ! pgrep rtpengine >>"${LOG_FILE}" 2>&1 ; then
    state ngcp errors "rtpengine is stopped? Should be UP on ACTIVE node!"
    return
  fi

  if ! [ -f /proc/rtpengine/list ] ; then
    state ngcp errors "rtpengine w/o kernelspace-offloading? (missing /proc/rtpengine/list)"
    return
  fi

  local _table
  _table=$(cat /proc/rtpengine/list)
  if [ -z "${_table}" ] ; then
    state ngcp errors "rtpengine w/o kernelspace-offloading? (empty /proc/rtpengine/list)"
    return
  elif ! [ -f "/proc/rtpengine/${_table}/status" ] ; then
    state ngcp errors "rtpengine w/o kernelspace-offloading? (missing /proc/rtpengine/${_table}/status)"
    return
  fi

  local _pid
  _pid=$(grep PID "/proc/rtpengine/${_table}/status" | cut -d: -f2 | sed 's/ //g')
  if [ "${_pid}" = "" ] || [ "${_pid}" = "0" ] ; then
    state ngcp errors "rtpengine w/o kernelspace-offloading! No daemon registered!"
    return
  fi

  if ! rtpengine --nftables-status >>"${LOG_FILE}" 2>&1 ; then
    state ngcp errors "rtpengine w/o kernelspace-offloading! Missing nftables rules!"
    return
  fi
}

check_core_dumps_plain() {
  local _path
  _path=$(dirname "$(cat /proc/sys/kernel/core_pattern)") || true

  local _file
  _file=$(basename "$(cat /proc/sys/kernel/core_pattern)") || true

  if ! [ -d "${_path}" ] ; then
    return
  fi

  local _mask
  _mask=$(echo "${_file}" | sed 's/\%e/[a-z]*/g' | sed 's/\%s/[0-9]*/g' | sed 's/%p/[0-9]*/g') || true
  local _all_cores
  _all_cores=$(find "${_path}" -type f -name "${_mask}" | wc -l)
  local _recent_cores
  _recent_cores=$(find "${_path}" -type f -name "${_mask}" -mtime "-${COREDUMP_EXPIRATION}" | wc -l)

  if [ "${_recent_cores}" -gt 0 ] ; then
    state core errors "found ${_recent_cores} fresh core dump(s), see folder ${_path} !"
  fi

  if [ "${_all_cores}" -gt 0 ] ; then
    state core warnings "found ${_all_cores} old core dump(s), clean folder ${_path}"
  fi
}

check_core_dumps() {
  if ! grep -q /lib/systemd/systemd-coredump /proc/sys/kernel/core_pattern ; then
    state core warnings "not using systemd-coredump, can not check for coredump files"
    return 0
  fi

  local _path='/var/lib/systemd/coredump/'

  local _all_cores
  _all_cores=$(find "${_path}" -type f | wc -l)
  local _recent_cores
  _recent_cores=$(find "${_path}" -type f -mtime "-${COREDUMP_EXPIRATION}" | wc -l)

  if [ "${_recent_cores}" -gt 0 ] ; then
    state core errors "found ${_recent_cores} fresh core dump(s), see folder '${_path}'!"
  fi

  if [ "${_all_cores}" -gt 0 ] ; then
    state core warnings "found ${_all_cores} old core dump(s), clean folder '${_path}'"
  fi
}

check_cdr_errors() {
  local _path='/var/log/ngcp/cdr-errors.log'

  if [ -s "${_path}" ] ; then
    state cdr errors "CDR errors file '${_path}' exists and is not empty. Check contents and delete file when done."
  fi
}


check_subscribers() {
  declare -a _opts=("$@")

  if "${PRO_EDITION}" || "${CARRIER_EDITION}" ; then
    local _prev_host

    for _host in $(ngcpcfg get 'hosts.keys.sort'); do
      local _host_subs=""
      if [ "$(ngcpcfg get "ngcp.has_role('${_host}', 'db')")" -eq "1" ]; then
        _host_subs=$(timeout 2 ngcp-ssh "${_host}" "mysql ${_opts[*]} -e 'select count(id) from kamailio.subscriber;'" 2>/dev/null) || true
      fi

      if [ -n "${_host_subs}" ] && [ -n "${ngcp_subscribers}" ] && \
         [ "${_host_subs}" != "${ngcp_subscribers}" ]; then
        state ngcp errors "different amount of subscribers on ${_prev_host}/${_host}: ${ngcp_subscribers}/${_host_subs}!"
      fi

      if [ -n "${_host_subs}" ]; then
        ngcp_subscribers="${_host_subs}"
      fi
      _prev_host="${_host}"
    done
  else # CE
    ngcp_subscribers=$(timeout 2 mysql "${_opts[@]}" -e 'select count(id) from kamailio.subscriber;') || true
  fi

  [[ "${ngcp_subscribers}" =~ [0-9]+ ]] || ngcp_subscribers="unknown"
  state ngcp subscribers "${ngcp_subscribers}"
}

check_locations() {
  local ngcp_locations
  local ngcp_active_dialogs

  ngcp_locations=$(timeout 2 ngcp-usr-location --all --batch 2>/dev/null) || true
  [[ "${ngcp_locations}" =~ [0-9]+ ]] || ngcp_locations=0
  state ngcp locations "${ngcp_locations}"

  if [ "${ngcp_ha_status}" = "ACTIVE" ] ; then
    ngcp_active_dialogs=$(ngcp-kamctl proxy fifo stats.get_statistics dialog active_dialogs | jq -r '.[0] | split(" = ") | .[1]' 2>/dev/null) || true
    [[ "${ngcp_active_dialogs}" =~ [0-9]+ ]] || ngcp_active_dialogs=0
    state ngcp active_dialogs "${ngcp_active_dialogs}"

    if [ "${ngcp_locations}" -le 0 ] ; then
      if "${PRODUCTION_MODE}" ; then
        state ngcp warnings "active node has no registrations, strange for production"
      else
        state ngcp info "active node has no registrations"
      fi
    fi

    if [ "${ngcp_active_dialogs}" -le 0 ] ; then
      if "${PRODUCTION_MODE}" ; then
        state ngcp warnings "active node has no calls, strange for production"
      else
        state ngcp info "active node has no calls"
      fi
    fi
  fi
}

check_voip_status() {
  if [ "${NGCP_IS_PROXY}" != "yes" ] ; then
    return
  fi
  if [ "${KAMAILIO_PROXY_ENABLE}" != "yes" ] ; then
    return
  fi


  if ! [ -r /etc/default/ngcp-roles ] ; then
    warn "cannot read /etc/default/ngcp-roles, skipped check_voip_status"
    return
  fi

  # shellcheck disable=SC1091
  . /etc/default/ngcp-roles

  declare -a _opts=()
  _opts+=("--defaults-extra-file=/etc/mysql/sipwise_extra.cnf")
  _opts+=("-N")
  _opts+=("-B")

  if "${CE_EDITION}" || "${PRO_EDITION}" ; then
    if ! timeout 2 mysql "${_opts[@]}" -e "SELECT 1 FROM DUAL" >/dev/null 2>&1 ; then
      state mysql errors "Is central MariaDB up? Failed to check voip_status!"
      return
    fi
  fi

  spawn check_subscribers "${_opts[@]}"
  spawn check_locations
}

check_ngcpcfg_git_branch() {
  local _path="/etc/ngcp-config"

  if ! [ -d "${_path}" ] ; then
    warn "missing ${_path}, skipping check_ngcpcfg_git_branch"
    return
  fi

  pushd "${_path}"  >/dev/null

  local _branch
  _branch=$(git rev-parse --abbrev-ref HEAD) || true

  if [ "${_branch}" != "master" ] ; then
    state ngcpcfg errors "ngcpcfg uses branch '${_branch}' instead of 'master'"
  fi

  popd >/dev/null
}

check_udp_drops() {
  local _err="packet receive errors"
  if ! diff <(sleep 0.25s; netstat -su | grep "${_err}") <(netstat -su | grep "${_err}") >/dev/null 2>&1
  then
    state ngcp errors "UDP drops detected (overloading?), check 'netstat -su'"
  fi
}

check_network_errors() {
  # static file /var/tmp/ngcp-status-skip-network-check is for ability to skip some interface till the next reboot
  local _skip_file="/var/tmp/ngcp-status-skip-network-check"

  local _devs
  mapfile -t _devs < <(tail -n +3 /proc/net/dev | awk -F: '{print $1}'| sed "s/\s*//" | grep -ve '^vmnet' -ve '^vboxnet' -ve '^docker' -ve '^usb' | sort -u)

  for _dev in "${_devs[@]}" ; do
    if [ -r "${_skip_file}" ] && grep -q "${_dev}" "${_skip_file}" ; then
      continue
    fi

    if ! ip -j stats show dev "${_dev}" group link \
         | jq '..|objects|select(.errors != 0).errors|select(. != null)|halt_error(5)' &>/dev/null; then
      state lan errors "errors on interface ${_dev}, run 'ip stats show dev ${_dev} group link'"
    fi
  done
}

check_packages_version() {
  if [ "${ngcp_version}" = "" ] || [ "${ngcp_version}" = "unknown" ] ; then
    warn "Cannot detect NGCP version, skipped packages version check"
    return
  fi

  if "${CARRIER_EDITION}" || "${PRO_EDITION}" ; then
    _package="ngcp-ngcp-pro"
  else
    _package="ngcp-ngcp-ce"
  fi

  if ! is_package_installed "${_package}" ; then
    state ngcp errors "Missing mandatory package '${_package}'"
    return
  fi

  local _version
  # shellcheck disable=SC2016
  _version=$(dpkg-query -W -f='${Version}\n' "${_package}" | grep -oE "mr[0-9]+\.[0-9]+\.[0-9]+")

  if dpkg -l | grep -v "${_version}" | grep -q "+0~mr" ; then
    local _count
    _count=$(dpkg -l | grep -v "${_version}" | grep -c "+0~mr")
    local _file="/var/tmp/ngcp-wrong-packages"
    echo "Expected NGCP version ${_version} (detected from ${_package}), while installed:" >"${_file}"
    dpkg -l | grep -v "${_version}" | grep "+0~mr" >>"${_file}" 2>&1
    state ngcp errors "${_count} package(s) with wrong version, see ${_file}"
  fi
}

check_backup_status() {
  if [ -r /etc/ngcp-backup-tools/ngcp-backup.conf ] ; then
    # shellcheck disable=SC1091
    . /etc/ngcp-backup-tools/ngcp-backup.conf
  else
    return
  fi

  case "${BACKUP_MYSQL}" in
    yes|true|1)
      true
      ;;
    *)
      state ngcp warnings "MariaDB backup is disabled. Please, configure backup today"
      ;;
  esac
}

check_support_access() {
  if is_package_installed ngcp-support-noaccess; then
    if "${CE_EDITION}" ; then
      state ngcp info "Sipwise access denied, installed deb ngcp-support-noaccess"
    else
      state ngcp warnings "Sipwise access denied, installed deb ngcp-support-noaccess"
    fi
    return
  fi

  if ! is_package_installed ngcp-support-access; then
    state ngcp warnings "Sipwise access denied, missing deb ngcp-support-access"
  fi

  if ! getent passwd sipwise >>"${LOG_FILE}" 2>&1 ; then
    state ngcp errors "missing POSIX user 'sipwise'"
  fi

  if ! getent group sipwise >>"${LOG_FILE}" 2>&1 ; then
    state ngcp errors "missing POSIX group 'sipwise'"
  fi

  if ! id -nG sipwise 2>/dev/null | grep -q 'sipwise' ; then
    state ngcp errors "POSIX user 'sipwise' is NOT in group 'sipwise'"
  fi
}

check_apache_packages() {
  local apache_packages="apache apache-common apache-utils apache2 apache2-common apache2-doc "
  apache_packages+="apache2-mpm-itk apache2-suexec apache2-suexec-custom apache2-utils "
  apache_packages+="apache2.2-bin apache2.2-common"
  local installed_packages=""

  local tmp_file="/run/ngcp-apache.list"

  for package in ${apache_packages} ; do
    if is_package_installed "${package}"; then
      installed_packages+="${package} "
    fi
  done

  if [ -n "${installed_packages}" ] ; then
    state ngcp warnings "purge Apache packages, NGCP uses Nginx, see ${tmp_file}"
    cat > "${tmp_file}" << EOF
# Here the list of Apache packages which can be purged:
apt-get purge ${installed_packages}
EOF
  fi
}

check_hold_packages() {
  local tmp_file="/var/tmp/ngcp-hold-pkg"

  if dpkg --get-selections | grep 'hold$' > "${tmp_file}" ; then
    state ngcp warnings "detected packages on-hold, see ${tmp_file}"
  fi
}

check_dpkg_packages() {
  local tmp_file
  tmp_file="$(mktemp -t ngcp-hold-pkg-XXXXXXXXXX)"

  if ! dpkg -C >>"${LOG_FILE}" 2>&1 ; then
    state ngcp errors "dpkg found packages issue, run: dpkg -C"
  fi

  rm -f "${tmp_file}"
}

check_sems_packages () {
  if is_package_installed ngcp-sems-app ; then
    state ngcp errors "ngcp-sems-app is installed, purge it: apt-get purge ngcp-sems-app"
  fi
  if is_package_installed ngcp-sems-ha ; then
    state ngcp errors "ngcp-sems-ha is installed, purge it: apt-get purge ngcp-sems-ha"
  fi
}

check_init_packages () {
  local _pkgs_wanted
  local _pkgs_unwanted

  case "${INIT_SYSTEM}" in
    systemd)
      _pkgs_wanted=(systemd systemd-sysv)
      _pkgs_unwanted=(sysvinit-core systemd-shim)
      ;;
    sysv)
      _pkgs_wanted=(sysvinit-core)
      _pkgs_unwanted=(systemd systemd-sysv systemd-shim)
      ;;
    *)
      state ngcp errors "unknown init system ${INIT_SYSTEM}"
      ;;
  esac

  for _pkg in "${_pkgs_wanted[@]}"; do
    if ! is_package_installed "${_pkg}" ; then
      state ngcp errors "mandatory package ${_pkg} is NOT installed"
    fi
  done

  for _pkg in "${_pkgs_unwanted[@]}"; do
    if is_package_installed "${_pkg}" ; then
      state ngcp errors "${_pkg} is installed, purge it: apt-get purge ${_pkg}"
    fi
  done
}

check_license_key() {
  key_file="/etc/ngcp-license-key"

  if [ ! -f "${key_file}" ] ; then
    state ngcp modules "License OFF,"
    state ngcp warnings "Missing license key file '${key_file}'"
    return
  fi

  license_key=$(cat "${key_file}" 2>/dev/null)

  if [ -z "${license_key}" ] ; then
    state ngcp modules "License OFF,"
    return
  fi

  state ngcp modules "License ON,"
  NGCP_LICENSE=true
}

check_ngcp_modules() {
  check_license_key

  if grep -qE "\s+cloudpbx\s+1" /etc/ngcp-panel/ngcp_panel.conf 2>/dev/null ; then
    state ngcp modules "CloudPBX ON,"
    if ! ${NGCP_LICENSE} ; then
      state ngcp warnings "No license key, CloudPBX LOCKED"
    fi
  fi

  if ngcp-service is-managed pushd 2>/dev/null ; then
    state ngcp modules "Pushd ON"

    if grep -qE "gcm_enable\s+=\s+yes" /etc/ngcp-pushd/pushd.conf 2>/dev/null ; then
      state ngcp modules "(GCM ON,"
    else
      state ngcp modules "(GCM OFF,"
    fi

    if grep -qE "apns_enable\s+=\s+yes" /etc/ngcp-pushd/pushd.conf 2>/dev/null ; then
      state ngcp modules "APNS ON),"
    else
      state ngcp modules "APNS OFF),"
    fi

    if ! ${NGCP_LICENSE} ; then
      state ngcp warnings "No license key, Pushd LOCKED"
    fi
  fi

  if [[ "${NGCP_FAXSERVER_ENABLE}" = "yes" ]]; then
    state ngcp modules "Fax ON,"
  fi
}

check_cloudpbx_feature() {
  if [ "${PBX_ENABLE}" = "no" ]; then
    # CloudPBX feature disabled, nothing to do here
    return
  fi

  if [ "${KAMAILIO_PROXY_PRESENCE_ENABLE}" = "no" ]; then
    state ngcp warnings "Enable CloudPBX option kamailio.proxy.presence.enable"
  fi

  if [ "${KAMAILIO_PROXY_ALLOW_REFER_METHOD}" = "no" ]; then
    state ngcp warnings "Enable CloudPBX option kamailio.proxy.allow_refer_method"
  fi
}

check_emergency_mode() {
  if ! grep -qE '^ENABLED=1' /etc/ngcp-emergency-mode/ngcp-emergency-mode.conf 2>/dev/null ; then
    # emergency-mode feature disabled, nothing to do here
    return
  fi

  state ngcp modules "Emergency mode ON,"

  if [ -f /run/ngcp-emergency-mode-enabled ] ; then
    state ngcp warnings "Emergency mode enabled, run: ngcp-emergency-mode status all"
  fi
}

check_mysql_admin_pass() {
  local _host="$1"
  local _port="$2"
  declare -a _opts=()
  _opts+=("--defaults-extra-file=/etc/mysql/sipwise_extra.cnf")
  _opts+=("-h${_host}")
  _opts+=("-P${_port}")
  _opts+=("-Bs")
  _opts+=("-N")

  # checking nonsalted admin passwords on central DB only
  # (other Carrier nodes have non-salted default passwords)
  if [ "${NGCP_IS_DB:-yes}" != "yes" ] ; then
    return
  fi

  if ! timeout 2 mysql "${_opts[@]}" -e "SELECT 1 FROM DUAL" >/dev/null 2>&1 ; then
    state mysql errors "Is MariaDB up on ${_host}:${_port}? Failed to check non-salted password!"
    return
  fi

  local _query="SELECT login FROM billing.admins WHERE md5pass IS NOT NULL;"
  declare -a _logins
  mapfile -t _logins < <(timeout 2 mysql "${_opts[@]}" -e "${_query}" || true )

  if [ "${#_logins[@]}" -gt 0 ] ; then
    state mysql errors "Found non-salted Admin's password, do WEB re-login: $(IFS=$'\n'; echo "${_logins[@]}")"
  fi
}

check_mysql_client_sslcert() {
  local _host="$1"
  local _port="$2"
  declare -a _opts=()
  _opts+=("--defaults-extra-file=/etc/mysql/sipwise_extra.cnf")
  _opts+=("-h${_host}")
  _opts+=("-P${_port}")
  _opts+=("-Bs")
  _opts+=("-N")

  if ! timeout 2 mysql "${_opts[@]}" -e "SELECT 1 FROM DUAL" >/dev/null 2>&1 ; then
    state mysql errors "Is MariaDB up on ${_host}:${_port}? Failed to check ssl_client_certificate!"
    return
  fi

  local _query="SELECT login FROM billing.admins WHERE ssl_client_certificate IS NOT NULL;"
  declare -a _logins
  mapfile -t _logins < <(timeout 2 mysql "${_opts[@]}" -e "${_query}" || true )

  if [ "${#_logins[@]}" -gt 0 ] ; then
    state mysql errors "SSL client certs in DB, inform sipwise and re-issue certs: $(IFS=$'\n'; echo "${_logins[@]}")"
  fi
}

check_mysql_users_without_password() {
  local _host="$1"
  local _port="$2"
  declare -a _opts=()
  _opts+=("--defaults-extra-file=/etc/mysql/sipwise_extra.cnf")
  _opts+=("-h${_host}")
  _opts+=("-P${_port}")
  _opts+=("-Bs")
  _opts+=("-N")

  if ! timeout 2 mysql "${_opts[@]}" -e "SELECT 1 FROM DUAL" >/dev/null 2>&1 ; then
    state mysql errors "Is MariaDB up on ${_host}:${_port}? Failed to check DB user passwords!"
    return
  fi

  local _query='SELECT CONCAT(p.User,"@",p.Host) FROM mysql.global_priv p \
                INNER JOIN (SELECT User, Host, json_value(priv, "$.authentication_string") as password, \
                json_value(priv, "$.account_locked") as locked FROM mysql.global_priv) as pp ON \
                (pp.User=p.User and pp.Host=p.Host AND (pp.password = "" OR pp.password IS NULL) AND pp.locked IS NULL);'
  declare -a _logins
  mapfile -t _logins < <(timeout 2 mysql "${_opts[@]}" -e "${_query}" || true )

  if [ "${#_logins[@]}" -gt 0 ] ; then
    state mysql warnings "DB ${_host}:${_port} contains ${#_logins[@]} users without password: $(IFS=$'\n'; echo "${_logins[@]}")"
  fi

  unset _query _logins

  local _query='SELECT CONCAT(User,"@",Host) FROM mysql.user WHERE User = "export";'
  declare -a _logins
  mapfile -t _logins < <(timeout 2 mysql "${_opts[@]}" -e "${_query}" || true )

  if [ "${#_logins[@]}" -gt 0 ] ; then
    state mysql errors "DB ${_host}:${_port} contains wrong user 'export', drop it!"
  fi
}

check_ssh_root() {
  local PermitRootLogin
  PermitRootLogin=$(grep -Ei "^PermitRootLogin " /etc/ssh/sshd_config | awk '{print $2; exit}')

  if [ -z "${PermitRootLogin}" ] ; then
    state ngcp warnings "missing PermitRootLogin in /etc/ssh/sshd_config, strange for NGCP"
    return
  fi

  case "${PermitRootLogin}" in
    yes)
      state ngcp warnings "allowed root remote SSH by password, use 'prohibit-password'"
      ;;
    no)
      state ngcp warnings "ngcpcfg pull/push might be broken due to PermitRootLogin=no in sshd_config"
      ;;
    forced-commands-only)
      # it might be used in some smart cases
      true
      ;;
    prohibit-password)
      # expected value for now
      true
      ;;
    *)
      state ngcp warnings "unknown PermitRootLogin value '${PermitRootLogin}' detected in sshd_config"
      ;;
  esac
}

check_ssh_listen() {
  if ! grep -Eiq "^ListenAddress " /etc/ssh/sshd_config ; then
    # listens on all interfaces - ok
    return
  fi

  # check for IPv4 localhost

  if ! grep -Eiq "^ListenAddress (0\\.0\\.0\\.0|127\\.0\\.0\\.1)$" /etc/ssh/sshd_config ; then
    state ngcp warnings "SSH is not listening on IPv4 localhost"
  fi

  # check for IPv6 localhost

  if ! grep -Eiq "^ListenAddress (::|::1)$" /etc/ssh/sshd_config ; then
    state ngcp warnings "SSH is not listening on IPv6 localhost"
  fi
}

check_dummy_module() {
  if ! grep -q dummy /proc/net/dev 2>/dev/null ; then
    return
  fi

  state ngcp modules "DUMMY ON,"

  if ! grep -q dummy /etc/modprobe.d/sipwise.conf 2>/dev/null ; then
    state lan errors "Missing module 'dummy' in /etc/modprobe.d/sipwise.conf, see manuals"
  fi

  if ! grep -q dummy /etc/modules-load.d/sipwise.conf 2>/dev/null ; then
    state lan errors "Missing module 'dummy' in /etc/modules-load.d/sipwise.conf, see manuals"
  fi
}

check_maintenance_mode() {
  if "${MAINTENANCE_MODE}" ; then
    state ngcp warnings "MAINTENANCE MODE is enabled"
  fi
}

check_shared_config_discontinued() {
  local discontinued_folder="/mnt/glusterfs/shared_config_discontinued"
  if [ -d "${discontinued_folder}" ] ; then
    state ngcp warnings "remove insecure ${discontinued_folder}"
  fi
}

check_policy_rc_d(){
  local file="/usr/sbin/policy-rc.d"
  if [ ! -f "${file}" ] ; then
    state ngcp errors "${file} missing, we expect it to be present"
  fi
}

get_bios_stats(){
  local version
  local vendor
  local release_date

  local tmp_file
  tmp_file="$(mktemp -t ngcp-bios-XXXXXXXXXX)"

  if ! dmidecode -t bios >"${tmp_file}" 2>/dev/null ; then
    state bios errors "failed to execute 'dmidecode -t bios'"
    rm -f "${tmp_file}"
    return
  fi

  version=$(awk -F": " '/Version:/ {print $NF; exit}' < "${tmp_file}")
  vendor=$(awk -F": " '/Vendor:/ {print $NF; exit}' < "${tmp_file}")
  release_date=$(awk -F": " '/Release Date:/ {print $NF; exit}' < "${tmp_file}")

  if [ -n "${version}" ] && [ "${#version}" -ge 20 ]; then
    version=$(printf "%.20s..." "${version}")
  fi

  if [ -n "${vendor}" ] && [ "${#vendor}" -ge 20 ]; then
    vendor=$(printf "%.20s..." "${vendor}")
  fi

  state bios info "${version:-unknown version} (${release_date:-unknown date}) by ${vendor:-unknown vendor}"

  rm -f "${tmp_file}"
}

get_chassis_info(){
  local product_name
  local product_serial
  local tmp_file
  tmp_file="$(mktemp -t ngcp-server-XXXXXXXXXX)"

  if ! ipmitool fru print 0x00 >"${tmp_file}" 2>/dev/null ; then
    rm -f "${tmp_file}"
    return
  fi

  product_name=$(awk -F": " '/Product Name/ {print $NF; exit}' "${tmp_file}" | tr -s "[:blank:]")
  if [ -z "${product_name}" ] ; then
    product_name=$(awk -F": " '/Board Product/ {print $NF; exit}' "${tmp_file}" | tr -s "[:blank:]")
  fi

  product_serial=$(awk -F": " '/Product Serial/ {print $NF; exit}' "${tmp_file}" | tr -s "[:blank:]")
  state chassis info "${product_name} (${product_serial})"

  rm -f "${tmp_file}"
}

get_hw_info() {
  local tmp
  # shellcheck disable=SC1004
  tmp=$(dmidecode -t system | awk -F: ' \
/Manufacturer:|Product Name:/ \
{
  gsub(/^[ \t]+/, "", $1); \
  gsub(/^[ \t]+/, "", $2); \
  gsub(/[ \t]+$/, "", $1); \
  gsub(/[ \t]+$/, "", $2); \
  if ($1 == "Manufacturer") \
    s=s " by " $2; \
  else \
    s=$2 s \
} \
END \
{ \
  print s \
}')

  state hw info "${tmp}"
}

check_sipwise_ppa() {
  local ppa_list
  local ppa_file
  local ppa_count
  ppa_file="/etc/apt/sources.list.d/sipwise_ppa.list"

  if [ ! -f "${ppa_file}" ]; then
    return
  fi

  ppa_count=$(grep -Ec "^deb " "${ppa_file}" || true)
  if [ "${ppa_count}" -gt 0 ]; then
    ppa_list=$(apt-get indextargets --format="\$(SOURCESENTRY) \$(CODENAME)" | \
                 perl -nE "m{^${ppa_file}} && print((split ' ')[1] . ' ')" || true)
    state ppa warnings "${ppa_count} found: ${ppa_list}"
  fi
}

check_sysv_packages() {
  local sysv_packages="insserv startpar initscripts sysv-rc"
  local installed_packages=""

  local tmp_file="/run/ngcp-sysv.list"

  for package in ${sysv_packages} ; do
    if is_package_installed "${package}"; then
      installed_packages+="${package} "
    fi
  done

  if [ -n "${installed_packages}" ] ; then
    state ngcp warnings "purge SysV packages, NGCP uses Systemd, see ${tmp_file}"
    cat > "${tmp_file}" << EOF
# Here the list of SysV packages which should be purged:
apt-get purge ${installed_packages}
EOF
  fi
}

check_openssl_seclevel() {
  local ssl_config="/etc/ssl/openssl.cnf"

  if grep -Eq "^CipherString.*DEFAULT@SECLEVEL=[0,1]" "${ssl_config}" >/dev/null 2>&1; then
    state ngcp warnings "insecure DEFAULT@SECLEVEL in ${ssl_config}"
  fi
}

check_license_limits() {
  declare -a lic_errors
  declare -A limit_names=(["calls"]="Calls"
    ["pbx_groups"]="PBXGroups"
    ["pbx_subscribers"]="PBXSubscribers"
    ["registered_subscribers"]="RegSubscribers"
    ["subscribers"]="Subscribers")

  for _path in /proc/ngcp/current/*; do
    local _metric
    _metric=$(basename "${_path}")
    if [ ! -f "/proc/ngcp/max/${_metric}" ] || [ ! -f "/proc/ngcp/current/${_metric}" ]; then
      continue
    fi
    local _max
    local _current
    _max=$(<"/proc/ngcp/max/${_metric}")
    _current=$(<"/proc/ngcp/current/${_metric}")
    if [ -n "${_current}" ] && [ -n "${_max}" ] && [ "${_max}" != "unlimited" ] && [ "${_max}" -gt 0 ] && [ "${_current}" -gt "${_max}" ]; then
      local _name=${limit_names["${_metric}"]}
      [ -n "${_name}" ] || _name=${_metric}
      lic_errors+=("${_name}")
    fi
  done

  if [ -n "${lic_errors[*]}" ]; then
    state license errors "License limit exceeded: ${lic_errors[*]}"
  fi
}

# keep track of which capabilities have been checked
declare -A license_capa_checked=()

check_license_capability() {
  local _capa="$1"
  local file="/proc/ngcp/flags/${_capa}"
  if [ ! -f "${file}" ]; then
    return 0
  fi
  local flag
  flag=$(<"/proc/ngcp/flags/${_capa}")
  license_capa_checked["${_capa}"]=1
  [ "${flag}" = "1" ]
}

check_license_capabilities() {
  declare -a capa_errors

  if [ "${PBX_ENABLE}" = "yes" ]; then
    if ! check_license_capability 'pbx'; then
      capa_errors+=('CloudPBX')
    fi
  fi
  if [ "${VOISNIFF_LI_ENABLE}" = "yes" ]; then
    if ! check_license_capability 'voisniff-x2x3'; then
      capa_errors+=('LI')
    fi
  fi
  if [ "${PREPAID_INEWRATE_ENABLE}" = "yes" ]; then
    if ! check_license_capability 'prepaid-inewrate'; then
      capa_errors+=('Inew-Prepaid')
    fi
  fi
  if [ "${PREPAID_SWRATE_ENABLE}" = "yes" ]; then
    if ! check_license_capability 'prepaid-swrate'; then
      capa_errors+=('SW-Prepaid')
    fi
  fi
  if [ "${PUSHD_ENABLE}" = "yes" ]; then
    if ! check_license_capability 'pushd'; then
      capa_errors+=('Pushd')
    fi
  fi
  if [ "${VOISNIFF_MYSQL_DUMP_ENABLE}" = "yes" ]; then
    if ! check_license_capability 'voisniff-mysql_dump'; then
      capa_errors+=('CallStats')
    fi
  fi
  if [ "${EXTERNAL_LNP_ENABLE}" = "yes" ]; then
    if ! check_license_capability 'external_lnp'; then
      capa_errors+=('External LNP')
    fi
  fi
  if [ "${NGCP_FAXSERVER_ENABLE}" = "yes" ]; then
    if ! check_license_capability 'fax'; then
      capa_errors+=('Fax Server')
    fi
  fi
  if [ "${SMS_ENABLE}" = "yes" ]; then
    if ! check_license_capability 'sms'; then
      capa_errors+=('SMS')
    fi
  fi
  if [ "${CALL_RECORDING_ENABLE}" = "yes" ]; then
    if ! check_license_capability 'call_recording'; then
      capa_errors+=('Call Recording')
    fi
  fi
  if [ "${BILLING_ENABLE}" = "yes" ]; then
    if ! check_license_capability 'billing'; then
      capa_errors+=('Billing')
    fi
  fi
  if [ "${XMPP_ENABLE}" = "yes" ]; then
    if ! check_license_capability 'xmpp'; then
      capa_errors+=('XMPP')
    fi
  fi

  # generic check for flags that weren't checked above
  for _path in /proc/ngcp/flags/*; do
    local _flag
    _flag=$(basename "${_path}")
    # skip "enforce" pseudo-flag
    [ "${_flag}" = "enforce" ] && continue
    [ "${license_capa_checked["${_flag}"]}" = 1 ] && continue
    if [ ! -f "/proc/ngcp/flags/${_flag}" ] || [ ! -f "/proc/ngcp/flags_usage/${_flag}" ]; then
      continue
    fi
    local _allowed
    local _used
    _allowed=$(<"/proc/ngcp/flags/${_flag}")
    _used=$(<"/proc/ngcp/flags_usage/${_flag}")
    if [ "${_allowed}" = 0 ] && [ "${_used}" = 1 ]; then
      capa_errors+=("${_flag}")
    fi
  done

  if [ -n "${capa_errors[*]}" ]; then
    state license errors "Services without license: ${capa_errors[*]}"
  fi
}

#####################################################################
# Display functions

print_header() {
  local now
  now="$(date +"%Y-%m-%d %H:%M:%S")"
  if [ "${FORMAT}" = "json" ]; then
    echo "{ \"format\" : 1, "
    echo "  \"data\" : [ "
    echo "{ \"type\" : \"meta\", \"tag\" : \"BEGIN\", \"time\" : \"${now}\" }, "
  else
    printf "+---------------------------+---------------------------------------------- %19s -+\n" "${now}"
  fi
}

print_footer() {
  if is_package_installed ngcp-status ; then
    local _version
    # shellcheck disable=SC2016
    _version=$(dpkg-query -W -f='${Version}\n' ngcp-status) || true
  else
    local _version="unknown"
  fi

  local _line="----------------------------------------------------------"
  local _line_s="-----------"

  [ -n "${start_seconds}" ] && SECONDS="$(( $(cut -d . -f 1 /proc/uptime) - start_seconds ))" || SECONDS="unknown"

  if [ "${FORMAT}" = "json" ]; then
    echo "{ \"type\" : \"meta\", \"tag\" : \"END\", \"version\" : \"${_version}\", \"seconds\": ${SECONDS} } ] }"
  else
    printf "+---------%15s---+-----%60s---+\n" "${_line_s:${#SECONDS}}${SECONDS} sec" "${_line:${#_version}}${_version}"
  fi
}

print_warning() {
  local _prefix="$1"
  local _label="$2"
  local _text="$3"

  if [ "${FORMAT}" = "json" ]; then
    echo "{ \"type\": \"warning\", \"tag\" : \"${_label}\", \"text\": \"${_text//\"/\'}\" }, "
  else
    printf "| %-25s ?${w}%5s:${x} ${y}%-60s${x} |\n" "${_prefix}" "${_label}" "${_text}"
  fi
}

print_warnings() {
  local _label="$1"
  local _arr=("${!2}")

  for _warning in "${_arr[@]}" ; do
    print_warning "" "${_label}" "${_warning}"
  done
}

print_error() {
  local _prefix="$1"
  local _label="$2"
  local _text="$3"

  if [ "${FORMAT}" = "json" ]; then
    echo "{ \"type\" : \"error\", \"tag\" : \"${_label}\", \"text\" : \"${_text//\"/\'}\" }, "
  else
    printf "| %-25s !${w}%5s:${x} ${r}%-60s${x} |\n" "${_prefix}" "${_label}" "${_text}"
  fi
}

print_errors() {
  local _label="$1"
  local _arr=("${!2}")
  local rc=0

  for _error in "${_arr[@]}" ; do
    print_error "" "${_label}" "${_error}"
    rc=1
  done

  # shellcheck disable=SC2086
  return ${rc}
}

print_first_line() {
  local _prefix="$1"
  local _label="$2"
  local _node_type="$3"
  local _node_version="$4"
  local _node_count="$5"
  local _node_roles="$6"

  if [ "${FORMAT}" = "json" ]; then
    echo "{ \"type\" : \"node-meta\", \"tag\" : \"${_label}\", \"node\": { \"type\" : \"${_node_type}\", \"version\" : \"${_node_version}\", \"count\" : ${_node_count}, \"roles\" : \"${_node_roles}\" }, \"context\" : \"${_prefix}\" }, "
  else
    printf "| ${g}%-25s${x} |${w}%5s:${x} ${b}%-18s${x} %-12s %27s |\n" \
           "${_prefix}" "${_label}" "${_node_type} ${_node_version}" "${_node_count} nodes setup" "[ ${_node_roles} ]"
  fi
}

print_node_info() {
  local _prefix="$1"
  local _label="$2"
  local _text="$3"

  if [ "${FORMAT}" = "json" ]; then
    echo "{ \"type\" : \"node-info\", \"tag\" : \"${_label}\", \"text\" : \"${_text//\"/\'}\", \"context\" : \"${_prefix}\" }, "
  else
    printf "| ${ha}%-25s${x} |${w}%5s:${x} %-60s |\n" "${_prefix}" "${_label}" "${_text}"
  fi
}

print_info() {
  local _prefix="$1"
  local _label="$2"
  local _text="$3"

  if [ "${FORMAT}" = "json" ]; then
    echo "{ \"type\" : \"info\", \"tag\" : \"${_label}\", \"text\" : \"${_text//\"/\'}\" }, "
  else
    printf "| %-25s |${w}%5s:${x} %-60s |\n" "${_prefix}" "${_label}" "${_text}"
  fi
}

print_infos() {
  local _label="$1"
  local _arr=("${!2}")

  for _data in "${_arr[@]}" ; do
    print_info "" "${_label}" "${_data}"
  done
}

print_important() {
  local _label="$1"
  local _errors=("${!2}")
  local _warnings=("${!3}")
  local _info=("${!4}")
  local rc=0

  if [ ${#_errors[@]} -gt 0 ] ; then
    print_errors "${_label}" _errors[@] || rc=$?
  elif [ ${#_warnings[@]} -gt 0 ] ; then
    print_warnings "${_label}" _warnings[@]
  elif [ ${#_info[@]} -gt 0 ] ; then
    print_infos "${_label}" _info[@]
  fi

  # shellcheck disable=SC2086
  return ${rc}
}

print_results() {
  local rc=0

  print_header

  load_mesg ngcp subscribers
  load_mesg ngcp locations
  load_mesg ngcp active_dialogs
  print_first_line "${server_ip}" "NGCP" "${ngcp_type}" "${ngcp_version}" "${ngcp_nodes_count}" "${ngcp_roles[*]}"
  print_node_info "${server_hostname}" "NGCP" "subscribers:${ngcp_subscribers} locations:${ngcp_locations} calls:${ngcp_active_dialogs}"
  print_node_info "${server_fqdn}" "OS" "${os_codename} (${os_release}) ${os_version} ${os_smp} (${os_root_fs_type})"
  print_node_info "(${production_label} - ${ngcp_ha_status})" "CPU" "${cpu_cores} = ${cpu_count} x${cpu_info} ${cpu_hyperthreading}"

  print_warnings "SELF" self_warnings[@] || rc=$?

  if "${PRO_EDITION}" || "${CARRIER_EDITION}" ; then
    print_infos "HW" chassis_info[@] || rc=$?
  fi
  print_infos "HW" hw_info[@] || rc=$?
  print_important "BIOS" bios_errors[@] null[@] bios_info[@] || rc=$?

  load_mesg ram errors
  load_mesg ram warnings
  load_mesg ram info
  print_important "RAM" ram_errors[@] ram_warnings[@] ram_info[@] || rc=$?
  load_mesg la errors
  load_mesg la warnings
  load_mesg la info
  print_important "LA" la_errors[@] la_warnings[@] la_info[@] || rc=$?
  load_mesg fsck warnings
  print_important "FSCK" null[@] fsck_warnings[@] null[@] || rc=$?
  load_mesg disks info
  print_infos "DISK" disks_info[@] || rc=$?
  load_mesg io_space errors
  load_mesg io_space warnings
  load_mesg io_space info
  print_important "DISK" io_space_errors[@] io_space_warnings[@] io_space_info[@] || rc=$?
  print_important "DISK" io_inodes_errors[@] io_inodes_warnings[@] io_inodes_info[@] || rc=$?
  load_mesg fstrim warnings
  print_warnings "DISK" fstrim_warnings[@] || rc=$?
  load_mesg swap errors
  load_mesg swap warnings
  print_important "SWAP" swap_errors[@] swap_warnings[@] null[@] || rc=$?

  load_mesg tmpfs errors
  print_errors "TMP" tmpfs_errors[@] null[@] null[@] || rc=$?

  load_mesg io errors
  load_mesg io warnings
  print_warnings "IO" io_warnings[@] || rc=$?
  print_errors "IO" io_errors[@] || rc=$?
  load_mesg apt errors
  load_mesg apt warnings
  print_important "APT" apt_errors[@] apt_warnings[@] null[@] || rc=$?

  load_mesg lan errors
  load_mesg lan warnings
  print_warnings "LAN" lan_warnings[@] || rc=$?
  print_errors "LAN" lan_errors[@] || rc=$?
  load_mesg wan warnings
  print_warnings "WAN" wan_warnings[@] || rc=$?

  load_mesg tz errors
  print_errors "TZ" tz_errors[@] || rc=$?

  load_mesg ngcpcfg info
  load_mesg ngcpcfg errors
  print_infos "CFG" ngcpcfg_info[@] || rc=$?
  print_errors "CFG" ngcpcfg_errors[@] || rc=$?

  load_mesg tls info
  print_infos "TLS" tls_info[@] || rc=$?

  load_mesg ntp errors
  load_mesg ntp warnings
  print_important "NTP" ntp_errors[@] ntp_warnings[@] null[@] || rc=$?

  load_mesg core errors
  load_mesg core warnings
  print_important "CORE" core_errors[@] core_warnings[@] null[@] || rc=$?
  load_mesg cdr errors
  print_warnings "CDR" cdr_errors[@] || rc=$?
  load_mesg ngcp errors
  load_mesg ngcp warnings
  load_mesg ngcp info
  print_warnings "NGCP" ngcp_warnings[@] || rc=$?
  print_errors "NGCP" ngcp_errors[@] || rc=$?
  print_infos "NGCP" ngcp_info[@] || rc=$?

  load_mesg mysql errors
  load_mesg mysql warnings
  load_mesg mysql info
  print_important "DB" mysql_errors[@] mysql_warnings[@] mysql_info[@] || rc=$?
  print_warnings "PPA" ppa_warnings[@] || rc=$?

  load_mesg systemd errors
  load_mesg systemd warnings
  print_important "SYSD" systemd_errors[@] systemd_warnings[@] null[@] || rc=$?
  load_mesg service errors
  load_mesg service warnings
  print_important "SRV" service_errors[@] service_warnings[@] null[@] || rc=$?

  if "${PRO_EDITION}" || "${CARRIER_EDITION}" ; then
    load_mesg ha errors
    load_mesg ha warnings
    print_important "HA" ha_errors[@] ha_warnings[@] null[@] || rc=$?
    load_mesg glusterfs errors
    print_errors "GFS" glusterfs_errors[@] || rc=$?
    load_mesg collective_check errors
    print_errors "MON" collective_check_errors[@] || rc=$?
  fi

  load_mesg monitoring errors
  print_errors "MON" monitoring_errors[@] || rc=$?

  load_mesg ngcp modules
  # shellcheck disable=SC2154
  if [ "${#ngcp_modules[@]}" -gt 0 ]; then
    print_info "" "MODs" "${ngcp_modules[*]}" || rc=$?
  fi
  load_mesg license errors
  print_errors "LIC" license_errors[@] || rc=$?

  print_footer

  # shellcheck disable=SC2086
  return ${rc}
}

print_profiling_footer() {
  echo "$(date) Finished ngcp-status in ${SECONDS} seconds" >> "${LOG_FILE}"
}

#####################################################################
# Main part

run_func check_own_dependencies
run_func check_root_privileges
run_func load_config
run_func set_colors

spawn_func check_load_average

run_func get_production_label
run_func get_ngcp_version
run_func get_ngcp_type
spawn_func get_ngcp_nodes
run_func get_ha_status
run_func get_ngcpcfg_commit_id
run_func get_tls_config_info

run_func get_os_stats
run_func get_server_stats
run_func get_bios_stats
run_func get_cpu_stats
run_func get_mem_stats
run_func get_hw_info
run_func get_disks_types

if "${PRO_EDITION}" || "${CARRIER_EDITION}" ; then
  spawn_func check_ngcp_collective_check
fi

spawn_func check_fsck_repairs
spawn_func check_io_stats
spawn_func check_mysql
spawn_func check_ngcpcfg_status

spawn_func check_debian_release
spawn_func check_apt_list
spawn_func check_approx_config
spawn_func check_approx_cache
spawn_func check_timezone
spawn_func check_customtt_files
spawn_func check_ngcp_files
spawn_func check_sipwise_hosts
spawn_func check_dns_servers
spawn_func check_resolvconf
spawn_func check_free_space
spawn_func check_disk_inodes
spawn_func check_swap_usage
spawn_func check_tmpfs
spawn_func check_lvm
spawn_func check_core_dumps
spawn_func check_cdr_errors
spawn_func check_voip_status
spawn_func check_ngcpcfg_git_branch
spawn_func check_udp_drops
spawn_func check_network_errors
spawn_func check_packages_version
spawn_func check_mount_options
spawn_func check_filesystem_type
spawn_func check_ntp_status
spawn_func check_fstrim_status
spawn_func check_support_access
spawn_func check_apache_packages
spawn_func check_hold_packages
spawn_func check_dpkg_packages
spawn_func check_sems_packages
spawn_func check_init_packages
spawn_func check_ngcp_modules
spawn_func check_cloudpbx_feature
spawn_func check_emergency_mode
spawn_func check_ssh_root
spawn_func check_ssh_listen
spawn_func check_dummy_module
spawn_func check_maintenance_mode
spawn_func check_policy_rc_d
spawn_func check_sipwise_ppa
spawn_func check_sysv_packages
spawn_func check_systemd_missing_reload
spawn_func check_systemd_failed_units
spawn_func check_service_status
spawn_func check_ngcpcfg_api_service
spawn_func check_openssl_seclevel
spawn_func check_license_limits
spawn_func check_license_capabilities

if "${PRO_EDITION}" || "${CARRIER_EDITION}" ; then
  run_func get_ngcp_roles
  run_func get_chassis_info
  spawn_func check_ha_status
  spawn_func check_glusterfs_status
  spawn_func check_ping_nodes
  spawn_func check_rtpengine_kernel_mode
  spawn_func check_backup_status
  spawn_func check_shared_config_discontinued
fi

spawn_func check_grafana_config

#####################################################################
# Display results

wait "${worker_pid[@]}"

run_func check_zombies

set -o pipefail
run_func print_results || RC=$?

exit ${RC}
