#!/bin/bash

set +e
set -u

check() {
  local name="$*"

  echo
  echo "=> $name:"
}

check_version_package() {
  local diff_opts="-U0 -w"
  local cmd="dpkg-query --list"
  $cmd | ngcp-parallel-ssh -I all "diff $diff_opts <($cmd) -"
}

RC=0

check "Show content of /etc/ngcp_version on all nodes"
ngcp-parallel-ssh -t 5 all \
  "echo -n '/etc/ngcp_version: '; cat /etc/ngcp_version" || RC=$?

check "Ensure the metapackages version is equal to the ngcp version above"
ngcp-parallel-ssh -t 5 all \
  "apt-cache policy ngcp-ngcp-pro | grep Installed:" || RC=$?

check "Ensure all nodes have identical Debian packages installed"
check_version_package || RC=$?

echo
echo "NOTE: Software must be identical on all nodes (before and after the upgrade!)"

check "Check SSH connectivity from the current node to all other nodes"
ngcp-check-cluster-ssh || RC=$?

check "Check SSH cross-connectivity from all nodes to all other nodes"
ngcp-check-cluster-ssh cross || RC=$?

check "Check all required services are running on all corresponding nodes"
ngcp-parallel-ssh -t 5 all "ngcp-service summary" || RC=$?

check "Check all node states (active node(s) must print 'active', others 'stanby')"
ngcp-parallel-ssh -t 5 all "echo -n 'HA reports status: '; ngcp-check-active -v || [ \$? -eq 1 ] && true" || RC=$?

check "Check collective-check on all nodes"
ngcp-parallel-ssh all "ngcp-collective-check" || RC=$?

check "Ensure date and time are in sync on all node"
ngcp-parallel-ssh all "date --rfc-3339=ns"

check "Ensure the DNS configuration is consistent among all nodes"
ngcp-parallel-ssh all "grep -E ^nameserver /etc/resolv.conf"

check "Check MariaDB replication status on all nodes"
ngcp-parallel-ssh all "ngcp-mariadb-replication-check -a -v" || RC=$?

exit $RC
