#!/bin/bash
# Purpose: Sipwise NGCP platform upgrade framework

set -e
set -u
set -o pipefail

declare ME
ME="$(basename "$0")"
declare -r OPTIONS=("$@")

declare -r STATUS_FOLDER="/tmp/"
declare -r STATUS_FILE="status"

declare -r BASE_FOLDER="/usr/share/ngcp-upgrade"
declare -r CONF_FOLDER="${BASE_FOLDER}/conf"
declare -r SCENARIOS_FOLDER="${BASE_FOLDER}/scenarios"
declare -r STEPS_FOLDER="${BASE_FOLDER}/steps"
declare UPGRADE_DIR="/ngcp-data/ngcp-upgrade"
declare STEPS_TMP_FOLDER="${UPGRADE_DIR}"
declare LOGFILE="${UPGRADE_DIR}/ngcp-upgrade.log"

declare -r TIME_FORMAT="+%F %T"

declare DISPLAY_HOSTNAME
if command -v ngcp-hostname &>/dev/null; then
  DISPLAY_HOSTNAME="$(ngcp-hostname)"
else
  DISPLAY_HOSTNAME="$(hostname)"
fi

UPGRADE_VERSION=""
OLD_VERSION=""
UPGRADE_DEVEL_MODE=false
FORCE_UPGRADE=${FORCE_UPGRADE:-false}
PAUSE_BEFORE_STEP=""

### helper functions ######################################################
help() {
  echo ""
  echo "  ${ME} - Sipwise NGCP platform upgrade framework"
  echo ""
  echo "  Synopsis:"
  echo "    ${ME} [OPTION]... [TARGET_VERSION]..."
  echo ""
  echo "  Options:"
  echo "    -h|--help           - display help message and exit"
  echo "    -t|--target         - specify target upgrade version"
  echo ""
  echo "  Extra options:"
  echo "    --devel-mode        - allowing upgrades to NGCP developers versions"
  echo "    --force             - forcing requested action, skipping any confirmations"
  echo "    --skip-db-backup    - skip DB backup"
  echo "    --step-by-step      - confirm before proceeding to next step"
  echo "    --pause-before-step - pause execution before step, with argument being either"
  echo "                          the name of the script (e.g. \"backup_mysql_db\")"
  echo "                          or the number of execution in this upgrade (e.g. \"25\")"
}

get_options () {
  local _cmdline_opts="devel-mode,force,help,skip-db-backup,step-by-step,pause-before-step:,target:"
  local _opt_temp

  _opt_temp=$(getopt --name "${ME}" -o +ht: --long $_cmdline_opts -- "${OPTIONS[@]}")

  eval set -- "${_opt_temp}"

  while :; do
    case "$1" in
      --devel-mode)
        UPGRADE_DEVEL_MODE=true
        ;;
      --force)
        FORCE_UPGRADE=true
        ;;
      --help|-h)
        help
        exit 0
        ;;
      --skip-db-backup)
        # this will be used by conf/*/configuration and upgrade steps, several
        # of steps/common/mr*/backup_mysql_db
        DO_DB_BACKUP=no
        export DO_DB_BACKUP
        ;;
      --step-by-step)
        CONFIRM_STEP_BY_STEP="yes"
        export CONFIRM_STEP_BY_STEP
        ;;
      --pause-before-step)
        shift
        PAUSE_BEFORE_STEP="$1"
        ;;
      --target|-t)
        shift; UPGRADE_VERSION="$1"
        ;;
      --)
        shift; break
        ;;
      *)
        echo "Internal getopt error! $1" >&2
        exit 1
        ;;
    esac
    shift
  done
}

set_upgrade_status() {
  [ -n "$1" ] || return 1
  echo "$*" > "${STATUS_FOLDER}/${STATUS_FILE}"
}

enable_upgrade_status_server() {
  mkdir -p "${STATUS_FOLDER}"

  # get rid of already running process
  PID=$(pgrep -f 'python.*SimpleHTTPServer') || true
  [ -n "$PID" ] && kill "$PID"

  (
    cd "${STATUS_FOLDER}"
    python -m SimpleHTTPServer 4242 >/tmp/status_server.log 2>&1 &
  )
}

check_ngcp_installation() {
  if ! [ -f /etc/sipwise_ngcp_version ] ; then
    echo "ERROR: this does not look like an existing Sipwise NGCP installation." >&2
    exit 1
  fi
}

check_root_user() {
  if [ "$(id -u 2>/dev/null)" != 0 ] ; then
    echo "ERROR: this script requires root permissions." >&2
    exit 1
  fi
}

check_for_terminal_multiplexer() {
  if "$FORCE_UPGRADE" ; then
    echo "Skipping check for terminal multiplexer due to FORCE_UPGRADE"
    return 0
  fi

  if ! ngcp-screen-check ; then
    echo "Hint, run: sudo apt-get install screen && screen -S ngcp-upgrade" >&2
    echo "and then execute 'ngcp-upgrade' inside the screen session." >&2
    exit 1
  fi
}

check_upgrade_packages_count() {
  # make sure there's exactly *one* ngcp-upgrade-* package installed,
  # this is just to be bulletproof if we once forget any conflicts...
  local match_lines
  match_lines="$(dpkg-query -f "\${db:Status-Status} \${db:Status-Eflag} \${Package}\n" -W 'ngcp-upgrade-*' 2>/dev/null | grep "^installed ok ")"
  # note that it needs 'echo "${match_lines}"', with the inner double
  # quotes, to not remove newlines
  local match_lines_count
  match_lines_count="$(echo "${match_lines}" | wc -l)"

  if [ "${match_lines_count}" -eq 0 ]; then
    echo "ERROR: no ngcp-upgrade-* Debian package installed." >&2
    echo "Can't execute ngcp-upgrade procedure without according ngcp-upgrade-* package." >&2
    echo "Exiting due to fatal error." >&2
    set_upgrade_status "error"
    exit 1
  elif [ "${match_lines_count}" -ne 1 ]; then
    echo "ERROR: no or multiple versions of ngcp-upgrade-* Debian packages installed." >&2
    echo
    echo "Please make sure to have only one package installed, being the one"
    echo "that can upgrade your running release to the next stable release."
    echo
    echo "List of installed ngcp-upgrade-* packages:"
    echo "${match_lines}" | awk '{print "  " $3}'
    set_upgrade_status "error"
    exit 1
  fi
}

check_proc_availability() {
  if ! [ -r /proc/meminfo ]; then
     echo "Can not read /proc/meminfo - please make sure /proc is mounted, exiting." >&2
     set_upgrade_status "error"
     exit 1
  fi
}

get_old_version(){
  local root="${1:-/}"
  # check which version we are running right now
  if [ -r "${root}"/etc/ngcp_version ] ; then
    OLD_VERSION="$(cat "${root}"/etc/ngcp_version)"
  else
    OLD_VERSION="undefined"
  fi
}

confirm_continue_after_upgrade_paths_blocked_problems() {
  if "$FORCE_UPGRADE" ; then
    echo "Skipping 'upgrade path blocked' problem due to FORCE_UPGRADE"
    return 0
  else
    echo -n "Continue anyway? (yes/no): "
    while true; do
      read -r accept
      case "${accept,,}" in
        yes)
          return 0
          ;;
        no)
          echo "Aborted upgrade as requested!"
          set_upgrade_status "error"
          exit 1
          ;;
        *)
          echo -n "Please answer one of 'yes' or 'no': "
          ;;
      esac
      unset accept
    done
  fi
}

get_upgrade_paths_blocked() {
  local UPGRADE_PATHS_URL="deb.sipwise.com/ngcp"

  if ! [[ -f /etc/default/ngcp-roles ]]; then
    echo "Missing file /etc/default/ngcp-roles, cannot continue!" >&2
    exit 1
  fi

  # shellcheck disable=SC1091
  . /etc/default/ngcp-roles
  if [[ "${NGCP_TYPE}" != "spce" ]]; then
    if ! [[ -f /etc/ngcp_mgmt_node ]]; then
      echo "Missing file /etc/ngcp_mgmt_node, cannot continue!" >&2
      exit 1
    fi
    local NGCP_MGMT_NODE
    NGCP_MGMT_NODE=$(cat /etc/ngcp_mgmt_node)

    local APPROX_RO_PORT
    APPROX_RO_PORT=$(ngcpcfg get bootenv.ro_port)

    UPGRADE_PATHS_URL="${NGCP_MGMT_NODE}:${APPROX_RO_PORT}"
  fi

  UPGRADE_PATHS_BLOCKED_FILE="$(mktemp -t ngcp-upgrade-blocked-XXXXXX)"
  wget --tries=3 --retry-connrefused --timeout=15 --quiet "${UPGRADE_PATHS_URL}/upgrade-paths-blocked" -O "${UPGRADE_PATHS_BLOCKED_FILE}" || rc=$?

  if [[ ${rc:-0} -ne 0 ]] ; then
    echo "ERROR: could not access URL to check for blocked upgrade paths: ${UPGRADE_PATHS_URL}/upgrade-paths-blocked"
    confirm_continue_after_upgrade_paths_blocked_problems
  fi
}

# if the upgrade path is blocked, it will output the reason; otherwise empty
check_upgrade_path_blocked() {
  local check_from=$1
  local check_to=$2

  if [[ -n "${UPGRADE_PATHS_BLOCKED_FILE}" ]] && [[ -r "${UPGRADE_PATHS_BLOCKED_FILE}" ]] ; then
    local output
    output=$(awk "/^<${check_from},${check_to}>/ {print \$0}" "${UPGRADE_PATHS_BLOCKED_FILE}")
    echo "${output}"
  fi
}

# wrapper around check_upgrade_path_blocked() to verify blocked upgrade paths
# between versions, print appropriate messages and request confirmations
check_any_upgrade_paths_blocked() {
  local to_version="$1"
  local msg_ok="$2"
  local msg_blocked="$3"

  local upgrade_blocked
  upgrade_blocked="$(check_upgrade_path_blocked "${OLD_VERSION}" "${to_version}")"
  if [[ -z "${upgrade_blocked}" ]] ; then
    echo "${msg_ok}"
  else
    echo "${msg_blocked}"
    echo
    echo "  ${upgrade_blocked}"
    echo

    confirm_continue_after_upgrade_paths_blocked_problems
  fi
}

get_candidates_unfiltered() {
  local name="$1"

  local base="upgrade-"
  find "${SCENARIOS_FOLDER}" \
    -regextype sed -regex ".*/${base}${name}" -exec basename {} \; | \
    awk -F- '{print $NF}' | sort -u
}

get_candidates() {
  local name="$1"

  local upgrade_pkg upgrade_vers
  upgrade_pkg=$(dpkg-query -S /usr/sbin/ngcp-upgrade | cut -d: -f1)
  # shellcheck disable=SC2016
  upgrade_vers=$(dpkg-query --show -f'\${Version}\n' "${upgrade_pkg}" | awk -F~ '{print $2}')

  for cand in $(get_candidates_unfiltered "${name}"); do
    if [[ "${cand}" = "trunk" ]]; then
      echo "${cand}"
      continue
    fi

    if dpkg --compare-versions "${cand:2}" le "${upgrade_vers:2}" ; then
      echo "${cand}"
    fi
  done
}

get_upgrade_version() {
  local name="${OLD_VERSION}-to-mr[0-9]\{1,\}\.[0-9]\{1,\}\.[0-9]\{1,\}"
  local abort="Abort upgrade"

  if "${UPGRADE_DEVEL_MODE}" ; then
    echo "Sipwise developers environment detected, enabling all upgrade options"
    name="\(${OLD_VERSION}-to-mr.*\|stable-to-trunk\)"
  fi

  local last_record
  last_record="$(tail -1 /etc/sipwise_ngcp_version)"
  local releases
  if [[ "${last_record}" =~ ^Upgrade\ started ]] ; then
    local from_ver="${last_record#Upgrade started. from=}"
    from_ver="${from_ver% to=*}"
    releases="${last_record#Upgrade started. *to=}"
    releases="${releases%% *}"
    echo "It looks like the upgrade is in progress, will continue with ${from_ver}->${releases}"
  else
    releases=$(get_candidates "${name}")
  fi

  if [ -z "${releases}" ] ; then
    echo "No possible upgrades found, probably you have the latest version already."
    echo
    echo "Please install the latest ngcp-upgrade-ce/ngcp-upgrade-pro package and retry."
    echo "Also refer to the handbook for further information:"
    echo "  https://www.sipwise.org/products/spce/documentation/"
    set_upgrade_status "finished"
    exit 0
  fi

  if [ -n "$UPGRADE_VERSION" ] ; then
    check_any_upgrade_paths_blocked "${UPGRADE_VERSION}" \
      "Upgrading to '${UPGRADE_VERSION}' as requested manually" \
      "Upgrade to '${UPGRADE_VERSION}', requested manually, is blocked for the following reason:"

    return
  fi

  if [ "$(echo "${releases}" | wc -w)" = "1" ] ; then
    UPGRADE_VERSION="${releases}"

    check_any_upgrade_paths_blocked "${UPGRADE_VERSION}" \
      "Upgrading to '${UPGRADE_VERSION}' as no other candidates found" \
      "Upgrade to '${UPGRADE_VERSION}', as the only candidate, is blocked for the following reason:"

    return
  fi

  local upgrade_blocked_array=()
  for cand in ${releases} ; do
    local upgrade_blocked
    upgrade_blocked="$(check_upgrade_path_blocked "${OLD_VERSION}" "${cand}")"
    if [[ -n "${upgrade_blocked}" ]] ; then
      upgrade_blocked_array+=("${upgrade_blocked}")
    fi
  done
  if [[ "${#upgrade_blocked_array[@]}" -ne 0 ]] ; then
    echo "WARNING: Some of the possible upgrade paths are blocked,"
    echo "         please avoid them unless you know what you are doing"
    echo
    for entry in "${upgrade_blocked_array[@]}"; do
      echo "  ${entry}"
    done
    echo

    read -r -n 1 -p "Press any key to acknowledge the warning above and continue... "
  fi

  echo "It is possible to upgrade to multiple releases, please choose the one"
  echo "that you would like to upgrade to:"
  select choice in ${releases} "${abort}" ; do
    if [ "${choice}" = "${abort}" ]; then
      echo "Aborted upgrade as requested!" >&2
      set_upgrade_status "aborted"
      exit 1
    elif [ "${choice}" != "" ] ; then
      UPGRADE_VERSION="${choice}"

      check_any_upgrade_paths_blocked "${UPGRADE_VERSION}" \
        "Upgrading to '${UPGRADE_VERSION}' as chosen" \
        "Upgrade to chosen '${UPGRADE_VERSION}' is blocked for the following reason:"

      break
    else
      echo "Exit as requested."
      set_upgrade_status "finished"
      exit 0
    fi
  done
}

adjust_old_version() {
  # Do not depend on any specific OLD_VERSION but instead assume we're running
  # latest stable version when we're upgrading to trunk.
  if [ "${UPGRADE_VERSION}" = "trunk" ] ; then
    OLD_VERSION=stable
  fi
}

check_upgrade_versions() {
  local old_version="$1"
  local new_version="$2"

  if [ -z "${old_version}" ] ; then
    echo "ERROR: could not identify old version." >&2
    set_upgrade_status "error"
    exit 1
  fi
  if [ -z "${new_version}" ] ; then
    echo "ERROR: could not identify upgrade version." >&2
    set_upgrade_status "error"
    exit 1
  fi

  if [ "${old_version}" = "${new_version}" ] ; then
    local last_record
    last_record="$(tail -1 /etc/sipwise_ngcp_version)"

    if [[ "${last_record}" =~ ^System\ configured || "${last_record}" =~ ^Upgrade\ finished ]] ; then
      echo "Nothing to do, upgrade procedure of '${new_version}' has been executed already."
      set_upgrade_status "error"
      exit 0
    elif [[ "${last_record}" =~ ^Upgrade\ started ]] ; then
      echo "Old version '${old_version}' equal to new version '${new_version}'"
      echo "It means this is the second run of ngcp-upgrade after reboot"
      echo "Get the old version from /ngcp-fallback"
      get_old_version "/ngcp-fallback"
    else
      echo "ERROR: can not detect current system status from /etc/sipwise_ngcp_version." >&2
      echo "ERROR: the last record is <${last_record}>" >&2
      set_upgrade_status "error"
      exit 1
    fi
  fi
}

check_upgrade_possibility() {
  local old_version="$1"
  local new_version="$2"

  local path="${SCENARIOS_FOLDER}/${new_version}/upgrade-${old_version}-to-${new_version}"
  if [ -r "${path}" ]; then
    UPGRADE_SCENARIO="${path}"
  else
    echo "ERROR: no upgrade scenarios found to upgrade from ${old_version} to ${new_version}." >&2
    echo "Scenarios folder is: ${SCENARIOS_FOLDER}/${new_version}/" >&2
    set_upgrade_status "error"
    exit 1
  fi
}

check_pause_step_exists() {
  # nothing to check, either not set or a number and not a script name
  if [[ -z "${PAUSE_BEFORE_STEP}" || "${PAUSE_BEFORE_STEP}" =~ ^[0-9]+$ ]] ; then
    return
  fi

  if [[ -n "${UPGRADE_SCENARIO}" ]] ; then
    if ! grep -q "/${PAUSE_BEFORE_STEP}\"" "${UPGRADE_SCENARIO}" &> /dev/null ; then
       echo "ERROR: upgrade step '${PAUSE_BEFORE_STEP}' not found in upgrade scenario: ${UPGRADE_SCENARIO}" >&2
       set_upgrade_status "error"
       exit 1
    fi
  else
    echo "ERROR: upgrade scenario not valid (not determined yet?)" >&2
    set_upgrade_status "error"
    exit 1
  fi
}

confirm_upgrade() {
  local old_version="$1"
  local new_version="$2"

  UPGRADE_DIR="${UPGRADE_DIR}/${old_version}-${new_version}"
  STEPS_TMP_FOLDER="${UPGRADE_DIR}/steps"
  LOGFILE="${UPGRADE_DIR}/logs/ngcp-upgrade-${old_version}-${new_version}-$(date +%s).log"

  if "$FORCE_UPGRADE" ; then
    echo "Skipping upgrade confirmation due to FORCE_UPGRADE"
    return 0
  fi

  echo "NOTE: Upgrading the system will cause downtime of services!"
  echo -n "Please confirm upgrade from '${old_version}' to '${new_version}' (yes/no): "
  while true; do
    read -r a
    case "${a,,}" in
        yes) break ;;
        no) set_upgrade_status "aborted" ; exit 1 ;;
        * ) echo -n "Please answer 'yes' or 'no': " ;;
    esac
    unset a
  done
}

# shellcheck disable=SC2329,SC2317
confirm_reboot() {
  local _REBOOT=false

  touch /run/reboot-required

  if "$FORCE_UPGRADE" ; then
    echo "Finished installation, skipping reboot check due to environment variable FORCE_UPGRADE."
    return 0
  fi

  echo "Need reboot to finish installation."
  echo -n "Reboot now? (yes/no): "
  while true; do
    read -r a
    case "${a,,}" in
      yes) echo "Will reboot system in a few seconds." ; _REBOOT=true ; break ;;
      no) echo "Not rebooting, please do not forget to reboot system to finish installation." ; _REBOOT=false ; break ;;
      * ) echo "Please answer 'yes' or 'no': " ;;
    esac
    unset a
  done

  if $_REBOOT ; then
    reboot
  fi
}

export_variables() {
  local old_version="$1"
  local new_version="$2"

  export BACKUP="/ngcp-data/backup/ngcp-upgrade/ngcp-${old_version}-${new_version}/"
  export LOGFILE
  export OLD_VERSION="${old_version}"
  export UPGRADE_VERSION="${new_version}"
  export NGCP_UPGRADE="ngcp-upgrade"
  export STEPS_FOLDER
  export STEPS_TMP_FOLDER
  export CONF_FOLDER
  export FORCE_UPGRADE
  export UPGRADE_DIR
}

# shellcheck disable=SC2329,SC2317
pause_continue() {
  echo "Continue with this step?"
  echo "  y: yes, and proceed with upgrade without confirming step-by-step"
  echo "  s: yes, and continue with upgrade confirming step-by-step"
  echo -n "Your reply? "
  while true; do
    read -r accept
    case "${accept,,}" in
      y) break ;;
      s) export CONFIRM_STEP_BY_STEP=yes ; break ;;
      *) echo -n "Please answer one of 'y' or 's': " ;;
    esac
    unset accept
  done
}

# shellcheck disable=SC2329,SC2317
run_step() {
  local step="$1"
  local step_for_lockfile

   step_for_lockfile=$(basename "${step}")

  # TT#121952 If there are arguments to the step, include them (with some
  # sanitizing, converting all non-basic characters to '+') in the "lockfile" to
  # track execution of the step.  Use a small checksum to avoid "clashes" when
  # the arguments are similar and only differ in characters that we "sanitize".
  if [[ $# -gt 1 ]] ; then
    local args=( "$@" )
    unset 'args[0]' # remove step-name
    local args_translated
    args_translated=$(echo -n "${args[@]}" | tr -c '0-9A-Za-z:=._-' '+')
    local args_translated_checksum
    args_translated_checksum=$(echo -n "${args[@]}" | md5sum)
    step_for_lockfile+="-${args_translated}-${args_translated_checksum:0:8}-"
  fi

  local profiling="${STEPS_TMP_FOLDER}/ngcp-upgrade-profile-${OLD_VERSION}-${UPGRADE_VERSION}"
  local step_lock_file
  step_lock_file="${STEPS_TMP_FOLDER}/ngcp-upgrade-step-${step_for_lockfile}-${OLD_VERSION}-${UPGRADE_VERSION}"
  local block_steps_skipping="${STEPS_TMP_FOLDER}/ngcp-upgrade-block-skipping-PID$$"

  local RC=0
  local profiling_message="PASS"

  local timestamp
  timestamp="$(date "$TIME_FORMAT")"

  # block for pause-before-step logic
  if [[ -n "${PAUSE_BEFORE_STEP}" ]]; then
    if [[ "${PAUSE_BEFORE_STEP}" =~ ^[0-9]+$ ]]; then
      local current_counter
      current_counter=$(get_upgrade_steps_counter "${OLD_VERSION}" "${UPGRADE_VERSION}")
      # strip [ and /*]
      current_counter=${current_counter:1}
      current_counter=${current_counter%%/*}
      if [[ "${PAUSE_BEFORE_STEP}" -eq ${current_counter} ]]; then
        echo "${timestamp}: $(get_upgrade_steps_counter "${OLD_VERSION}" "${UPGRADE_VERSION}") Pausing as requested before step #${current_counter}: ${step}"
        pause_continue
      fi
    else
      if [[ "${step}" =~ ^((mr|2\.|3\.).*/)?"${PAUSE_BEFORE_STEP}"$ ]]; then
        echo "${timestamp}: $(get_upgrade_steps_counter "${OLD_VERSION}" "${UPGRADE_VERSION}") Pausing as requested before: ${step}"
        pause_continue
      fi
    fi
  fi

  if [ -f "${step_lock_file}" ] && [ ! -f "${block_steps_skipping}" ] ; then
    echo "${timestamp}: $(get_upgrade_steps_counter "${OLD_VERSION}" "${UPGRADE_VERSION}") Step '$step' has been executed already, continuing with the next step..."
    bump_upgrade_steps_counter "${OLD_VERSION}" "${UPGRADE_VERSION}"
    return
  else
    # prevent the following steps skipping if current one is executed
    # TODO/FIXME: as discussed with mprokop@/vseva@ on latest dev meeting
    # this is wrong way while we need to merge it to be able continue refactoring.
    # The properly way is:
    #  - generate upgrade scenario from template
    #  - each step in template must have UUID
    #  - we should operate with steps using UUID
    # All above are planned changes, while to keep upgrade.git in working state
    # we have to merge current implementation, otherwise steps fix_PW_password will be skipped.
    touch "${block_steps_skipping}"
  fi

  local start_step_time
  start_step_time="$(cut -d . -f 1 /proc/uptime)"

  shift # drop step name from "$@"
  echo "${timestamp}: $(get_upgrade_steps_counter "${OLD_VERSION}" "${UPGRADE_VERSION}") '${DISPLAY_HOSTNAME}' Running: ${STEPS_FOLDER}/${step} $*"

  if [[ "${CONFIRM_STEP_BY_STEP:-no}" == "yes" ]] ; then
    echo -n "Execute this step?  ([Y]es / [n]o / [c]ontinue without asking): "
    while true; do
      read -r accept
      case "${accept,,}" in
        y) break ;;
        n) echo "Stopping as requested" ; exit 0 ;;
        c) export CONFIRM_STEP_BY_STEP=no ; break ;;
        * ) echo -n "Please answer one of 'y', 'n' or 'c': " ;;
      esac
      unset accept
    done
  fi

  # TT#121952 For the purpose of resuming execution at the correct place by
  # using "#number" or "#string" as an appendix, strip the appendix when using
  # it as a binary executing the step
  local step_binary="${STEPS_FOLDER}/${step}"
  if [[ "${step_binary}" =~ .*#.* ]] ; then
    step_binary="${step_binary%#*}"
    echo "INFO: Detected step with repetition '${step}', actual step binary to be run will be:"
    echo "      ${step_binary}"
  fi

  if "${step_binary}" "$@" ; then
    bump_upgrade_steps_counter "${OLD_VERSION}" "${UPGRADE_VERSION}"
    date > "${step_lock_file}"
  else
    RC=1
    profiling_message="FAIL"
  fi

  local step_duration
  step_duration="$(( $(cut -d . -f 1 /proc/uptime) - start_step_time ))"
  echo "${profiling_message}: '${step}' in ${step_duration} seconds" >> "${profiling}"

  if [ "0" != "${RC}" ] ; then
    local failed_step
    failed_step="$(get_upgrade_steps_counter "${OLD_VERSION}" "${UPGRADE_VERSION}")"
    # strip surrounding [], replace "/" with "-of-"
    failed_step="${failed_step//\[/}"
    failed_step="${failed_step//]/}"
    failed_step="${failed_step//\//-of-}"

    echo "ERROR: the step '${step}' failed, upgrade aborted!" >&2
    echo "Please fix the root of the issue and restart 'ngcp-upgrade'" >&2
    echo "(see error details in log file ${LOGFILE} )." >&2
    rm -f "${step_lock_file}"
    set_upgrade_status "error"
    exit "${RC}"
  fi

  # We have special step named 'stop' which indicates that we need to stop
  # ngcp-upgrade and exit. Upgrade can be continued later with additional run
  # of ngcp-upgrade.
  if [[ $(basename "${step}") == 'stop' ]]; then
    exit 0
  fi
}

get_upgrade_steps_counter_file() {
  local old_version="$1"
  local new_version="$2"

  mkdir -p "${STEPS_TMP_FOLDER}"
  echo "${STEPS_TMP_FOLDER}/ngcp-upgrade-steps-counter-${old_version}-${new_version}"
}

init_upgrade_steps_counter() {
  local old_version="$1"
  local new_version="$2"

  mkdir -p "${STEPS_TMP_FOLDER}"
  mkdir -p "${UPGRADE_DIR}/logs"

  touch "${LOGFILE}"
  chown root:root "${LOGFILE}"
  chmod 0700 "${LOGFILE}"
  exec  > >(tee -a >&1 "$LOGFILE")
  exec 2> >(tee -a >&2 "$LOGFILE")

  # calculate amount of steps and init counter

  local steps_counter
  steps_counter="$(get_upgrade_steps_counter_file "${old_version}" "${new_version}")"

  if [ ! -r "${UPGRADE_SCENARIO}" ] ; then
    echo "ERROR: cannot read upgrade scenario '${UPGRADE_SCENARIO}'" >&2
    set_upgrade_status "error"
    exit 1
  fi

  if grep -q run_step "${UPGRADE_SCENARIO}" ; then
    total_run_step="$(grep -c '^run_step ' "${UPGRADE_SCENARIO}")"
    echo "1/${total_run_step}" > "${steps_counter}"
  fi
}

# shellcheck disable=SC2329,SC2317
bump_upgrade_steps_counter() {
  local old_version="$1"
  local new_version="$2"

  local steps_counter
  steps_counter="$(get_upgrade_steps_counter_file "${old_version}" "${new_version}")"

  if [ ! -f "${steps_counter}" ] ; then
    echo "WARNING: missing upgrade steps counters file '${steps_counter}'"
  else
    done_run_step="$(awk -F/ '{print $1}' "${steps_counter}")"
    total_run_step="$(awk -F/ '{print $2}' "${steps_counter}")"
    echo "$((done_run_step + 1))/${total_run_step}" > "${steps_counter}"
  fi
}

# shellcheck disable=SC2329,SC2317
get_upgrade_steps_counter() {
  local old_version="$1"
  local new_version="$2"

  local steps_counter
  steps_counter="$(get_upgrade_steps_counter_file "${old_version}" "${new_version}")"

  if [ -r "${steps_counter}" ] ; then
    echo "[$(cat "${steps_counter}")]"
  fi
}

## end helper functions ######################################################

get_options "$@"

check_root_user
check_ngcp_installation
check_for_terminal_multiplexer

enable_upgrade_status_server
set_upgrade_status "init"

check_upgrade_packages_count
check_proc_availability

get_upgrade_paths_blocked

get_old_version
get_upgrade_version
adjust_old_version

check_upgrade_versions "${OLD_VERSION}" "${UPGRADE_VERSION}"
check_upgrade_possibility "${OLD_VERSION}" "${UPGRADE_VERSION}"
check_pause_step_exists
confirm_upgrade "${OLD_VERSION}" "${UPGRADE_VERSION}"

export_variables "${OLD_VERSION}" "${UPGRADE_VERSION}"
init_upgrade_steps_counter "${OLD_VERSION}" "${UPGRADE_VERSION}"

set_upgrade_status "upgrading"
echo "Running scenario: ${UPGRADE_SCENARIO}"
# shellcheck disable=SC1090
source "${UPGRADE_SCENARIO}"
RC=$?

if [ "${RC}" = "0" ] ; then
  set_upgrade_status "finished"
else
  set_upgrade_status "error"
fi

echo "Upgrade to '${UPGRADE_VERSION}' was finished"
exit ${RC}

## END OF FILE #################################################################
