#!/bin/bash

set -e
set -u
set -o pipefail

# set expected umask to avoid warnings by apt
umask 0022

# shellcheck disable=SC2155
declare -r ME="$(basename "$0")"
declare -r OPTIONS=("$@")
invoked_by="$0 $*"

declare -r URL_REGEX="http://[^ ]+\.deb"

if [ ! -f /etc/default/ngcp-proxy ] ; then
  echo "Error: missing config file /etc/default/ngcp-proxy" >&2
  exit 1
fi

# shellcheck disable=SC1091
. /etc/default/ngcp-proxy
declare -r RW_PORT="${APPROX_RW_PORT}"
declare -r RO_PORT="${APPROX_RO_PORT}"
declare -r CURRENT_DEBIAN_RELEASE="${CURRENT_DEBIAN_RELEASE}"
declare -r DEBIAN_REPO_LIST="${DEBIAN_REPO_LIST}"
declare -r NGCP_REPO_LIST="${NGCP_REPO_LIST}"
declare -r APPROX_HOME="/var/cache/approx"

if [ ! -f /etc/ngcp_mgmt_node ] ; then
  echo "Error: missing file /etc/ngcp_mgmt_node" >&2
  exit 1
fi

if [ ! -f /etc/default/ngcp-roles ] ; then
  echo "Error: missing file /etc/default/ngcp-roles" >&2
  exit 1
fi
# shellcheck disable=SC1091
source /etc/default/ngcp-roles
if [[ -z "${NGCP_MAINTENANCE}" ]] ; then
  echo "Error: Missing mandatory environment variable '\$NGCP_MAINTENANCE', exiting." >&2
  exit 1
fi

DEFAULT_MGMT_NODE="$(cat /etc/ngcp_mgmt_node)"
MGMT_NODE="${DEFAULT_MGMT_NODE}"
fake_mgmt_node=false

update=false
update_filter=false
list=false
force=false
policy=false
check=false
clean=false
clean_check=false
download=false
validate=false
check_local_approx=true
noninteractive=false
declare -a filter=()
extra_debian_release=''
extra_ngcp_release=''
skip_debian_repos=false
skip_ngcp_repos=false
skip_other_repos=false
skip_apt_update=false

#########################################################################
# functions

bailout() {
  [ -n "${1:-}" ] && EXITCODE="$1" || EXITCODE=1
  rm -rf "${TMPDIR:-}"
  rm -f "${DEBSRCFILE:-}" "${TMPOUTPUT:-}" "${TMPCACHEOUTPUT:-}"
  exit "${EXITCODE}"
}
trap bailout SIGHUP SIGINT SIGQUIT SIGABRT SIGALRM SIGTERM

help() {
  cat <<EOT

${ME} - Approx cache helper script for NGCP PRO/Carrier systems

${ME} [<option(s)>] <action(s)>

  Actions:
   -a|--auto                     - perform all the necessary actions automatically
   -c|--check                    - check connectivity to deb.sipwise.com
   -C|--clean                    - clean cache, e.g. remove unnecessary repos after previous upgrades
   -d|--download                 - download new packages in central approx and local apt caches,
                                   new packages will only be downloaded (installed packages will be skipped).
                                   The action '--download' can be combined with action '--update'.
   -h|--help                     - display help message and exit
   -p|--policy <package>         - run 'apt-cache policy <package>' using meta data in cache
   -u|--update                   - update apt meta data in approx cache
   -F|--update-filter <filter>   - update apt meta data in approx cache ONLY for the <filter> repository
   -v|--validate                 - validate approx cache

  Options:
   -f|--force                - forcing requested action, skipping any confirmations, try to fix detected problems
   -l|--list                 - display used apt repositories
   -N|--node <node>          - Approx node (to overwrite default active mgmt node)
   -S|--skip-approx-check    - skip local approx connectivity check (for usage e.g. in ngcp-installer during initial setup)
   -r|--extra-debian-release - the name of the debian release which should be downloaded to cache
   -w|--extra-ngcp-release   - the version of Sipwise NGCP release which should be downloaded to cache
   -n|--non-interactive      - non interactive mode, no question will be asked, any problem will cause error exit
      --clean-check          - report if cache contains old repos to be cleaned
      --skip-apt-update      - skip running 'apt update' at the end of approx cache update
      --skip-all-repos       - skip all _local_ APT repositories
      --skip-debian-repos    - skip _default_ local APT Debian repository list
      --skip-ngcp-repos      - skip _default_ local APT Sipwise NGCP repository list
      --skip-other-repos     - skip all other _non-default_ local APT repositories

  DEPRECATED: Options:
   --release     - the option has been renamed to '--extra-debian-release', '--release' will be removed soon.
   -k|--keys     - no need to download any key from site, they are provided by ngcp-archive-keyring package, '--keys' will be removed soon.
   --skip-grml-repos - we do not use grml repo here so the option will be removed soon.

EOT
}

get_options() {
  local _cmdline_opts="auto,check,clean,download,help,policy:,"
  _cmdline_opts+="update,update-filter:,force,list,node:,validate,skip-approx-check,extra-debian-release:,extra-ngcp-release:,"
  _cmdline_opts+="non-interactive,clean-check,skip-all-repos,skip-debian-repos,skip-ngcp-repos,skip-other-repos,skip-apt-update"

  # deprecated options (will be removed one day)
  _cmdline_opts+=",release:,keys,skip-grml-repos"

  local _opt_temp
  _opt_temp=$(getopt -n "${ME}" -o "+uF:acCdlfp:hN:vSr:w:n" -l "${_cmdline_opts}" -- "${OPTIONS[@]}")

  eval set -- "${_opt_temp}"

  while :; do
    case "$1" in
    --help|-h)
      help
      exit 0
      ;;
    --update|-u)
      update=true
      ;;
    --update-filter|-F)
      shift; filter=("$1")
      update_filter=true
      update=true
      ;;
    --download|-d)
      download=true
      ;;
    --list|-l)
      list=true
      ;;
    --force|-f)
      force=true
      ;;
    --policy|-p)
      shift; package="$1"
      policy=true
      ;;
    --keys|-k)
      echo "Warning: the option '--keys|-k' is deprecated."
      ;;
    --auto|-a)
      update=true
      check=true
      validate=true
      ;;
    --check|-c)
      check=true
      ;;
    --clean|-C)
      clean=true
      ;;
    --clean-check)
      clean_check=true
      ;;
    --node|-N)
      shift; MGMT_NODE="$1"
      fake_mgmt_node=true
      ;;
    --validate|-v)
      validate=true
      ;;
    --skip-approx-check|-S)
      check_local_approx=false
      ;;
    --release)
      echo "Warning: the option '--release' is deprecated. It has been renamed to '--extra-debian-release'"
      shift; extra_debian_release="$1"
      ;;
    --extra-debian-release|-r)
      shift; extra_debian_release="$1"
      ;;
    --extra-ngcp-release|-w)
      shift; extra_ngcp_release="$1"
      ;;
    --non-interactive|-n)
      noninteractive=true
      ;;
    --skip-all-repos)
      skip_debian_repos=true
      skip_ngcp_repos=true
      skip_other_repos=true
      ;;
    --skip-debian-repos)
      skip_debian_repos=true
      ;;
    --skip-ngcp-repos)
      skip_ngcp_repos=true
      ;;
    --skip-grml-repos)
      echo "Warning: the option '--skip-grml-repos' is deprecated."
      ;;
    --skip-other-repos)
      skip_other_repos=true
      ;;
    --skip-apt-update)
      skip_apt_update=true
      ;;
    --)
      shift; break
      ;;
    *)
      echo "Internal getopt error! $1" >&2
      exit 1
      ;;
    esac
    shift
  done
}

check_options() {
  if "${policy}" ; then
    echo "Checking policy for package '${package}'"
  elif "${check}" ; then
    echo "Checking Sipwise servers connectivity..."
  elif "${clean_check}" ; then
    echo "Checking if the cache could be cleaned (e.g. old repos)..."
  elif "${clean}" ; then
    echo "Cleaning approx cache..."
  elif ! "${update}" && ! "${validate}" && ! "${download}"; then
    echo
    echo "ERROR: Missing 'action'. Please check the following usage instructions." >&2
    help >&2
    bailout 1
  fi
}

# clean obsolete debs from old releases, the Debian part (not mr*), by:
# 1) gathering a list of packages in the pools
# 2) getting from Packages.* of each release component the Filename (path within
#    pool/) for all packages in the current release (e.g. buster)
# 3) "subtracting" from the files existing in pools those who are from the
#    current release -- to not remove those
# 4) removing the resulting files
#
# this is done once per "debian*" subdirectory (like debian-security,
# debian-debug, debian itself) because even if unlikely in today's practice,
# there might be more than one package with the same path/filename, so
# restricting matches to the same "debian*" subdir is slightly more robust and
# future-proof
clean_unnecessary_cached_debs() {
  local base_dir="${APPROX_HOME}"
  local codename=""
  codename="${extra_debian_release:-$(lsb_release --codename --short)}"

  if "${NGCP_MAINTENANCE:-true}"; then
    echo -e "INFO: It is not possible to remove obsolete cached package\\n" \
            "      from ${base_dir}/debian*/pool dirs during upgrades"
    return
  fi

  echo -e "INFO: Will try to remove obsolete cached package files previous to\\n" \
          "      '${codename}' from ${base_dir}/debian*/pool dirs"
  if ! request_confirmation ; then
    return
  fi

  local list_pool=""
  list_pool="$(mktemp -t ngcp-approx-cache-pool-XXXXXXXXXX)"
  local list_dists=""
  list_dists="$(mktemp -t ngcp-approx-cache-dists-XXXXXXXXXX)"
  local list_to_remove=""
  list_to_remove="$(mktemp -t ngcp-approx-cache-list-XXXXXXXXXX)"
  local files_removed=""
  files_removed="$(mktemp -t ngcp-approx-cache-files-XXXXXXXXXX)"

  local d=""
  for d in "${base_dir}"/debian*/; do
    pushd "${d}" > /dev/null

    # sometimes "pool/" might not exist, in which case nothing to do
    if [[ ! -d "pool" ]]; then
      echo "INFO: non-existing pool subdir, skipping ${d}"
      continue
    fi

    # step 1)
    find pool/ -type f -name '*.deb' | sort > "${list_pool}"
    if [[ ! -s "${list_pool}" ]]; then
      echo "INFO: Empty pool, skipping ${d}"
      continue
    fi

    # step 2)
    xzgrep '^Filename: ' dists/"${codename}"*/*/binary-amd64/Packages.* | cut -d' ' -f 2 | sort -u > "${list_dists}"
    if [[ ! -s "${list_dists}" ]]; then
      echo "WARNING: No filenames from 'dists' files, something seems wrong, skipping ${d}"
      continue
    fi

    # step 3)
    comm -2 -3 "${list_pool}" "${list_dists}" > "${list_to_remove}"
    if [[ ! -s "${list_to_remove}" ]]; then
      echo "INFO: No files to remove, skipping ${d}"
      continue
    fi

    # step 4)
    local files_to_remove=()
    while IFS= read -r line; do
      local f=./"${line}"
      if [[ -f "${f}" ]]; then
        files_to_remove+=("${f}")
      fi
    done < "${list_to_remove}"

    if [[ ${#files_to_remove[@]} -gt 0 ]]; then
      local size
      size="$(du -shc "${files_to_remove[@]}" | awk '/total$/ {print $1}')"
      echo "INFO: Will remove obsolete cached package files from ${d}"
      echo " - ${#files_to_remove[@]} files, with a total size of ${size}, full list in: ${files_removed}"

      # shellcheck disable=SC2129
      echo "Files removed for dir ${d}:" >> "${files_removed}"
      xargs --delimiter='\n' --arg-file="${list_to_remove}" --max-lines=1000 rm -fv &>> "${files_removed}"
      echo >> "${files_removed}"
    else
      echo "INFO: No obsolete cached package files from ${d}"
    fi

    popd > /dev/null
  done

  # remove temporary files except "${files_removed}", to act as log
  rm -f "${list_pool}" "${list_dists}" "${list_to_remove}"
}

get_unnecessary_repos() {
  local base_dir="${APPROX_HOME}/sppro"
  local current_version_file="/etc/ngcp_version"
  # by reference
  local -n dirs_to_remove_ref="$1"
  dirs_to_remove_ref=()

  if [[ ! -r "${current_version_file}" ]] ; then
    echo "ERROR: could not read file: ${current_version_file}" >&2
    bailout 1
  fi

  local current_version=""
  current_version=$(< "${current_version_file}")

  if [[ ! -d "${base_dir}" ]] ; then
    echo "INFO: Dir does not exist, nothing to clean: ${base_dir}"
    return
  elif ! ls -d "${base_dir}"/*/ &> /dev/null ; then
    echo "INFO: Dir does not contain subdirs, nothing to clean: ${base_dir}"
    return
  elif [[ ! -w "${base_dir}" ]] ; then
    echo "ERROR: no write access for dir: ${base_dir}" >&2
    bailout 1
  fi

  # we do want to use the current form to "filter out" with grep, not to "filter
  # in", so override shellcheck
  #
  # shellcheck disable=SC2010
  for d in $(ls --sort=version -d "${base_dir}"/*/ 2> /dev/null | grep -v "/${current_version}/$"); do
    local ver_to_check="${current_version}"
    if [[ ! -d "${d}" ]]; then
      echo "ERROR: does not appear to be a directory: ${d}" >&2
      bailout 1
    elif ! [[ "${d}" =~ ${base_dir}/.*/ ]]; then
      echo "ERROR: does not appear to be a directory inside ${base_dir}, aborting for safety: ${d}" >&2
      bailout 1
    fi

    d_basename=$(basename "${d}")

    # If the current version of the system is mrX.Y.Z and there is mrX.Y
    # in approx cache dpkg --compare-versions shows that  mrX.Y.Z > mrX.Y
    # i.e. current version is higher so ngcp-approx-cache deletes
    # mrX.Y directory. But it isn't true in our versioning - mrX.Y is
    # higher than mrX.Y.Z.
    # To handle this case reduce current version to mrX.Y for comparison.
    if [[ "${d_basename}" =~ ^mr[0-9]+\.[0-9]+$ && "${ver_to_check}" =~ ^mr[0-9]+\.[0-9]+\.[0-9]+ ]]; then
      ver_to_check="$(echo "${ver_to_check}" | sed -r 's/(mr[0-9]+\.[0-9]+).+/\1/')"
    fi

    if dpkg --force-bad-version --compare-versions "${d_basename}" ge "${ver_to_check}" &>/dev/null ; then
      echo "INFO: ignoring directory, seems to be from same or higher version: ${d}"
      continue
    fi

    dirs_to_remove_ref+=( "${d}" )
  done

  if [[ "${#dirs_to_remove_ref[@]}" -eq 0 ]] ; then
    echo "INFO: Dir does not contain subdirs that need cleaning: ${base_dir}"
    return
  fi
}

get_upgrade_files() {
  echo "INFO: Updating meta-release file"

  wget --tries=0 -q -O "${TMPDIR}/meta-release" "http://${MGMT_NODE}:${RW_PORT}/meta-release"
  wget --tries=0 -q -O "${TMPDIR}/upgrade-paths-blocked" "http://${MGMT_NODE}:${RW_PORT}/upgrade-paths-blocked"

  chown approx:approx "${TMPDIR}/meta-release" "${TMPDIR}/upgrade-paths-blocked"

  cp -a "${TMPDIR}/meta-release" "${APPROX_HOME}/meta-release"
  cp -a "${TMPDIR}/upgrade-paths-blocked" "${APPROX_HOME}/upgrade-paths-blocked"
}

clean_unnecessary_repos() {
  declare -a dirs_to_remove

  # get "dirs_to_remove" as array
  get_unnecessary_repos dirs_to_remove

  if [[ "${#dirs_to_remove[@]}" -eq 0 ]] ; then
    return
  fi

  echo "INFO: Will remove the following dirs, with space savings:"
  du -shc "${dirs_to_remove[@]}"
  echo

  if request_confirmation ; then
    echo "INFO: Removing the following dirs:"
    for d in "${dirs_to_remove[@]}"; do
      echo " - ${d}"
      rm -fr "${d}"
    done
  fi

  unset dirs_to_remove
}

check_clean_unnecessary_cache() {
  declare -a dirs_to_remove

  # get "dirs_to_remove" as array
  get_unnecessary_repos dirs_to_remove

  if [[ "${#dirs_to_remove[@]}" -ne 0 ]] ; then
    echo "Warning: There is old content that could be removed" >&2
    bailout 1
  fi

  echo "Info: The cache looks clean"
  bailout 0
}

add_non_default_repository() {
  local _from_port="$1"
  local _to_port="$2"
  local _sed_opts="s/:${_from_port}/:${_to_port}/g"

  if [[ "${skip_other_repos}" == "true" ]]; then
    echo "Skipping update for non-default local Debian repositories as requested"
    return
  fi

  echo "# Non-default local Debian repositories" >> "${DEBSRCFILE}"
  find /etc/apt/ -name '*\.list' \
    -not -wholename "${DEBIAN_REPO_LIST}" \
    -not -wholename "${NGCP_REPO_LIST}" \
    -exec grep -E "^deb " {} \; | sed "${_sed_opts}" >> "${DEBSRCFILE}"
}

add_debian_repository() {
  local _from_port="$1"
  local _to_port="$2"
  local _sed_opts=""

  if [[ "${skip_debian_repos}" == "true" ]]; then
    echo "Skipping update for default Debian repositories as requested"
    return
  fi

  if "${update_filter}" ; then
    for repo in "${filter[@]}"; do
      _sed_opts+="/${repo}/s/:${_from_port}/:${_to_port}/g;"
    done
  else
    _sed_opts="s/:${_from_port}/:${_to_port}/g"
  fi

  if ! grep -qE "^deb " "${DEBIAN_REPO_LIST}" >/dev/null 2>&1; then
    echo "Warning: no enabled Debian repositories found in '${DEBIAN_REPO_LIST}'"
  else
    echo "# Default Debian repository" >> "${DEBSRCFILE}"
    grep -E "^deb " "${DEBIAN_REPO_LIST}" | sed "${_sed_opts}" >> "${DEBSRCFILE}"
  fi
}

add_ngcp_repository() {
  local _from_port="$1"
  local _to_port="$2"
  local _sed_opts=""

  if [[ "${skip_ngcp_repos}" == "true" ]]; then
    echo "Skipping update for default NGCP repositories as requested"
    return
  fi

  if "${update_filter}" ; then
    for repo in "${filter[@]}"; do
      _sed_opts+="/${repo}/s/:${_from_port}/:${_to_port}/g;"
    done
  else
    _sed_opts="s/:${_from_port}/:${_to_port}/g"
  fi

  if ! grep -qE "^deb " "${NGCP_REPO_LIST}" >/dev/null 2>&1; then
    echo "Warning: no enabled NGCP repositories found in '${NGCP_REPO_LIST}'"
  else
    echo "# Default Sipwise NGCP repository" >> "${DEBSRCFILE}"
    grep -E "^deb " "${NGCP_REPO_LIST}" | sed "${_sed_opts}" >> "${DEBSRCFILE}"
  fi
}

apply_fake_mgmt_node_on_list() {
  if "${fake_mgmt_node}" ; then
    sed -i "s/${DEFAULT_MGMT_NODE}/${MGMT_NODE}/" "${DEBSRCFILE}"
  fi
}

add_extra_debian_repository() {
  local _debian_release="$1"
  local _port="$2"

  [[ -z "${_debian_release}" ]] && return

  echo "Adding extra Debian repository for release ${_debian_release} to the list"

  {
    local deb_server="deb http://${MGMT_NODE}:${_port}"
    local suites="main contrib non-free non-free-firmware"
    echo "# Extra Debian APT repositories requested for release '${_debian_release}' by user"
    echo "${deb_server}/debian/                ${_debian_release}                ${suites}"
    echo "${deb_server}/debian/                ${_debian_release}-updates        ${suites}"
    echo "${deb_server}/debian-security/       ${_debian_release}-security       ${suites}"
    echo "${deb_server}/debian-debug/          ${_debian_release}-debug          ${suites}"
    echo "${deb_server}/debian-security-debug/ ${_debian_release}-security-debug ${suites}"
  } >> "${DEBSRCFILE}"
}

add_extra_ngcp_repository() {
  local _ngcp_release="$1"
  local _port="$2"
  # use extra debian release if defined by user, otherwise use the current one
  # The logic behind:
  #   - we are on stretch in mrX.X, user needs ngcp-upgrade-pro from mrY.Y
  #     user has to specify "--extra-ngcp-release mrY.Y" as Debian version is the same
  #   - we are on stretch in mrX.X, user needs ngcp-upgrade-pro from mrY.Y which is on buster
  #     user has to specify "--extra-ngcp-release mrY.Y --extra-debian-release buster"
  #     it will fill both debian and ngcp approx caches for new release buster (both will be necessary)
  #   - additionally, when mrY.Y is still trunk, there might be both 'stretch' and 'buster' available
  #     and developers need an ability to install necessary one only during local testing
  local _debian_release="${3:-${CURRENT_DEBIAN_RELEASE}}"

  [[ -z "${_ngcp_release}" ]] && return

  echo "Adding extra NGCP repository for release ${_ngcp_release} for Debian release '${_debian_release}'"

  if [[ "${_ngcp_release}" = "trunk" ]]; then
    REPO_STRING="http://${MGMT_NODE}:${_port}/autobuild/ release-${_ngcp_release}-${_debian_release} main"
  else
    # We can hardcode 'sppro' here as CE doesn't have approx support and it is not planned.
    REPO_STRING="http://${MGMT_NODE}:${_port}/sppro/${_ngcp_release}/ ${_debian_release} main"
  fi

  {
    echo "# Extra NGCP APT repository requested for release '${_ngcp_release}' by user"
    echo "deb ${REPO_STRING}"
  } >> "${DEBSRCFILE}"
}

check_force_option() {
  if "${force}" ; then
    echo "Forcing action as requested via --force"
    return 0
  fi
  if "${noninteractive}" ; then
    echo "Forcing action as requested via --non-interactive"
    return 0
  fi
  echo
  echo "You are going to update apt meta data in approx cache on management node."
  echo "It is necessary when you need to install newer version of some package."
  echo "When you finish, please ensure all servers have identical packages version."
  echo
  echo "NOTE: This action cannot be undone!!!"
  echo -n "Please, confirm your action (yes/NO): "
  read -r a
  case ${a,,} in
    yes|y) ;;
    *) echo "Aborting action as requested" >&2
       bailout 1
       ;;
  esac
  unset a
}

add_source_repos() {
  local _tmpfile
  _tmpfile=$(mktemp -t ngcp-approx-cache-deb-src-XXXXXXXXXX)

  if ! grep -qE "^deb " "${DEBSRCFILE}" >/dev/null 2>&1; then
    echo "Skipping update for apt source lists as no repositories are going to be updated"
    return
  fi

  echo "Add deb-src repositories for all deb records"
  echo "# Auto-generated deb-src ------------------------" >> "${DEBSRCFILE}"

  grep -E "^deb " "${DEBSRCFILE}" | sed -r "s/^deb (.*)/deb-src \1/g" >> "${_tmpfile}"
  cat "${_tmpfile}" >> "${DEBSRCFILE}"

  rm -f "${_tmpfile}"
}

update_apt_metadata() {
  if ! "${update}" ; then
    echo "Something wrong, we shouldn't be here. Better safe then sorry, exiting" >&2
    bailout 1
  fi

  check_force_option
  if "${update_filter}" ; then
    echo -e "\nUpdating APT cache for requested PPA repositories ONLY (in '${TMPDIR}'):"
  else
    echo -e "\nUpdating APT cache repositories meta data (in '${TMPDIR}'):"
  fi

  add_debian_repository "${RO_PORT}" "${RW_PORT}"
  add_ngcp_repository "${RO_PORT}" "${RW_PORT}"
  add_non_default_repository "${RO_PORT}" "${RW_PORT}"
  add_extra_debian_repository "${extra_debian_release}" "${RW_PORT}"
  add_extra_ngcp_repository "${extra_ngcp_release}" "${RW_PORT}" "${extra_debian_release}"
  apply_fake_mgmt_node_on_list

  # we always update 'deb-src' meta data in approx cache to be consistent with 'deb' meta data
  add_source_repos
}

read_apt_metadata() {
  echo -e "\nReading Grml/NGCP cache repositories meta data (in '${TMPDIR}'):"
  add_debian_repository "${RW_PORT}" "${RO_PORT}"
  add_ngcp_repository "${RW_PORT}" "${RO_PORT}"
  add_non_default_repository "${RW_PORT}" "${RO_PORT}"
  add_extra_debian_repository "${extra_debian_release}" "${RO_PORT}"
  add_extra_ngcp_repository "${extra_ngcp_release}" "${RO_PORT}" "${extra_debian_release}"
  apply_fake_mgmt_node_on_list
}

unify_sources_list_entries() {
  if ! [ -r "${DEBSRCFILE}" ] ; then
    echo "WARNING: sources.list file ${DEBSRCFILE} can't be read."
    return 0
  fi

  # avoid duplicate entries in sources.list:
  # 1) drop comment lines (to avoid ending up out of order)
  # 2) replace multiple spaces with single space char
  # 3) sort lines + report only unique entries
  grep -v '^#' "${DEBSRCFILE}" | tr -s '[:space:]' | sort -u > "${DEBSRCFILE}.filtered.tmp"
  mv "${DEBSRCFILE}.filtered.tmp" "${DEBSRCFILE}"
}

get_apt_metadata() {
  unify_sources_list_entries

  if "${update_filter}" ; then
    echo -e "\nGetting APT meta data for requested PPA repositories only:"
  else
    echo -e "\nGetting APT meta data:"
  fi

  if "${list}" ; then
    echo "Handle the following Debian repositories:"
    echo "-------------------------------------------"
    cat "${DEBSRCFILE}"
    echo "-------------------------------------------"
    echo
  fi

  apt_get "update" "${TMPOUTPUT}"

  if grep -Eq '^Err ' "${TMPOUTPUT}" ;  then
    echo "ERROR: failed to fetch repository files (see above), exiting." >&2
    bailout 1
  fi
}

apt_get() {
  local _opt="$1"
  local _output="$2"

  # shellcheck disable=SC2086
  DEBIAN_FRONTEND='noninteractive' NGCP_SHOW_HINT=false \
    apt-get -o dir::cache="${TMPDIR}/cachedir" \
    -o dir::state="${TMPDIR}/statedir" -o dir::etc::sourcelist="${DEBSRCFILE}" \
    -o dir::etc::sourceparts=/dev/null ${_opt} | tee "${_output}"
}

apt_cache() {
  local _opt="$1"
  local _output="$2"

  # shellcheck disable=SC2086
  DEBIAN_FRONTEND='noninteractive' apt-cache -o dir::cache="${TMPDIR}/cachedir" \
    -o dir::state="${TMPDIR}/statedir" -o dir::etc::sourcelist="${DEBSRCFILE}" \
    -o dir::etc::sourceparts=/dev/null ${_opt} | tee "${_output}"
}

sub_policy() {
  local _package="$1"

  echo -e "\nPackage status in approx cache:"
  apt_cache "policy ${_package}" "${TMPOUTPUT}"

  apt_get "-y -d --print-uris --reinstall install ${_package}" "${TMPOUTPUT}" >/dev/null
  echo -e "\nDownloading URL(s):"
  grep -E -o "${URL_REGEX}" "${TMPOUTPUT}"
}

unset_http_proxy() {
  if [ -n "${http_proxy:-}" ] || [ -n "${HTTP_PROXY:-}" ] ; then
    echo "HTTP proxy detected, skipping it for approx use case."
    export http_proxy=
    export HTTP_PROXY=
  fi

  if [ -n "${https_proxy:-}" ] || [ -n "${HTTPS_PROXY:-}" ] ; then
    echo "HTTPS proxy detected, skipping it for approx use case."
    export https_proxy=
    export HTTPS_PROXY=
  fi
}

check_apt_config() {
  if ! apt-config dump | grep -qi 'acquire::http::proxy' ; then
    return
  fi

  if "${force}" ; then
    echo "Ignoring apt option 'acquire::http::proxy' as requested via --force"
  else
    echo
    echo "WARNING: detected apt proxy option 'acquire::http::proxy',"
    echo "normally it means the script will fail as NGCP local approx cache"
    echo "cannot be accessed through remote proxy server."
    if "${noninteractive}" ; then
      echo "Aborting as requested via --non-interactive" >&2
      bailout 1
    fi
    echo -n "Should we continue here anyway (yes/NO): "
    read -r a
    case ${a,,} in
      yes|y) ;;
      *) echo "Aborting action as requested" >&2
         bailout 1
         ;;
    esac
    unset a
  fi
}

check_sipwise_connectivity() {
  local connection_issues=false
  local curl_http_code

  local urls=(
    http://debian.sipwise.com/debian/
    https://debian.sipwise.com/debian/
    http://deb.sipwise.com/debian/
    https://deb.sipwise.com/debian/
    http://deb.sipwise.com/sppro/
    https://deb.sipwise.com/sppro/
  )

  if ${check_local_approx} ; then
    if [[ -d "${APPROX_HOME}/sppro" ]]; then
      urls+=("http://${MGMT_NODE}:${RW_PORT}/sppro/")
    elif [[ -d "${APPROX_HOME}/autobuild" ]]; then
      urls+=("http://${MGMT_NODE}:${RW_PORT}/autobuild/")
    else
      # strange case here, at least 'sppro' should exist in approx cache
      # adding it to 'urls' to warn users if both 'sppro' and 'autobuild' are missing.
      urls+=("http://${MGMT_NODE}:${RW_PORT}/sppro/")
    fi
  fi

  for url in "${urls[@]}" ; do
    declare -a PROXY_OPTS
    if [ -n "${NGCP_HTTPS_PROXY:-}" ] && echo "${url}" | grep -q "^https:" ; then
      PROXY_OPTS=(--proxy "${NGCP_HTTPS_PROXY}")
    elif [ -n "${NGCP_HTTP_PROXY:-}" ] && echo "${url}" | grep -q "^http:" ; then
      PROXY_OPTS=(--proxy "${NGCP_HTTP_PROXY}")
    fi

    if echo "${url}" | grep -Eq "^http(s)?://${MGMT_NODE}" ; then
      # we do not need proxy to access local mgmt node
      PROXY_OPTS=()
    fi

    echo -n "Checking: curl ${PROXY_OPTS[*]} ${url} :"
    curl_http_code=$(curl "${PROXY_OPTS[@]}"  --head --connect-timeout 11 --silent --output /dev/null --write-out "%{http_code}" "${url}" || true)
    case "${curl_http_code:-000}" in
      200)
        echo " 200 OK."
        ;;
      403)
        echo " 403 Forbidden! Fix access first!"
        connection_issues=true
        ;;
      000)
        echo " cannot connect!"
        connection_issues=true
        ;;
      *)
        echo " Received ${curl_http_code} (expected 200)!"
        connection_issues=true
        ;;
    esac
  done

  if ${connection_issues} ; then
    echo "WARNING: connection issues were detected, see above."
    if "${force}" ; then
      echo "Ignoring as requested via --force"
      return 0
    fi
    if "${noninteractive}" ; then
      echo "Aborting as requested via --non-interactive" >&2
      bailout 1
    fi
    echo -n "Should we continue here anyway (yes/NO): "
    read -r a
    case ${a,,} in
      yes) ;;
      *) echo "Aborting action as requested" >&2
         bailout 1
         ;;
    esac
    unset a
  fi
}

validate_approx_cache_pre() {
  echo -e "\nInitial approx cache state validation..."
  echo "(it can take some time depending on approx cache size)"

  if [ ! -h "${APPROX_HOME}" ] ; then
    echo "ERROR: ${APPROX_HOME} is not a symlink" >&2
    bailout 1
  fi

  local tmp
  tmp=$(mktemp -t ngcp-approx-cache-pre-XXXXXXXXXX)
  get_empty_approx_files > "${tmp}"

  if [ "$(wc -l < "${tmp}")" != "0" ] ; then
    echo "WARNING: found corrupted files in approx cache which should be REMOVED:" >&2
    cat "${tmp}"

    if request_confirmation ; then
      echo "Removing corrupted files..."
      while read -r file ; do
        rm "${file}"
      done < "${tmp}"
    else
      echo "ERROR: approx cache contains corrupted files, cannot continue." >&2
      rm -f "${tmp}"
      bailout 1
    fi
  fi

  echo "Initial validation finished successfully."
  rm -f "${tmp}"
}

validate_approx_cache_post() {
  echo -e "\nFinal approx cache state validation..."
  echo "(it can take some time depending on approx cache size)"

  local tmp
  tmp=$(mktemp -t ngcp-approx-cache-post-XXXXXXXXXX)
  get_empty_approx_files > "${tmp}"

  if [ "$(wc -l < "${tmp}")" != "0" ] ; then
    echo "ERROR: found corrupted files in approx cache, please retrigger the script" >&2
    rm -f "${tmp}"
    bailout 1
  fi

  echo "Final validation finished successfully."
  rm -f "${tmp}"
}

get_empty_approx_files() {
  # ---------- 1 approx approx 0 May 22 09:32 /mnt/glusterfs/mgmt-share/approx/sppro/mr4.5.4/dists/jessie/main/binary-amd64/Packages

  local approx_path="/mnt/glusterfs/mgmt-share/approx/"

  if [ ! -d "${approx_path}" ] ; then
    echo "ERROR: missing folder ${approx_path}" >&2
    bailout 1
  fi

  find "${approx_path}" -type f -not -name "Translation-*" -size 0
}

request_confirmation() {
  if "${force}" ; then
    echo "Forcing action as requested via --force"
    return 0
  fi
  if "${noninteractive}" ; then
    echo "Aborting action as requested via --non-interactive"
    return 1
  fi

  # clearing STDIN
  while read -r -t 0; do read -r; done

  while true ; do
    echo -n "Should we continue here? (yes/no): "
    read -r a
    case "${a,,}" in
      yes)
        echo "Continue as requested."
        return 0
        ;;
      no)
        echo "Aborted as requested." >&2
        return 1
        ;;
      *)
        echo "Please answer 'yes' or 'no'."
        ;;
    esac
    unset a
  done

  # we shouldn't be here
  return 42
}

download_new_packages() {
  echo
  echo "Downloading new packages into central approx and local apt caches..."
  echo "Executing: apt-get --download-only dist-upgrade"
  apt_get "-y --download-only dist-upgrade" "${TMPOUTPUT}"
}

run_apt_update_localhost() {
  echo
  echo "Updating APT meta data on localhost ONLY."
  echo "(no need to call 'apt update' on this node)"
  NGCP_SHOW_HINT=false apt-get update 2>&1 | tee "${TMPOUTPUT}"

  if grep -Eq '^Err ' "${TMPOUTPUT}" ;  then
    echo "ERROR: failed to run 'apt-get update' on this node, exiting." >&2
    bailout 1
  fi
}

create_snapshot() {
  local snap_name
  snap_name="system-due-to-${ME}-$(TZ=UTC date +"%Y%m%d%H%M%S")"
  SNAP_NAME="${snap_name}" INVOKED_BY="${invoked_by}" ngcp-approx-snapshots --create
}

####################################################################################
# Lets the party begin!

get_options "$@"

if ( "${update}" && ! "${skip_apt_update}" ) || "${download}" ; then
  if [ "$(id -u 2>/dev/null)" != 0 ] ; then
    echo "Error: executing the script with any of the --auto/--download/--update* options requires root permissions." >&2
    bailout 1
  fi
fi

check_options

TMPDIR=$(mktemp -d -t ngcp-approx-cache-tmpdir-XXXXXXXXXX)
mkdir -p "${TMPDIR}/statedir/lists/partial" "${TMPDIR}/cachedir/archives/partial"
# avoid `W: [...] couldn't be accessed by user '_apt'. - pkgAcquire::Run (13: Permission denied)`
chown _apt:root "${TMPDIR}" "${TMPDIR}/statedir/" "${TMPDIR}/cachedir/"

DEBSRCFILE=$(mktemp -t ngcp-approx-cache-debsrc-XXXXXXXXXX)
TMPOUTPUT=$(mktemp -t ngcp-approx-cache-tmpout-XXXXXXXXXX)
TMPCACHEOUTPUT=$(mktemp -t ngcp-approx-cache-tmpcache-XXXXXXXXXX)

unset_http_proxy
check_apt_config

if "${check}" ; then
  check_sipwise_connectivity
fi

if "${clean_check}" ; then
  check_clean_unnecessary_cache
fi

if "${validate}" ; then
  validate_approx_cache_pre
fi

if "${clean}" ; then
  clean_unnecessary_repos
  clean_unnecessary_cached_debs
fi

if "${update}" ; then
  create_snapshot
  # prepare apt config from approx (port 9999)
  update_apt_metadata
fi

if "${policy}" || "${download}" ; then
  read_apt_metadata
fi

if "${update}" || "${policy}" || "${download}" ; then
  # downloading new APT meta data
  get_apt_metadata "$@"
  get_upgrade_files
fi

if "${policy}" ; then
  sub_policy "${package}"
elif "${download}" ; then
  # Download new packages (to be installed) in approx and local apt caches
  download_new_packages
fi

if "${update}" && "${validate}" ; then
  validate_approx_cache_post
fi

if "${update}" && ! "${skip_apt_update}"; then
  run_apt_update_localhost
fi

echo -e "\nThe job successfully finished \o/"
bailout 0
