#!/usr/bin/perl
#----------------------------------------------------------------------
# Synchronizes database from a remote (master) host to the current one
#----------------------------------------------------------------------
use strict;
use warnings;
use English;
use DBI qw(:sql_types);
use File::Path qw(mkpath);
use Data::Dumper;
use Getopt::Long;
use POSIX qw(SIGTERM strftime);
use Capture::Tiny qw(capture);
use IPC::System::Simple qw(capturex);
use YAML::XS;
use IO::Handle;
#----------------------------------------------------------------------
use Readonly;
Readonly my $MYSQL_CREDENTIALS => '/etc/mysql/sipwise_extra.cnf';
Readonly my $CONSTANTS => YAML::XS::LoadFile('/etc/ngcp-config/constants.yml');
my $retcode     = 0;
local $SIG{INT} = \&interrupted;

END {
    cleanup();
    exit $retcode;
}

my $debug        = 0;
my $force        = 0;
my $sync_mode    = 'online';
my $backup_dir   = '/ngcp-data/backup/ngcp-sync-db';
my $ssh_tunnel   = '';
my $local_backup = undef;
my $keep_backups = undef;
my $repl_mode    = 'master-master';

my $master_host  = undef;
my $master_port  = 3306;
my $master_user  = undef;
my $master_pass  = undef;

my $local_host   = '127.0.0.1';
my $local_port   = 3306;
my $local_user   = undef;
my $local_pass   = undef;

my @databases    = (); # all
my @ignore_tbls  = (); # none

# for internal use
my $ssh_cmd      = undef;
my $master_dbh   = undef;
my $local_dbh    = undef;
my $multi_masters_dbh = undef;
my $slaves_dbh   = undef;
my $backup_file  = undef;
my $no_cleanup   = 0;
my $ssh_opts     = '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null';
my $ssh_host     = '127.0.0.1';

my $init_replication   = undef;
my $repl_connect_retry = 10;
my $ngcp_db_schema_dir = '/usr/share/ngcp-db-schema';
my $slave_check_delay  = 3; # delay after slave start to check for errors
                            # as the negotiation may take a second or two.
my $fix_db_schema      = undef;
my $set_local_grants   = undef;
my $use_central_db     = undef;
my $config_gtid        = undef;
my @set_multi_master;
my @set_fix_slaves;
my $db_sync_stage      = 0; # 0 - not started,
                            # 1 - started,
                            # 2 - completed w/o replication setup
                            # 3 - fully completed

my $ssh         = '/usr/bin/ssh';
my $gzip        = '/bin/gzip';
my $gunzip      = '/bin/gunzip';
my $mysql       = "/usr/bin/mysql";
my $mysqldump   = "/usr/bin/mysqldump";
my $ngcp_hostname = qx(ngcp-hostname);
   chomp($ngcp_hostname);

my $password_length = 20;

my $pair_name_regex = qr/^sp\d+$/;
my $restore_master_state = {};
my $restore_slave_state = {};
my $_gtid_slave_pos;

sub Usage {
    print <<USAGE;
==
    Synchronizes database from a remote (master) host to the current one
==
$PROGRAM_NAME [options]
Options:
    --help|-h|-?   -- this help
    --verbose|-v   -- verbose mode
    --force        -- force execution without a user confirmation
    --master-host  -- master database host (default: autodetected sp1 or sp2)
    --master-port  -- master database port (default: 3306)
    --master-user  -- master database username (default: sipwise, taken from $MYSQL_CREDENTIALS)
    --master-pass  -- master database password (default: taken from $MYSQL_CREDENTIALS)
    --local-host   -- local database host (default: 127.0.0.1)
    --local-port   -- local database port (default: 3306)
    --local-user   -- local database username (default: sipwise, taken from $MYSQL_CREDENTIALS)
    --local-pass   -- local database password (default: taken from $MYSQL_CREDENTIALS)
    --sync-mode    -- 'online|backup' online: performs synchronization on the fly,
                      creates a backup first and then restores the db from the backup
                      (default: online)
    --repl-mode    -- 'master-master|master-slave|multi-master' default: master-master
    --databases    -- default: all (specified as a list --databases db1 db2 db3 ...)
    --ignore-tables -- default: none (specified as a list --ignore-tables kamailio.acc_backup kamailio.acc_trash ...)
    --backup-dir   -- default '/ngcp-data/backup/ngcp-sync-db' (used only with '-sync-mode=backup')
    --local-backup -- create 'local' db backup before synchronization
    --keep-backups -- do not remove backups after completion
    --ssh-tunnel   -- enables ssh tunnel to the 'master-host' and 'master-port' on
                      the specified local port (to overcome localhost permissions on the
                      master host. you must specify a custom local port for that.
                      (e.g.: --ssh-tunnel=33125).
                      By default it is assumed that 'master' mysql is on 127.0.0.1
                      but it is on a different interface you can provide it
                      using the format: port:host (e.g.: --ssh-tunnel=33125:192.168.0.70)

    [ expert options ]
    --init-replication -- [this option is currently always in effect]
                          if there is no replication info on 'master' use this option to automatically
                          initialize it during the synchronization between 'master' and 'local'.
                          if used in conjunction with --repl-mode=master-slave and --use-central-db
                          replication is initialised using all nodes listed in
                          constants.yml:database.central.dbmaster as 'multi-master'
    --fix-db-schema    -- after successful import adjusts ngcp.db_schema
                          to set all "not_replicated" revisions valid for the current node
                          (automatically issued in repl-mode='multi-master')
    --set-local-grants -- set mysql.user root grants for the current hostname on 'local'
    --use-central-db   -- master host is automatically taken from
                          constants.yml:database.dbhost/dbport
    --config-gtid      -- reconfigures 'local' to use GTID based replication,
                          please note that gtid_domain_id must be configured in MariaDB
    --multi-masters    -- (requires --repl-mode=multi-master)
                          a comma separated list of master servers to attach the dbnode to,
                          (eg: sp1,rdb01a,rdb01b)
    --fix-slaves       -- in scenarios when there are slaves of the 'local', their gtid_slave_pos
                          need to be fixed or otherwise after sync is completed they will have broken
                          replication. this option expects a list of host:port combination (e.g: prx01a:3308,prx01b:3308)

USAGE
    exit 0;
}

GetOptions("h|?|help"      => \&Usage,
           "v|verbose"     => \$debug,
           "force"         => \$force,
           "master-host=s" => \$master_host,
           "master-port=i" => \$master_port,
           "master-user=s" => \$master_user,
           "master-pass=s" => \$master_pass,
           "local-host=s"  => \$local_host,
           "local-port=i"  => \$local_port,
           "local-user=s"  => \$local_user,
           "local-pass=s"  => \$local_pass,
           "sync-mode=s"   => \$sync_mode,
           "repl-mode=s"   => \$repl_mode,
           "backup-dir=s"  => \$backup_dir,
           "local-backup"  => \$local_backup,
           "keep-backups"  => \$keep_backups,
           "fix-db-schema" => \$fix_db_schema,
           "ssh-tunnel=s"  => \$ssh_tunnel,
           #"init-replication" => \$init_replication,
           "set-local-grants" => \$set_local_grants,
           "use-central-db"   => \$use_central_db,
           "databases=s{1,}"  => \@databases,
           "ignore-tables=s{1,}" => \@ignore_tbls,
           "config-gtid"         => \$config_gtid,
           "multi-masters=s"     => \&opt_handle_set_multi_master,
           "fix-slaves=s"        => \&opt_handle_fix_slaves,
          ) or do {
    $retcode = 1;
    die "$ERRNO\n";
};

if ($repl_mode eq 'multi-master' && !@set_multi_master) {
    die "repl-mode=multi-master mode requires 'masters' option to be specified with a comma separated list of the master servers\n";
}

#----------------------------------------------------------------------
sub get_mysql_credentials {
  return $CONSTANTS->{credentials}->{mysql}->{system}->{u};
}

sub opt_handle_set_multi_master {
    my ($name, $value) = @_;

    @set_multi_master = map { $_ =~ s/\s+//gr } split(/,/, $value);

    return;
}

sub opt_handle_fix_slaves {
    my ($name, $value) = @_;

    @set_fix_slaves = map { $_ =~ s/\s+//gr } split(/,/, $value);

    return;
}

sub get_nodename {
    my @lines = capturex( [ 0 ], "/usr/sbin/ngcp-nodename");
    my $l = shift @lines;
    chomp $l;
    return $l;
}

sub connect_db {
    my ($dbhost, $dbport, $mysql_user, $mysql_pass) = @_;

    my $dsn = "DBI:mysql:database=mysql;host=$dbhost;port=$dbport";
    if ( $mysql_user eq get_mysql_credentials() ) {
        $dsn .= ";mysql_read_default_file=$MYSQL_CREDENTIALS";
    }

    my $dbh = DBI->connect($dsn, $mysql_user, $mysql_pass,
                            { PrintError => 0, mysql_auto_reconnect => 1 })
        or die "Can't connect to MySQL: ". $DBI::errstr;
    logger("connected to $dbhost:$dbport as $mysql_user");
    return $dbh;
}

sub pwgen {
    my @list = ("a".."z",0..9,"A".."Z");
    my @randoms;
    for (1..$password_length) {
        push @randoms, $list[int(rand($#list))];
    }

    return join "", @randoms;
}

sub create_definers {
    my $from_dbs =
        @databases ? 'WHERE db IN ('. join(',', map { "\"$_\"" } @databases).')'
                   : '';
    my $definers = $master_dbh->selectall_arrayref(<<SQL);
SELECT DISTINCT definer FROM mysql.proc $from_dbs
SQL
    die "Cannot select definers from master: $DBI::errstr" if $DBI::err;

    foreach my $r (@{$definers}) {
        my ($user,$host) = split(/\@/, $r->[0]);
        next if $user eq 'root' and $host eq 'localhost';
        my $pass = '!'.pwgen();
        $local_dbh->do(<<SQL);
CREATE USER IF NOT EXISTS '$user'\@'$host' IDENTIFIED BY '$pass'
SQL
        die "Cannot create a definer: $DBI::errstr" if $DBI::err;

        my $grants = $master_dbh->selectall_arrayref(<<SQL);
SHOW GRANTS FOR '$user'\@'$host'
SQL
        die "Cannot select grants for user '$user': $DBI::errstr" if $DBI::err;

        foreach my $g (@{$grants}) {
            $local_dbh->do($g->[0]);
            die "Cannot grant privileges for a definer: $DBI::errstr" if $DBI::err;
        }
    }
    $local_dbh->do("FLUSH PRIVILEGES");
    die "Cannot flush privileges: $DBI::errstr" if $DBI::err;

    return;
}

sub create_ssh_tunnel {
    my $ssh_tunnel_args = '';
    if ($ssh_tunnel =~ /^\d+$/) {
        $ssh_tunnel_args = "$ssh_tunnel:$ssh_host:$master_port";
    } elsif ($ssh_tunnel =~ /^(\d+):([a-zA-Z0-9\.\-]+)$/) {
        $ssh_tunnel_args = "$ssh_tunnel:$master_port";
    } else {
        die "Invalid --ssh-tunnel option provided '$ssh_tunnel', please check the options syntax";
    }
    $ssh_cmd = "/usr/bin/ssh $ssh_opts -fNL $ssh_tunnel_args $master_host";
    if (my $pid = check_ssh_tunnel()) {
        $no_cleanup = 1;
        die "Error ssh tunnel on port $ssh_tunnel_args is already running with pid '$pid'";
    }
    logger("creating ssh tunnel $ssh_tunnel_args to $master_host");
    my ($rc, undef, $err) = run_cmd($ssh_cmd);
    if ($err) {
        $err = join(", ", split(/\r*\n/, $err));
        die "Error creating ssh tunnel: $err (errno: $rc)";
    }
    my $timeout = 10;
    my $cnt = 0;
    while (! (my $pid = check_ssh_tunnel())) {
        ++$cnt;
        die "Error checking ssh tunnel '$ssh_cmd'" if $cnt >= $timeout;
        sleep 1;
    }
    return;
}

sub check_ssh_tunnel {
    return unless $ssh_cmd;

    open(my $pgrep, "-|", "pgrep -x -f '$ssh_cmd'")
        || die "Cannot check ssh tunnel: $ERRNO";

    my $pid;
    while (<$pgrep>) {
        $pid = $_;
        chomp $pid;
        last;
    }

    return $pid;
}

sub get_master_node {
    my $mhost = get_nodename();

    if ($use_central_db) {
        $mhost = $CONSTANTS->{database}{central}{dbhost} || return;
    } elsif ($repl_mode eq 'multi-master') {
        $mhost = $master_host;
    } else {
        if ($mhost eq 'sp1') {
            $mhost = 'sp2';
        } elsif ($mhost eq 'sp2') {
            $mhost = 'sp1';
        } else {
            return;
        }
    }
    return $mhost;
}

sub get_slave_node {
    my $shost = get_nodename();

    if ($repl_mode eq 'multi-master') {
        $shost = $local_host;
    }
    return $shost;
}

sub get_master_master_host {
    my $master_name = shift // '';
    if ($repl_mode eq 'multi-master') {
        my $mm_name = $master_name =~ $pair_name_regex
                        ? $master_name eq 'sp1' ? 'sp2' : 'sp1'
                        : 'r'.$ngcp_hostname;
        chomp($mm_name);
        return $mm_name;
    }
    return $master_host // '';
}

sub get_replication_status {
    my $dbh = shift;
    my $type = shift || die "'master/slave' type should be specified";
    my $sname = shift;

    my @data = ();
    my $ch = $dbh->prepare($type eq 'master'
                            ? "show master status"
                            : defined $sname ? "show slave '$sname' status"
                                             : "show all slaves status")
        or die "Cannot prepare: ".$DBI::errstr;

    $ch->execute();
    if ($DBI::err) {
        $DBI::errstr !~ /There is no master connection/
            ? die "Cannot execute: ".$DBI::errstr
            : return \@data;
    }
    my $fields = $ch->{NAME};
    my $vals   = $ch->fetchall_arrayref();

    for (my $c=0;$c<=$#$vals;$c++) {
        for (my $i=0;$i<scalar @{$fields};$i++) {
            $data[$c]{$fields->[$i]} = $vals->[$c][$i];
        }
    }

    return \@data;
}

sub get_own_hostnames {
    my @own_hostnames = ($ngcp_hostname);

    if ($ngcp_hostname =~ /\d{2}(a|b)$/) {
        push @own_hostnames, $1 eq 'a' ? 'sp1' : 'sp2';
    }

    return \@own_hostnames;
}

sub check_slave_status {
    my $db = shift;
    my $state = shift;
    my $sname = shift;
    my $s_dbh = shift;

    die "Incorrect state specified for check_slave_status should be either 'started' or 'stopped'"
        unless $state =~ /^(started|stopped)$/;

    logger("checking '$db' slave status for state '$state'");
    my $dbh = $s_dbh ? $s_dbh : ($db eq 'master' ? $master_dbh : $local_dbh);
    my $running_state = $state eq 'started' ? 'Yes' : 'No';
    my $status = get_replication_status($dbh, 'slave', $sname);
    foreach my $slave (@{$status}) {
        if ($state eq 'stopped' && $slave->{Slave_IO_Running}) {
            unless ($slave->{Slave_IO_Running} eq $running_state &&
                    $slave->{Slave_SQL_Running} eq $running_state) {
                die sprintf "Running state does not match. Current: Slave_IO_Running: %s Slave_SQL_Running: %s",
                    @{$slave}{qw(Slave_IO_Running Slave_SQL_Running)};
            }
        }
        if ($state eq 'started' && $slave->{Last_SQL_Errno}) {
            die sprintf "There are slave errors on the '$db' db. Please check/fix them Errno: %d Error: %s", @{$slave}{qw(Last_SQL_Errno Last_SQL_Error)};
        }
    }
}

sub reset_slave_db {
    my @dbs = @_;
    foreach my $db (@dbs) {
        die "Unknown db specified $db (should be either 'master' or 'local')"
            unless $db =~ /^(master|local)$/;
        my $dbh = $db eq 'master' ? $master_dbh : $local_dbh;

        $dbh->do("STOP ALL SLAVES");
        die "Cannot stop slave(s) on '$db': ".$DBI::errstr if $DBI::err;
        check_slave_status($db, 'stopped');

        $dbh->do("RESET SLAVE ALL");
        die "Cannot reset slave(s) on '$db': ".$DBI::errstr if $DBI::err;

        if ($db eq 'master') {
            my $server_id = get_server_id($local_dbh);
            my $gtid_domain_id = get_gtid_domain_id($local_dbh);

            my $gtid_slave_pos = get_gtid_slave_pos($master_dbh);
            my @filtered_gtids;
            foreach my $gtid (split /,/, $gtid_slave_pos) {
                next if $gtid =~ /^${gtid_domain_id}-${server_id}-/;
                push @filtered_gtids, $gtid;
            }
            my $filtered_gtid_slave_pos = join(',', @filtered_gtids);
            logger("set gtid_slave_pos=$filtered_gtid_slave_pos on 'master'");
            set_gtid_slave_pos($master_dbh, $filtered_gtid_slave_pos);
        }
    }
    return;
}

sub multi_stop_slave_db {

    return unless $repl_mode eq 'multi-master';
    return unless %{$multi_masters_dbh};

    foreach my $mname (@set_multi_master) {
        my $m_dbh = $multi_masters_dbh->{$mname};

        my $mm_name = get_master_master_host();
        my $is_pair = $mname =~ $pair_name_regex;
        my $mc_name = $is_pair ? '' : $mm_name;

        logger("stop sibling slave '$mc_name' on 'master' $mname");
        $m_dbh->do("STOP SLAVE '$mc_name'");
        check_slave_status($mc_name, 'stopped', $mc_name, $m_dbh);
    }

    logger("stop all slaves on 'local'");
    $local_dbh->do("STOP ALL SLAVES");
    die "Cannot stop all slaves on 'local': ".$DBI::errstr if $DBI::err;
    check_slave_status('local', 'stopped');

    logger("reset all slaves on 'local'");
    $local_dbh->do("RESET SLAVE ALL");
    die "Cannot reset all slaves on 'local': ".$DBI::errstr if $DBI::err;

    set_gtid_slave_pos($local_dbh, '');

    return;
}

sub multi_reset_gtid_slave_pos {
    my $server_id = get_server_id($local_dbh);
    my $gtid_domain_id = get_gtid_domain_id($local_dbh);
    my $is_mhost_pair = $master_host =~ $pair_name_regex;

    foreach my $mname (@set_multi_master) {
        my $m_dbh = $multi_masters_dbh->{$mname};

        my $mm_name = get_master_master_host();
        my $is_pair = $mname =~ $pair_name_regex;
        my $mc_name = $is_pair ? '' : $mm_name;

        logger("stop all running slaves on 'master' $mname");

        my $status = get_replication_status($m_dbh, 'slave');
        foreach my $slave (@{$status}) {
            my $s_mc_name = $slave->{Connection_name} // '';
            my $s_mhost   = $slave->{Master_Host} // '';
            my $is_s_pair = $is_pair && $s_mhost =~ $pair_name_regex;

            next if $slave->{Slave_IO_Running} eq 'No' &&
                    $slave->{Slave_SQL_Running} eq 'No';

            my $skip_restore = $slave->{Slave_IO_Running} eq 'No' ||
                               $slave->{Slave_SQL_Running} eq 'No' ||
                               $is_s_pair ||
                               ($mm_name eq $s_mc_name && !$is_pair);

            logger("stop slave '$s_mc_name' on 'master' $mname");
            $m_dbh->do("STOP SLAVE '$s_mc_name'");
            check_slave_status($s_mc_name, 'stopped', $s_mc_name, $m_dbh);
            push @{$restore_master_state->{$mname}}, $s_mc_name unless $skip_restore;
        }

        logger("reset slave '$mc_name' on 'master' $mname");
        $m_dbh->do("RESET SLAVE '$mc_name'");

        my $gtid_slave_pos = get_gtid_slave_pos($m_dbh);
        my @filtered_gtids;
        foreach my $gtid (split /,/, $gtid_slave_pos) {
            next if $gtid =~ /^${gtid_domain_id}-${server_id}-/;
            push @filtered_gtids, $gtid;
        }
        my $filtered_gtid_slave_pos = join(',', @filtered_gtids);
        logger("set gtid_slave_pos=$filtered_gtid_slave_pos on 'master' $mname");
        set_gtid_slave_pos($m_dbh, $filtered_gtid_slave_pos);
    }

    return;
}

sub multi_restore_master_state {

    foreach my $mname (sort { $a cmp $b } keys %{$restore_master_state}) {
        my $m_dbh = $multi_masters_dbh->{$mname};
        foreach my $m_sname (@{$restore_master_state->{$mname}}) {
            logger("restore slave started state '$m_sname' on 'master' $mname");
            $m_dbh->do("START SLAVE '$m_sname'");
            check_slave_status($m_sname, 'started', $m_sname, $m_dbh);
        }
    }

    return;
}

sub multi_start_slave_db {
    my $mm_name = get_master_master_host();

    return unless $repl_mode eq 'multi-master';
    return unless %{$multi_masters_dbh};

    foreach my $mname (@set_multi_master) {
        my $m_dbh = $multi_masters_dbh->{$mname};

        my $mm_name = get_master_master_host();
        my $is_pair = $mname =~ $pair_name_regex;
        my $mc_name = $is_pair ? '' : $mm_name;

        logger("start sibling slave '$mc_name' on master $mname");
        $m_dbh->do("START SLAVE '$mc_name'");
        check_slave_status($mc_name, 'started', $mc_name, $m_dbh);
    }

    return;
}

sub get_slave_connection_name {
    my $dbh = shift;
    my $mhost = shift;

    my $status = get_replication_status($dbh, 'slave');
    foreach my $slave (@{$status}) {
        my $s_mc_name = $slave->{Connection_name} // '';
        my $s_mhost   = $slave->{Master_Host} // '';

        if ($s_mhost eq $mhost) {
            return $s_mc_name;
        }
    }

    die "Could not find slave by master host '$mhost'\n";
}

sub fix_slaves_stop_slave_db {

    return unless %{$slaves_dbh};

    foreach my $mname (@set_fix_slaves) {
        my ($mhost, $mport) = split(/:/, $mname);
        my $s_dbh = $slaves_dbh->{$mname};

        my $status = get_replication_status($s_dbh, 'slave');
        foreach my $slave (@{$status}) {
            my $s_mc_name = $slave->{Connection_name} // '';
            my $s_mhost   = $slave->{Master_Host} // '';

            next if $slave->{Slave_IO_Running} eq 'No' &&
                    $slave->{Slave_SQL_Running} eq 'No';

            next if $s_mhost ne $ngcp_hostname;

            logger("stop sibling slave '$s_mc_name' on '$mname'");
            $s_dbh->do("STOP SLAVE '$s_mc_name'");
            check_slave_status($s_mc_name, 'stopped', $s_mc_name, $s_dbh);
        }
    }

    return;
}

sub fix_slaves_adjust_start_slave_db {

    return unless %{$slaves_dbh};

    my $server_id = get_server_id($local_dbh);
    my $gtid_domain_id = get_gtid_domain_id($local_dbh);

    foreach my $mname (@set_fix_slaves) {
        my ($mhost, $mport) = split(/:/, $mname);
        my $s_dbh = $slaves_dbh->{$mname};

        my $sc_name = get_slave_connection_name($s_dbh, $ngcp_hostname);

        logger("stop all running slaves on '$mname'");

        my $status = get_replication_status($s_dbh, 'slave');
        foreach my $slave (@{$status}) {
            my $s_mc_name = $slave->{Connection_name} // '';
            my $s_mhost   = $slave->{Master_Host} // '';

            next if $slave->{Slave_IO_Running} eq 'No' &&
                    $slave->{Slave_SQL_Running} eq 'No';

            my $skip_restore = $slave->{Slave_IO_Running} eq 'No' ||
                               $slave->{Slave_SQL_Running} eq 'No' ||
                               $s_mhost eq $ngcp_hostname;

            logger("stop slave '$s_mc_name' on '$mname'");
            $s_dbh->do("STOP SLAVE '$s_mc_name'");
            check_slave_status($s_mc_name, 'stopped', $s_mc_name, $s_dbh);
            push @{$restore_slave_state->{$mname}}, $s_mc_name unless $skip_restore;
        }

        logger("reset sibling slave '$sc_name' on '$mname'");
        $s_dbh->do("RESET SLAVE '$sc_name'");

        my $gtid_slave_pos = get_gtid_slave_pos($s_dbh);
        my @filtered_gtids;
        foreach my $gtid (split /,/, $gtid_slave_pos) {
            next if $gtid =~ /^${gtid_domain_id}-${server_id}-/;
            push @filtered_gtids, $gtid;
        }
        my $filtered_gtid_slave_pos = join(',', @filtered_gtids);
        logger("set gtid_slave_pos=$filtered_gtid_slave_pos on '$mname'");
        set_gtid_slave_pos($s_dbh, $filtered_gtid_slave_pos);

        $s_dbh->do("CHANGE MASTER '$sc_name' TO MASTER_USE_GTID=slave_pos");
        die "Cannot change MASTER_USE_GTID=slave_pos on '$mname': ".$DBI::errstr if $DBI::err;

        logger("start sibling slave '$sc_name' on '$mname'");
        $s_dbh->do("START SLAVE '$sc_name'");
        check_slave_status($sc_name, 'started', $sc_name, $s_dbh);
    }

    foreach my $mname (sort { $a cmp $b } keys %{$restore_slave_state}) {
        my $s_dbh = $slaves_dbh->{$mname};
        foreach my $m_sname (@{$restore_slave_state->{$mname}}) {
            logger("restore slave started state '$m_sname' on '$mname'");
            $s_dbh->do("START SLAVE '$m_sname'");
            check_slave_status($m_sname, 'started', $m_sname, $s_dbh);
        }
    }

    return;
}

sub store_local_gtid_slave_pos {
    my $server_id = get_server_id($local_dbh);
    my $gtid_domain_id = get_gtid_domain_id($local_dbh);
    my $gtid_slave_pos = get_gtid_slave_pos($local_dbh);

    my @gtids = split(/,/, $gtid_slave_pos);
    my @filtered_gtids;
    foreach my $gtid (@gtids) {
        next if $gtid =~ /^${gtid_domain_id}-${server_id}-/;
        push @filtered_gtids, $gtid;
    }

    $_gtid_slave_pos = join(',', @filtered_gtids);
    logger("saving 'local' gtid_slave_pos (excluding own gtid) gtid=$_gtid_slave_pos");

    return;
}

sub get_mysql_datadir {
    my $dbh = shift;
    my $mysql_datadir = $dbh->selectrow_array('SELECT @@datadir');
    die "Cannot fetch datadir variable: ".$DBI::errstr if $DBI::err;
    die "Undefined mysql datadir" unless $mysql_datadir && $mysql_datadir =~ /^\//;
    return $mysql_datadir;
}

sub is_gtid_used {
    my $dbh = shift;

    my $status = get_replication_status($dbh, 'slave');

    foreach my $slave (@{$status}) {
        if ($slave->{Using_Gtid} && $slave->{Using_Gtid} =~ /_Pos/) {
            return 1;
        }
    }
    return 0;
}

sub get_server_id {
    my $dbh = shift;

    my $server_id = $dbh->selectrow_array('SELECT @@server_id');
    die "Cannot select server_id: ".$DBI::errstr if $DBI::err;

    return $server_id;
}

sub get_gtid_domain_id {
    my $dbh = shift;

    my $gtid_domain_id = $dbh->selectrow_array('SELECT @@gtid_domain_id');
    die "Cannot select gtid_domain_id: ".$DBI::errstr if $DBI::err;

    return $gtid_domain_id;
}

sub get_gtid_current_pos {
    my $dbh = shift;

    my $gtid_current_pos = $dbh->selectrow_array('SELECT @@gtid_current_pos');
    die "Cannot select gtid_current_pos: ".$DBI::errstr if $DBI::err;

    return $gtid_current_pos;
}

sub get_gtid_slave_pos {
    my $dbh = shift;

    my $gtid_slave_pos = $dbh->selectrow_array('SELECT @@gtid_slave_pos');
    die "Cannot select gtid_slave_pos: ".$DBI::errstr if $DBI::err;

    return $gtid_slave_pos;
}

sub set_gtid_slave_pos {
    my $dbh = shift;
    my $gtid_slave_pos = shift;

    $dbh->do('SET GLOBAL gtid_slave_pos = ?', undef, $gtid_slave_pos);
    die "Cannot set gtid_slave_pos: ".$DBI::errstr if $DBI::err;

    return;
}

sub config_gtid_mode {

    # check if GTID domains are the same
    my $master_gtid_domain_id = $master_dbh->selectrow_array('SELECT @@gtid_domain_id');
    my $local_gtid_domain_id = $local_dbh->selectrow_array('SELECT @@gtid_domain_id');
    unless (defined $master_gtid_domain_id && defined $local_gtid_domain_id &&
        $master_gtid_domain_id == $local_gtid_domain_id) {
            logger(sprintf "Skip GTID configuration as 'master' gtid_domain_id=%d does not match 'local' gtid_domain_id=%d",
                   $master_gtid_domain_id, $local_gtid_domain_id);
        return
    }

    my $db = 'local';

    $local_dbh->do("STOP ALL SLAVES");
    die "Cannot stop slave(s) on '$db': ".$DBI::errstr if $DBI::err;
    check_slave_status($db, 'stopped');

    if ($use_central_db && $repl_mode eq "master-slave") {
        adjust_replication_master_info();
    } else {
        my $gtid_slave_pos = $local_dbh->selectrow_array('SELECT @@gtid_slave_pos');
        die "Cannot select gtid_slave_pos: ".$DBI::errstr if $DBI::err;
        logger(sprintf "configuring '$db' to use gtid_slave_pos '%s', enabling GTID based replication", $gtid_slave_pos);
        $local_dbh->do('CHANGE MASTER TO MASTER_USE_GTID=slave_pos');
        die "Cannot enable MASTER_USE_GTID=slave_pos on '$db': ".$DBI::errstr if $DBI::err;
    }

    $local_dbh->do("START ALL SLAVES");
    die "Cannot start slave(s) on '$db': ".$DBI::errstr if $DBI::err;
    check_slave_status($db, 'started');

    logger('GTID based replication has been successfully enabled.');

    return;
}

sub adjust_local_system_user {
    my $sys_mnt_user = $CONSTANTS->{credentials}{mysql}{debian}{u};
    my $sys_mnt_pass = $CONSTANTS->{credentials}{mysql}{debian}{p};
    die "Error while setting up mysql replication, no sys_mnt_user or sys_mnt_pass"
        unless $sys_mnt_user && $sys_mnt_pass;

    my $sys_mnt_user_exists =
        int $local_dbh->selectrow_array(<<SQL, undef, $sys_mnt_user, 'localhost');
SELECT count(User) from mysql.user
WHERE User = ?
AND Host = ?
SQL
    die "Cannot select 'local' system user '$sys_mnt_user': ".$DBI::errstr if $DBI::err;

    if ($sys_mnt_user_exists) {
        logger("synchronizing 'local' system user '$sys_mnt_user'");
        # flush privileges are required to update freshly imported user
        $local_dbh->do("FLUSH PRIVILEGES");
        $local_dbh->do(<<SQL, undef, $sys_mnt_user, 'localhost', $sys_mnt_pass);
SET PASSWORD for ?@? = PASSWORD(?)
SQL
        die "Cannot update 'local' system user '$sys_mnt_user': ".$DBI::errstr if $DBI::err;
        $local_dbh->do("FLUSH PRIVILEGES");
        die "Cannot flush privileges: ".$DBI::errstr if $DBI::err;
    }

    return;
}

sub adjust_replication_master_info {
    #print Dumper($constants);
    my $repl_user    = $CONSTANTS->{credentials}{mysql}{replicator}{u};
    my $repl_pass    = $CONSTANTS->{credentials}{mysql}{replicator}{p};
    die "Error while setting up mysql replication, no replication_user or replication_pass"
        unless $repl_user && $repl_pass;

    adjust_local_system_user();

    foreach my $db (qw(master local)) {
        next if $db eq "master" and $repl_mode eq "master-slave";
        my $dbh  = $db eq 'master' ? $master_dbh : $local_dbh;
        my $host = $db eq 'master' ? get_nodename() : get_master_node();
        my $port = $db eq 'master' ? $local_port : $master_port;

        my $master_hosts = [$host];
        if ($use_central_db && $repl_mode eq "master-slave") {
            $master_hosts = $CONSTANTS->{database}{central}{dbmaster} // [];
        }

        my $named_master = 0;
        foreach my $mhost (@{$master_hosts}) {
            my $mname = $named_master ? $mhost : '';
            my $sname = $named_master ? 'slave_name='.$mname.',' : '';
            change_master_to($dbh, 'local', $mname, $mhost, $local_host, $port, $repl_user, $repl_pass, $repl_connect_retry);
            $named_master = 1;
        }
    }

    return;
}

sub adjust_replication_multi_master_info {
    my $repl_user    = $CONSTANTS->{credentials}{mysql}{replicator}{u};
    my $repl_pass    = $CONSTANTS->{credentials}{mysql}{replicator}{p};
    die "Error while setting up mysql replication, no replication_user or replication_pass"
        unless $repl_user && $repl_pass;

    adjust_local_system_user();

    multi_reset_gtid_slave_pos();

    my $mm_name = get_master_master_host();
    my $port  = $local_port;

    foreach my $mname (@set_multi_master) {
        my $dbh   = $multi_masters_dbh->{$mname};
        my $cname = $mm_name;
        my $rname = $mm_name;
        my $lname = $mname;

        my $is_pair = $mname =~ $pair_name_regex;
        if ($is_pair) { # paired connection handling for reverse master -> local
            $cname = '';
            $lname = '';
            $rname = $mname eq 'sp1' ? 'sp2' : 'sp1';
        }

        # reverse master -> local
        change_master_to($dbh, 'master', $cname, $rname, $mname, $port, $repl_user, $repl_pass, $repl_connect_retry);

        # local -> master
        change_master_to($local_dbh, 'local', $lname, $mname, $local_host, $port, $repl_user, $repl_pass, $repl_connect_retry);
    }

    multi_restore_master_state();

    return;
}

sub change_master_to {
    my ($dbh, $db, $mname, $mhost, $host, $port, $repl_user, $repl_pass, $repl_connect_retry) = @_;

    my $ch = $dbh->prepare(<<SQL);
CHANGE MASTER '$mname' TO
   MASTER_HOST=?,
   MASTER_PORT=?,
   MASTER_USER=?,
   MASTER_PASSWORD=?,
   MASTER_CONNECT_RETRY=?,
   MASTER_USE_GTID=slave_pos
SQL
    logger("adjusting '$db=$host' replication info to: slave_name=$mname,host=$mhost,port=$port,user=$repl_user,connect_retry=$repl_connect_retry,master_use_gtid=slave_pos");
    $ch->bind_param(1, $mhost);
    $ch->bind_param(2, $port, SQL_INTEGER);
    $ch->bind_param(3, $repl_user);
    $ch->bind_param(4, $repl_pass);
    $ch->bind_param(5, $repl_connect_retry, SQL_INTEGER);
    $ch->execute()
        or die "Cannot execute SQL statement: ".$DBI::errstr;
    $ch->finish;

    return;
}

sub fix_db_schema {
    my $local_node = get_nodename();
    die "Error while performing fix_db_schema, cannot get nodename" unless $local_node;
    logger("fixing ngcp.db_schema for '$local_node'");
    my $revisions = $local_dbh->selectall_arrayref(<<SQL, undef, $local_node, $local_node);
SELECT a.revision
  FROM ngcp.db_schema a
 WHERE a.node != ?
   AND a.revision NOT IN (
        SELECT b.revision FROM ngcp.db_schema b WHERE b.node = ?
       )
SQL
    die "Cannot select from ngcp.db_schema: ".$DBI::errstr if $DBI::err;
    if (scalar @{$revisions}) {
        my $cmd = 'find '.$ngcp_db_schema_dir.' -name "*not_replicated.up" | sed "s/.*\/\([0-9]*\)_not_replicated.*/\1/"';
        my ($rc, $out, $err) = run_cmd($cmd);
        if ($err) {
            die "Error while getting not_replicated revisions from '$ngcp_db_schema_dir': $err";
        }
        if ($out) {
            my @fix_revisions;
            my @not_replicated = split(/[\s\n]/, $out);
            foreach my $revision (map { $_->[0] } @{$revisions}) {
                if (grep { $_ == $revision } @not_replicated) {
                    push @fix_revisions, $local_node;
                    push @fix_revisions, $revision;
                }
            }
            if (scalar @fix_revisions) {
                my $records = (scalar @fix_revisions)/2;
                logger(
                    sprintf "adding %d 'not_replicated' records into ngcp.db_schema",
                            $records
                      );
                my $phs = join(',', ('(?,?)') x $records);
                # do not write binlogs for multi-master as that will break replication
                # on other nodes
                if ($repl_mode eq 'multi-master') {
                    $local_dbh->do(<<SQL, undef, @fix_revisions);
SET sql_log_bin=0;
SQL
                    die "Cannot set sql_log_bin=0: ".$DBI::errstr if $DBI::err;
                    $local_dbh->do(<<SQL, undef, @fix_revisions);
INSERT INTO ngcp.db_schema (node,revision) VALUES $phs;
SQL
                    die "Cannot insert into ngcp.db_schema: ".$DBI::errstr if $DBI::err;
                    $local_dbh->do(<<SQL, undef, @fix_revisions);
SET sql_log_bin=1;
SQL
                    die "Cannot set sql_log_bin=1: ".$DBI::errstr if $DBI::err;
                } else {
                    $local_dbh->do(<<SQL, undef, @fix_revisions);
INSERT INTO ngcp.db_schema (node,revision) VALUES $phs
SQL
                    die "Cannot insert into ngcp.db_schema: ".$DBI::errstr if $DBI::err;
                }
            }
        }
    }
    return;
}

sub run_cmd {
    my $cmd = shift;
    my $rc = 0;
    my ($out, $err) = capture {
        system($cmd);
        $rc = $CHILD_ERROR >> 8;
    };
    # error adjustments
    if ($err =~ 'Warning: Skipping the data of table mysql\.event') {
        $err = '';
    }
    if ($err =~ 'Warning: Permanently added') {
        $err = '';
    }

    return ($rc, $out, $err);
}

sub sync_db {
    # reset slaves
    if ($repl_mode eq 'master-master') {
        logger("stopping/resetting db slave on all hosts");
        reset_slave_db(qw(master local));
    } elsif ($repl_mode eq 'multi-master') {
        logger("stopping slave on all multi master hosts");
        multi_stop_slave_db();
    } else {
        logger("stopping/resetting db slave on local host");
        reset_slave_db(qw(local));
    }

    if (@set_fix_slaves) {
        logger("stopping slaves on ".join(',', @set_fix_slaves));
        fix_slaves_stop_slave_db();
    }

    # dump/restore db
    my $ssh_tunnel_port = $ssh_tunnel ? (split(/:/, $ssh_tunnel))[0] : '';
    my $host = $ssh_tunnel ? $ssh_host : $master_host;
    my $port = $ssh_tunnel ? $ssh_tunnel_port : $master_port;
    my $arg_master_pass = $master_pass ? "-p$master_pass" : "";
    my $arg_local_pass  = $local_pass  ? "-p$local_pass"  : "";
    my $mysqldump_cmd = $mysqldump;
    my $local_port_arg = $local_user eq 'root' && $local_host eq 'localhost'
                            ? '' # forced socket if user is root@localhost
                            : "-P $local_port";
    my $arg_dbs = @databases ? sprintf '--databases %s', join(' ', @databases)
                             : '--all-databases';
    my $arg_ign_tbls = @ignore_tbls ? join(' ', map { sprintf('--ignore-table=%s', $_) } @ignore_tbls)
                                        : '';

    # used to capute the full gtid_slave_pos on the master
    # it is important to have correct gtid_slave_pos of the other masters in the cluster
    my $arg_dump_slave = '';
    if ($repl_mode eq 'multi-master') {
        $arg_dump_slave .= "--dump-slave";
    }

    create_definers($master_dbh, $local_dbh);

    if ($local_backup) {
        $local_backup = $backup_dir.'/local-'.strftime('%Y%m%d%H%M%S', localtime).'.sql.gz';
        logger("dumping 'local' db into $local_backup");
        if ( $local_user eq get_mysql_credentials() ) {
            $mysqldump_cmd .= " --defaults-extra-file=$MYSQL_CREDENTIALS";
        }
        $mysqldump_cmd .= " -h $local_host $local_port_arg -u $local_user $arg_local_pass $arg_dbs $arg_ign_tbls $arg_dump_slave --master-data --gtid --single-transaction --quick --routines | $gzip > $local_backup";
        my ($rc, undef, $err) = run_cmd($mysqldump_cmd);
        if ($err) {
            $err = join(", ", split(/\r*\n/, $err));
            die "Error during db synchronization: $err (errno: $rc)";
        }
    }

    if ($sync_mode eq 'backup') {
        $backup_file = $backup_dir.'/master-'.strftime('%Y%m%d%H%M%S', localtime).'.sql.gz';
        logger("dumping db from 'master' into $backup_file");
        $mysqldump_cmd = $mysqldump;
        if ( $master_user eq get_mysql_credentials() ) {
          $mysqldump_cmd .= " --defaults-extra-file=$MYSQL_CREDENTIALS";
        }
        $mysqldump_cmd .= " -h $host -P $port -u $master_user $arg_master_pass $arg_dbs $arg_ign_tbls $arg_dump_slave --master-data --gtid --single-transaction --quick --routines | $gzip > $backup_file";
    } elsif ($sync_mode eq 'online') {
        logger("synchronizing db from 'master'");
        $mysqldump_cmd = $mysqldump;
        if ( $master_user eq get_mysql_credentials() ) {
          $mysqldump_cmd .= " --defaults-extra-file=$MYSQL_CREDENTIALS";
        }
        my $mysql_credential_cmd = $mysql;
        if ( $local_user eq get_mysql_credentials() ) {
          $mysql_credential_cmd .= " --defaults-extra-file=$MYSQL_CREDENTIALS";
        }
        $mysqldump_cmd .= " -h $host -P $port -u $master_user $arg_master_pass $arg_dbs $arg_ign_tbls $arg_dump_slave --master-data --gtid --single-transaction --quick --routines | $mysql_credential_cmd -u $local_user $arg_local_pass -h $local_host $local_port_arg";
        $db_sync_stage = 1;
    }
    my ($rc, undef, $err) = run_cmd($mysqldump_cmd);
    if ($err) {
        $err = join(", ", split(/\r*\n/, $err));
        die "Error during db synchronization: $err (errno: $rc)";
    } else {
        $db_sync_stage = 2;
    }

    if ($sync_mode eq 'backup' && -f $backup_file) {
        logger("'local' importing db from '$backup_file'");
        my $mysql_credential_cmd = $mysql;
        if ( $local_user eq get_mysql_credentials() ) {
          $mysql_credential_cmd .= " --defaults-extra-file=$MYSQL_CREDENTIALS";
        }
        $db_sync_stage = 1;
        ($rc, undef, $err) = run_cmd("$gunzip <  $backup_file | $mysql_credential_cmd -u $local_user $arg_local_pass -h $local_host $local_port_arg");
        if ($err) {
            $err = join(", ", split(/\r*\n/, $err));
            die "Error during db import: $err (errno: $rc)";
        } else {
            $db_sync_stage = 2;
        }
    }

    # after import we need to have gtid pos of all other masters as
    # change master to may reset it
    #store_local_gtid_slave_pos();

    if ($repl_mode eq 'master-master' || $repl_mode eq 'multi-master') {
        # reset master on 'local' to avoid replication of the imported data
        logger("resetting master on 'local'");
        $local_dbh->do("RESET MASTER");
        die "Cannot reset master on 'local': ".$DBI::errstr if $DBI::err;
    }

    # adjust "MASTER" host/user/pass on 'master' and 'local'
    # ('local' only in 'master-slave' repl-mode)
    $repl_mode eq 'multi-master' && @set_multi_master
        ? adjust_replication_multi_master_info()
        : adjust_replication_master_info();

    # restore saved earlier gtid_slave_pos so it points to the correct masters
    # pos (own gtid is excluded)
    #logger("restore saved gtid_slave_pos after import gtid=$_gtid_slave_pos");
    #set_gtid_slave_pos($local_dbh, $_gtid_slave_pos);

    # start local slaves
    logger("starting replication on 'local'");
    # after import the gtid_slave_pos is not persistent between restarts
    # therefore, implicitly re-set
    $local_dbh->do('SET GLOBAL gtid_slave_pos=@@gtid_slave_pos');
    die "Cannot set gtid_slave_pos: ".$DBI::errstr if $DBI::err;
    #
    $local_dbh->do("START ALL SLAVES");
    die "Cannot start slave: ".$DBI::errstr if $DBI::err;
    sleep $slave_check_delay;
    check_slave_status('local', 'started');

    if ($repl_mode eq 'master-master' || $repl_mode eq 'multi-master') {
        if ($repl_mode eq 'multi-master') {
            multi_start_slave_db();
        } else {
            logger("starting replication on 'master'");
            $master_dbh->do("START ALL SLAVES");
            die "Cannot start slave(s): ".$DBI::errstr if $DBI::err;
            sleep $slave_check_delay;
            check_slave_status('master', 'started');
        }

        if ($repl_mode eq 'multi-master' || $fix_db_schema) {
            fix_db_schema();
        }
    }

    if ($set_local_grants) {
        logger("setting grants to 'root'\@'$ngcp_hostname'");
        $local_dbh->do("GRANT ALL PRIVILEGES ON *.* TO 'root'\@'$ngcp_hostname'");
        die "Cannot grant local hostname privileges: ".$DBI::errstr if $DBI::err;
        $local_dbh->do("FLUSH PRIVILEGES");
        die "Cannot flush privileges: ".$DBI::errstr if $DBI::err;
    }

    if (@set_fix_slaves) {
        fix_slaves_adjust_start_slave_db();
    }

    $db_sync_stage = 3;

    logger('db synchronization has been succsefully completed.');

    return;
}

sub logger {
    my $str = shift;
    if ($str && $debug) {
        printf "[%s] --> %s\n", strftime('%Y-%m-%d %H:%M:%S', localtime), $str;
    }
    return;
}

sub init {
    $master_host ||= get_master_node();
    $master_user ||= get_mysql_credentials();
    $local_user  ||= get_mysql_credentials();

    if ($use_central_db) {
        $master_port = $CONSTANTS->{database}{central}{dbport};
        $local_host = $CONSTANTS->{database}{local}{dbhost};
        $local_port = $CONSTANTS->{database}{local}{dbport};
    }

    # if the databases list is provided as a single scalar
    if ($#databases == 0 && $databases[0] =~ /\s+/) {
        @databases = split(/\s+/, $databases[0]);
    }

    # ignore internal mysql databases
    my $non_empty_dbs = 0;
    for (my $i=$#databases; $i>=0; $i--) {
        my $db = $databases[$i];
        $non_empty_dbs = 1;
        if ($db eq 'information_schema' || $db eq 'performance_schema') {
            logger("ignore internal db '$db'");
            splice(@databases, $i,1);
        }
    }

    if ($non_empty_dbs && !@databases) {
        die "Provided databases cannot be used\n";
    }

    if ($#ignore_tbls == 0 && $ignore_tbls[0] =~ /\s+/) {
        @ignore_tbls = split(/\s+/, $ignore_tbls[0]);
    }

    my $dbs = @databases ? join(' ', @databases) : '--all';
    my $ign_tbls = @ignore_tbls ? join(' ', @ignore_tbls) : '';

    if ($config_gtid) {
        print <<EOF;
#--------------------------------------------------
Preparing to reconfigure the current DB to use GTID for replication

local_host=$local_host
local_port=$local_port
local_user=$local_user
#--------------------------------------------------
EOF
    } else {
        print <<EOF;
#--------------------------------------------------
Preparing to run DB sync with the following parameters
from 'master' db to 'local' db.

master_host=$master_host
master_port=$master_port
master_user=$master_user
local_host=$local_host
local_port=$local_port
local_user=$local_user
sync_mode=$sync_mode
repl_mode=$repl_mode
databases=$dbs
ignore=$ign_tbls
backup_dir=$backup_dir
ssh_tunnel=$ssh_tunnel
EOF

        if (@set_multi_master) {
            my $set_multi_master_str = join(',', @set_multi_master);
            print <<EOF;
multi-masters=$set_multi_master_str
EOF
        }

        if (@set_fix_slaves) {
            my $set_fix_slaves_str = join(',', @set_fix_slaves);
            print <<EOF;
fix-slaves=$set_fix_slaves_str
EOF
        }

        print <<EOF;
#--------------------------------------------------
EOF
        die "Wrong sync_mode '$sync_mode', can be either 'online' or 'backup'"
            if $sync_mode ne 'online' and $sync_mode ne 'backup';

        die "Wrong repl_mode '$repl_mode', can be either 'master-master', 'master-slave' or 'multi-master'"
            if $repl_mode ne 'master-master' and
               $repl_mode ne 'master-slave' and
               $repl_mode ne 'multi-master';

        die "multi master hosts cannot contain own hostname/pairname"
            if $repl_mode eq 'multi-master' and
               join '', map { my $c = $_; map { $c eq $_ } @set_multi_master } @{get_own_hostnames()};

        die "master host cannot be own hostname/pairname"
            if join '', map { $master_host eq $_ } @{get_own_hostnames()};

        if ($sync_mode eq 'backup') {
            die "backup_dir is not set" unless $backup_dir;
            die "backup_dir cannot be a file" if -f $backup_dir;
            die "backup_dir must be absolute path and not pointing at '/'"
                if (-l $backup_dir && readlink($backup_dir) eq '/')
                    || $backup_dir !~ /^\/.+$/;
            unless (-d $backup_dir) {
                mkpath($backup_dir) ||
                    die "Cannot create backip_dir $backup_dir: $ERRNO";
            }
        }
    }

    if ($force) {
        logger("Force mode is on, auto 'yes'");
    } else {
        print "Type 'YES' to continue, anything else to abort... ";
        my $input = '';
        if (my $io = IO::Handle->new_from_fd(fileno(STDIN),"r")) {
            $input = $io->getline;
            chomp $input;
            $io->close;
        } else {
            die "Error while accessing STDIN: $!";
        }
        if ($input ne 'YES') {
            print "Aborted.\n";
            exit 1;
        }
    }

    if ($ssh_tunnel) {
        create_ssh_tunnel();
        my $ssh_tunnel_port = (split(/:/, $ssh_tunnel))[0];
        $master_dbh = connect_db($ssh_host, $ssh_tunnel_port, $master_user, $master_pass);
    } else {
        $master_dbh = connect_db($master_host, $master_port, $master_user, $master_pass);
    }

    if ($repl_mode eq 'multi-master' && @set_multi_master) {
        foreach my $mname (@set_multi_master) {
            $multi_masters_dbh->{$mname} = connect_db($mname, $master_port, $master_user, $master_pass);
        }
    }

    foreach my $mname (@set_fix_slaves) {
        my ($shost, $sport) = split(/:/, $mname);
        $slaves_dbh->{$mname} = connect_db($shost, $sport, $master_user, $master_pass);
    }

    $local_dbh  = connect_db($local_host, $local_port, $local_user, $local_pass);

    return;
}

sub cleanup {
    return if $no_cleanup;
    if (my $pid = check_ssh_tunnel()) {
        logger("terminating ssh tunnel with pid $pid");
        kill SIGTERM, $pid;
    }
    unless ($keep_backups) {
        if ($backup_file && -f $backup_file) {
            logger("removing db backup file '$backup_file'");
            unlink($backup_file) || die "Cannot delete backup file: $backup_file";
        }
        if ($local_backup && -f $local_backup) {
            logger("removing db backup file '$local_backup'");
            unlink($local_backup) || die "Cannot delete backup file: $local_backup";
        }
    }
    return;
}

sub interrupted {
    print logger("interrupted");
    $retcode = 1;
    exit $retcode;
    return;
}

sub main {
    eval {
        init();
        $config_gtid ? config_gtid_mode() : sync_db();
    };
    my $err = $@ // '';
    $retcode = $err ? 1 : 0;
    logger("there were errors during db synchronization:\nERROR: $err") if $err;
    if ($db_sync_stage == 1) {
        logger("ERROR: !!! Database on 'local' is broken due to unfinished import !!!");
    } elsif ($db_sync_stage == 2) {
        logger("ERROR: Database import on 'local' is completed but replication setup has failed.");
    }
    return;
}
#----------------------------------------------------------------------
main();

exit $retcode;
# vim: ts=4 sw=4 et

__END__

=pod

=head1 NAME

ngcp-sync-db - synchronizes database from a remote (master) host to the current one

=head1 SYNOPSIS

B<ngcp-sync-db> [I<options>...]

=head1 DESCRIPTION

B<This program> synchronizes database from a master host to current one.
The local database is destroyed during the process.

=head1 OPTIONS

=over 8

=item B<--force>

Force execution without a confirmation prompt.

=item B<--master-host> I<host>

Master database host (default: autodetected sp1 or sp2)

=item B<--master-port> I<port>

Master database port (default: 3306)

=item B<--master-user> I<user>

Master database username (default: sipwise, taken from /etc/mysql/sipwise_extra.cnf)

=item B<--master-pass> I<pass>

Master database password (default: taken from /etc/mysql/sipwise_extra.cnf)

=item B<--local-host> I<host>

Local database host (default: 127.0.0.1)

=item B<--local-port> I<port>

Local database port (default: 3306)

=item B<--local-user> I<user>

Local database username (default: sipwise, taken from /etc/mysql/sipwise_extra.cnf)

=item B<--local-pass> I<pass>

Local database password (default: taken from /etc/mysql/sipwise_extra.cnf)

=item B<--sync-mode> I<mode>

'online|backup' online: performs synchronization on the fly, creates a
backup first and then restores the db from the backup (default: online).

=item B<--backup-dir> I<dir>

Default '/ngcp-data/backup/ngcp-sync-db' (used only in conjunction with
B<--sync-mode=backup>).

=item B<--local-backup>

Create 'local' db backup before synchronization.

=item B<--keep-backups>

Do not remove created backups after completion.

=item B<--ssh-tunnel> I<tunnel>

Creates an ssh tunnel to the 'master-host' and 'master-port' on
the specified local port (useful when there is no remote mysql access to the master host).
You must specify a custom local port for that. (e.g.: --ssh-tunnel=33125)
By default it is assumed that 'master' mysql is on 127.0.0.1
but it is on a different interface you can provide it
using the format: port:host (e.g.: --ssh-tunnel=33125:192.168.0.70)

=item B<--init-replication>

If there is no replication info on 'master' use this option to automatically
initialize it during the synchronization between 'master' and 'local'

=item B<--fix-db-schema>

After successful import adjusts ngcp.db_schema
to set all "not_replicated" revisions valid for the current node.

=item B<--set-local-grants>

Set mysql.user root local grants for the current hostname on 'local'.

=item B<--verbose>

Verbose mode with step by step tracing to STDOUT.

=item B<--help>, B<-h>, B<-?>

Prin this help message.

=back

=head1 EXIT STATUS

=over 8

=item B<exit code 0>

Everything is ok

=item B<exit code != 0>

Something is wrong, an error message raises

=back

=head1 DIAGNOSTICS

TODO

=head1 BUGS AND LIMITATIONS

Please report problems you notice to the Sipwise Development Team <support@sipwise.com>.

=head1 AUTHOR

Kirill Solomko <ksolomko@sipwise.com>

=head1 LICENSE

GPL-3+, Sipwise GmbH, Austria

=cut
