From 681f840f6142f6450bc57bbeec09583d780043c7 Mon Sep 17 00:00:00 2001 From: Jehan-Guillaume de Rorthais Date: Sat, 7 Mar 2020 00:59:23 +0100 Subject: [PATCH] Vagrant files to build a 2nodes+qdevice cluster --- extra/vagrant/2nodes-qdevice-vip/Makefile | 44 ++++++ extra/vagrant/2nodes-qdevice-vip/Vagrantfile | 126 ++++++++++++++++++ .../provision/cluster-common.bash | 91 +++++++++++++ .../2nodes-qdevice-vip/provision/cts.bash | 114 ++++++++++++++++ .../2nodes-qdevice-vip/provision/id_rsa | 27 ++++ .../2nodes-qdevice-vip/provision/id_rsa.pub | 1 + .../provision/log_sink.bash | 20 +++ .../provision/pacemaker.bash | 109 +++++++++++++++ .../provision/pgsql-primary.bash | 83 ++++++++++++ .../provision/pgsql-replicas.bash | 61 +++++++++ .../2nodes-qdevice-vip/provision/qdevice.bash | 20 +++ .../2nodes-qdevice-vip/provision/system.bash | 64 +++++++++ .../2nodes-qdevice-vip/vagrant.yml-dist | 15 +++ extra/vagrant/README.md | 4 +- 14 files changed, 777 insertions(+), 2 deletions(-) create mode 100644 extra/vagrant/2nodes-qdevice-vip/Makefile create mode 100644 extra/vagrant/2nodes-qdevice-vip/Vagrantfile create mode 100755 extra/vagrant/2nodes-qdevice-vip/provision/cluster-common.bash create mode 100755 extra/vagrant/2nodes-qdevice-vip/provision/cts.bash create mode 100644 extra/vagrant/2nodes-qdevice-vip/provision/id_rsa create mode 100644 extra/vagrant/2nodes-qdevice-vip/provision/id_rsa.pub create mode 100755 extra/vagrant/2nodes-qdevice-vip/provision/log_sink.bash create mode 100755 extra/vagrant/2nodes-qdevice-vip/provision/pacemaker.bash create mode 100755 extra/vagrant/2nodes-qdevice-vip/provision/pgsql-primary.bash create mode 100755 extra/vagrant/2nodes-qdevice-vip/provision/pgsql-replicas.bash create mode 100755 extra/vagrant/2nodes-qdevice-vip/provision/qdevice.bash create mode 100755 extra/vagrant/2nodes-qdevice-vip/provision/system.bash create mode 100644 extra/vagrant/2nodes-qdevice-vip/vagrant.yml-dist diff --git a/extra/vagrant/2nodes-qdevice-vip/Makefile b/extra/vagrant/2nodes-qdevice-vip/Makefile new file mode 100644 index 0000000..3328c9c --- /dev/null +++ b/extra/vagrant/2nodes-qdevice-vip/Makefile @@ -0,0 +1,44 @@ +export VAGRANT_BOX_UPDATE_CHECK_DISABLE=1 +export VAGRANT_CHECKPOINT_DISABLE=1 + +.PHONY: all create_vm pgsql-primary pgsql-replicas qdevice pacemaker cts prov clean validate + + +all: create_vm pgsql-replicas pacemaker + +create_vm: + vagrant up + +pgsql-replicas: pcmk-stop pgsql-primary + vagrant up --provision-with=pgsql-replicas + +qdevice: + vagrant up --provision-with=qdevice + +pacemaker: qdevice + vagrant up --provision-with=pacemaker + +pgsql-primary: pcmk-stop + vagrant up --provision-with=pgsql-primary + +prov: + vagrant up --provision + +clean: + vagrant destroy -f + +check: validate + +validate: + @vagrant validate + @if which shellcheck >/dev/null ;\ + then shellcheck provision/*bash ;\ + else echo "WARNING: shellcheck is not in PATH, not checking bash syntax" ;\ + fi + +cts: + vagrant up --provision-with=cts + +pcmk-stop: + vagrant ssh -c 'if [ -f "/etc/corosync/corosync.conf" ]; then sudo pcs cluster stop --all --wait; fi' + diff --git a/extra/vagrant/2nodes-qdevice-vip/Vagrantfile b/extra/vagrant/2nodes-qdevice-vip/Vagrantfile new file mode 100644 index 0000000..9a9eead --- /dev/null +++ b/extra/vagrant/2nodes-qdevice-vip/Vagrantfile @@ -0,0 +1,126 @@ +require 'ipaddr' +require 'yaml' + +#ENV['VAGRANT_NO_PARALLEL'] = 'yes' # uncomment to forbid parallel execution +ENV["LANG"] = "C" +ENV["LC_ALL"] = "C" + +boxname = 'centos/7' # vagrant box to use +pgver = '11' # pg version to use +hapass = 'hapass' # password for sys user hacluster +ssh_login = 'root' # ssh login to connect to the host when fencing a VM. + # put "./provision/id_rsa.pub" in your "~/.ssh/authorized_keys" +master_ip = '10.20.30.105' # vIP assigned to master +pg_nodes = 'srv1', 'srv2' # first will be primary +qd_node = 'qd' # name of the node receiving logs +log_node = 'log-sink' # name of the node receiving logs +vm_prefix = 'paf_2nqd' # VM prefix in libvrit +rhel_user = '' # RHEL user account +rhel_pass = '' # RHEL user account password + +if File.file?('vagrant.yml') and ( custom = YAML.load_file('vagrant.yml') ) + boxname = custom['boxname'] if custom.has_key?('boxname') + pgver = custom['pgver'] if custom.has_key?('pgver') + hapass = custom['hapass'] if custom.has_key?('hapass') + ssh_login = custom['ssh_login'] if custom.has_key?('ssh_login') + master_ip = custom['master_ip'] if custom.has_key?('master_ip') + pg_nodes = custom['pg_nodes'] if custom.has_key?('pg_nodes') + log_node = custom['log_node'] if custom.has_key?('log_node') + vm_prefix = custom['vm_prefix'] if custom.has_key?('vm_prefix') + rhel_user = custom['rhel_user'] if custom.has_key?('rhel_user') + rhel_pass = custom['rhel_pass'] if custom.has_key?('rhel_pass') +end + +Vagrant.configure(2) do |config| + + pgdata = "/var/lib/pgsql/#{pgver}/data" + next_ip = IPAddr.new(master_ip).succ + host_ip = (IPAddr.new(master_ip) & "255.255.255.0").succ.to_s + nodes_ips = {} + + ( pg_nodes + [ qd_node, log_node ] ).each do |node| + nodes_ips[node] = next_ip.to_s + next_ip = next_ip.succ + end + + # don't mind about insecure ssh key + config.ssh.insert_key = false + + # https://vagrantcloud.com/search. + config.vm.box = boxname + + # hardware and host settings + config.vm.provider 'libvirt' do |lv| + lv.cpus = 1 + lv.memory = 512 + lv.watchdog model: 'i6300esb' + lv.default_prefix = vm_prefix + lv.qemu_use_session = false + end + + # disable default share + config.vm.synced_folder ".", "/vagrant", disabled: true + + config.vm.synced_folder "../../..", "/vagrant", type: "rsync", + rsync__exclude: [ ".git/" ] + + # system setup for all nodes + (pg_nodes + [qd_node, log_node]).each do |node| + config.vm.define node do |conf| + conf.vm.network 'private_network', ip: nodes_ips[node] + conf.vm.provision 'system-setup', type: 'shell', path: 'provision/system.bash', + args: [ node, rhel_user, rhel_pass ] + nodes_ips.keys.map {|n| "#{n}=#{nodes_ips[n]}"}, + preserve_order: true + end + end + + # setup rsyslog to collect logs from other node on log-sink + config.vm.define log_node do |conf| + conf.vm.provision 'rsyslog-setup', type: 'shell', path: 'provision/log_sink.bash' + end + + # common postgresql+pacemaker installation and setup + pg_nodes.each do |node| + config.vm.define node do |conf| + conf.vm.provision 'cluster-common', type: 'shell', path: 'provision/cluster-common.bash', + args: [ pgver, hapass, master_ip ], + preserve_order: true + end + end + + # build primary pgsql instance + config.vm.define pg_nodes.first, primary:true do |conf| + conf.vm.provision 'pgsql-primary', type: 'shell', + path: 'provision/pgsql-primary.bash', + args: [ pgver, pgdata, master_ip, pg_nodes.first ], + run: 'never' + end + + # replicas setup. Use "vagrant up --provision-with=pgsql-replicas" + pg_nodes[1..-1].each do |node| + config.vm.define node do |conf| + conf.vm.provision 'pgsql-replicas', type: 'shell', path: 'provision/pgsql-replicas.bash', + args: [ pgver, pgdata, master_ip, node ], run: 'never' + end + end + + # cluster setup. Use "vagrant up --provision-with=qdevice" + config.vm.define qd_node do |conf| + conf.vm.provision 'qdevice', type: 'shell', path:'provision/qdevice.bash', + args: [ hapass ], + run: 'never' + end + + # cluster setup. Use "vagrant up --provision-with=pacemaker" + pg_nodes.each do |node| + config.vm.define node do |conf| + conf.vm.provision 'pacemaker', type: 'shell', path:'provision/pacemaker.bash', + args: [ pgver, hapass, master_ip, ssh_login, + vm_prefix, host_ip, pgdata, qd_node ] + pg_nodes, + run: 'never' + end + end + + # cluster test suite setup. Use "vagrant up --provision-with=cts" + config.vm.provision 'cts', type: 'shell', path: 'provision/cts.bash', run: 'never' +end diff --git a/extra/vagrant/2nodes-qdevice-vip/provision/cluster-common.bash b/extra/vagrant/2nodes-qdevice-vip/provision/cluster-common.bash new file mode 100755 index 0000000..79ef2fc --- /dev/null +++ b/extra/vagrant/2nodes-qdevice-vip/provision/cluster-common.bash @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +PGVER="$1" +HAPASS="$2" +MASTER_IP="$3" + +# shellcheck disable=SC1091 +source "/etc/os-release" +OS_ID="$ID" +OS_VER="$VERSION_ID" +YUM_INSTALL="yum install --nogpgcheck --quiet -y -e 0" + +# install required packages +if [ "$OS_ID" = "rhel" ]; then + # use yum instead of dnf for compatibility between EL 7 and 8 + yum-config-manager --enable "*highavailability-rpms" +fi + +if ! rpm --quiet -q "pgdg-redhat-repo"; then + if [ "${OS_VER:0:2}" = "8." ]; then + $YUM_INSTALL "https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm" + else + $YUM_INSTALL "https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm" + fi +fi + +# disable postgresql upstream module conflicting with pgdg packages in RHEL8 +if [ "${OS_VER:0:2}" = "8." ]; then + yum -qy module disable postgresql +fi + +PACKAGES=( + pacemaker corosync-qdevice pcs resource-agents fence-agents-virsh sbd perl-Module-Build + "postgresql${PGVER}" + "postgresql${PGVER}-server" + "postgresql${PGVER}-contrib" +) + +$YUM_INSTALL "${PACKAGES[@]}" + +# firewall setup +firewall-cmd --quiet --permanent --add-service=high-availability +firewall-cmd --quiet --permanent --add-service=postgresql +firewall-cmd --quiet --reload + +# cluster stuffs +systemctl --quiet --now enable pcsd +echo "${HAPASS}"|passwd --stdin hacluster > /dev/null 2>&1 +cp /etc/sysconfig/pacemaker /etc/sysconfig/pacemaker.dist +cat<<'EOF' > /etc/sysconfig/pacemaker +PCMK_debug=yes +PCMK_logpriority=debug +EOF + +# cleanup master ip everywhere +HAS_MASTER_IP=$(ip -o addr show to "${MASTER_IP}"|wc -l) + +if [ "$HAS_MASTER_IP" -gt 0 ]; then + DEV=$(ip route show to "${MASTER_IP}/24"|grep -Eom1 'dev \w+') + ip addr del "${MASTER_IP}/24" dev "${DEV/dev }" +fi + +# send logs to log-sinks +cat <<'EOF' >/etc/rsyslog.d/fwd_log_sink.conf +*.* action(type="omfwd" +queue.type="LinkedList" +queue.filename="log_sink_fwd" +action.resumeRetryCount="-1" +queue.saveonshutdown="on" +target="log-sink" Port="514" Protocol="tcp") +EOF + +systemctl --quiet restart rsyslog + +# cleanup pre-existing IP address +ip -o addr show to "${MASTER_IP}" | if grep -q "${MASTER_IP}" +then + DEV=$(ip route show to "${MASTER_IP}/24"|grep -Eo 'dev \w+') + ip addr del "${MASTER_IP}/24" dev "${DEV/dev }" +fi + +# install PAF +cd /vagrant +[ -f Build ] && perl Build distclean +sudo -u vagrant perl Build.PL --quiet >/dev/null 2>&1 +sudo -u vagrant perl Build --quiet +perl Build --quiet install diff --git a/extra/vagrant/2nodes-qdevice-vip/provision/cts.bash b/extra/vagrant/2nodes-qdevice-vip/provision/cts.bash new file mode 100755 index 0000000..b55b98d --- /dev/null +++ b/extra/vagrant/2nodes-qdevice-vip/provision/cts.bash @@ -0,0 +1,114 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +# install packages + +PACKAGES=( + pacemaker-cts patch +) + +yum install --nogpgcheck --quiet -y -e 0 "${PACKAGES[@]}" + +# do not drop any log messages from rsyslog +cat <<'EOF'>/etc/rsyslog.d/rateLimit.conf +$imjournalRatelimitInterval 0 +$imjournalRatelimitBurst 0 +EOF + +systemctl --quiet restart rsyslog + +# make journald logs persistent +mkdir -p /var/log/journal + +# do not drop any log messages from journald +mkdir -p /etc/systemd/journald.conf.d +cat <<'EOF'>/etc/systemd/journald.conf.d/rateLimit.conf +RateLimitInterval=0 +RateLimitBurst=0 +EOF + +systemctl --quiet restart systemd-journald + + +# shellcheck disable=SC1091 +source "/etc/os-release" +OS_VER="$VERSION_ID" +if [ "${OS_VER:0:2}" != "7." ]; then exit; fi + +# fix bug in the log watcher for EL7 +cat <<'EOF' | patch /usr/lib64/python2.7/site-packages/cts/watcher.py +*** /tmp/watcher.py.orig 2019-02-07 16:25:32.836265277 +0100 +--- /tmp/watcher.py 2019-02-07 16:27:03.296926885 +0100 +*************** +*** 124,130 **** + self.offset = "EOF" + + if host == None: +! host = "localhost" + + def __str__(self): + if self.host: +--- 124,130 ---- + self.offset = "EOF" + + if host == None: +! self.host = "localhost" + + def __str__(self): + if self.host: +*************** +*** 155,179 **** + class FileObj(SearchObj): + def __init__(self, filename, host=None, name=None): + global has_log_watcher +! SearchObj.__init__(self, filename, host, name) +! +! if host is not None: +! if not host in has_log_watcher: + +! global log_watcher +! global log_watcher_bin + +! self.debug("Installing %s on %s" % (log_watcher_file, host)) + +! os.system("cat << END >> %s\n%s\nEND" %(log_watcher_file, log_watcher)) +! os.system("chmod 755 %s" %(log_watcher_file)) + +! self.rsh.cp(log_watcher_file, "root@%s:%s" % (host, log_watcher_bin)) +! has_log_watcher[host] = 1 + +! os.system("rm -f %s" %(log_watcher_file)) + +! self.harvest() + + def async_complete(self, pid, returncode, outLines, errLines): + for line in outLines: +--- 155,176 ---- + class FileObj(SearchObj): + def __init__(self, filename, host=None, name=None): + global has_log_watcher +! global log_watcher +! global log_watcher_bin + +! SearchObj.__init__(self, filename, host, name) + +! self.debug("Installing %s on %s" % (log_watcher_file, self.host)) + +! os.system("cat << END >> %s\n%s\nEND" %(log_watcher_file, log_watcher)) +! os.system("chmod 755 %s" %(log_watcher_file)) + +! self.rsh.cp(log_watcher_file, "root@%s:%s" % (self.host, log_watcher_bin)) +! has_log_watcher[self.host] = 1 + +! os.system("rm -f %s" %(log_watcher_file)) + +! self.harvest() + + def async_complete(self, pid, returncode, outLines, errLines): + for line in outLines: + +EOF + diff --git a/extra/vagrant/2nodes-qdevice-vip/provision/id_rsa b/extra/vagrant/2nodes-qdevice-vip/provision/id_rsa new file mode 100644 index 0000000..7d6a083 --- /dev/null +++ b/extra/vagrant/2nodes-qdevice-vip/provision/id_rsa @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzI +w+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoP +kcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2 +hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NO +Td0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcW +yLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQIBIwKCAQEA4iqWPJXtzZA68mKd +ELs4jJsdyky+ewdZeNds5tjcnHU5zUYE25K+ffJED9qUWICcLZDc81TGWjHyAqD1 +Bw7XpgUwFgeUJwUlzQurAv+/ySnxiwuaGJfhFM1CaQHzfXphgVml+fZUvnJUTvzf +TK2Lg6EdbUE9TarUlBf/xPfuEhMSlIE5keb/Zz3/LUlRg8yDqz5w+QWVJ4utnKnK +iqwZN0mwpwU7YSyJhlT4YV1F3n4YjLswM5wJs2oqm0jssQu/BT0tyEXNDYBLEF4A +sClaWuSJ2kjq7KhrrYXzagqhnSei9ODYFShJu8UWVec3Ihb5ZXlzO6vdNQ1J9Xsf +4m+2ywKBgQD6qFxx/Rv9CNN96l/4rb14HKirC2o/orApiHmHDsURs5rUKDx0f9iP +cXN7S1uePXuJRK/5hsubaOCx3Owd2u9gD6Oq0CsMkE4CUSiJcYrMANtx54cGH7Rk +EjFZxK8xAv1ldELEyxrFqkbE4BKd8QOt414qjvTGyAK+OLD3M2QdCQKBgQDtx8pN +CAxR7yhHbIWT1AH66+XWN8bXq7l3RO/ukeaci98JfkbkxURZhtxV/HHuvUhnPLdX +3TwygPBYZFNo4pzVEhzWoTtnEtrFueKxyc3+LjZpuo+mBlQ6ORtfgkr9gBVphXZG +YEzkCD3lVdl8L4cw9BVpKrJCs1c5taGjDgdInQKBgHm/fVvv96bJxc9x1tffXAcj +3OVdUN0UgXNCSaf/3A/phbeBQe9xS+3mpc4r6qvx+iy69mNBeNZ0xOitIjpjBo2+ +dBEjSBwLk5q5tJqHmy/jKMJL4n9ROlx93XS+njxgibTvU6Fp9w+NOFD/HvxB3Tcz +6+jJF85D5BNAG3DBMKBjAoGBAOAxZvgsKN+JuENXsST7F89Tck2iTcQIT8g5rwWC +P9Vt74yboe2kDT531w8+egz7nAmRBKNM751U/95P9t88EDacDI/Z2OwnuFQHCPDF +llYOUI+SpLJ6/vURRbHSnnn8a/XG+nzedGH5JGqEJNQsz+xT2axM0/W/CRknmGaJ +kda/AoGANWrLCz708y7VYgAtW2Uf1DPOIYMdvo6fxIB5i9ZfISgcJ/bbCUkFrhoH ++vq/5CIWxCPp0f85R4qxxQ5ihxJ0YDQT9Jpx4TMss4PSavPaBH3RXow5Ohe+bYoQ +NE5OgEXk2wVfZczCZpigBKbKZHNYcelXtTt/nP3rsCuGcM4h53s= +-----END RSA PRIVATE KEY----- diff --git a/extra/vagrant/2nodes-qdevice-vip/provision/id_rsa.pub b/extra/vagrant/2nodes-qdevice-vip/provision/id_rsa.pub new file mode 100644 index 0000000..18a9c00 --- /dev/null +++ b/extra/vagrant/2nodes-qdevice-vip/provision/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key diff --git a/extra/vagrant/2nodes-qdevice-vip/provision/log_sink.bash b/extra/vagrant/2nodes-qdevice-vip/provision/log_sink.bash new file mode 100755 index 0000000..ac45b07 --- /dev/null +++ b/extra/vagrant/2nodes-qdevice-vip/provision/log_sink.bash @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +# setup log sink +cat <<'EOF' > /etc/rsyslog.d/log_sink.conf +$ModLoad imtcp +$InputTCPServerRun 514 + +$template RemoteLogsMerged,"/var/log/%HOSTNAME%/messages.log" +*.* ?RemoteLogsMerged + +$template RemoteLogs,"/var/log/%HOSTNAME%/%PROGRAMNAME%.log" +*.* ?RemoteLogs +#& ~ +EOF + +systemctl --quiet restart rsyslog diff --git a/extra/vagrant/2nodes-qdevice-vip/provision/pacemaker.bash b/extra/vagrant/2nodes-qdevice-vip/provision/pacemaker.bash new file mode 100755 index 0000000..d2dbc8d --- /dev/null +++ b/extra/vagrant/2nodes-qdevice-vip/provision/pacemaker.bash @@ -0,0 +1,109 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +PGVER="$1" +HAPASS="$2" +MASTER_IP="$3" +SSH_LOGIN="$4" +VM_PREFIX="$5" +HOST_IP="$6" +PGDATA="$7" +QD_NODE="$8" +shift 8 +NODES=( "$@" ) + +CUSTOMDIR="${PGDATA}/conf.d" + +PCMK_VER=$(yum info --quiet pacemaker|grep ^Version) +PCMK_VER="${PCMK_VER#*: }" # extract x.y.z +PCMK_VER="${PCMK_VER:0:1}" # extract x + +if [ "$PCMK_VER" -ge 2 ]; then + # if pacemaker version is 2.x, we suppose pcs support it (pcs >= 0.10) + # from pcs 0.10, pcs host auth must be exec'ed on each node + pcs host auth -u hacluster -p "${HAPASS}" "${NODES[@]}" "$QD_NODE" +else + # this could be run on one node, but it doesn't hurt if it runs everywhere, + # so we keep this piece of code with the one dedicated to pacemaker 2.x + pcs cluster auth -u hacluster -p "${HAPASS}" "${NODES[@]}" "$QD_NODE" +fi + +if [ "$(hostname -s)" != "${NODES[0]}" ]; then + exit 0 +fi + +# WARNING: +# Starting from here, everything is executed on first node only! + +if [ "$PCMK_VER" -ge 2 ]; then + pcs cluster setup cluster_pgsql --force "${NODES[@]}" +else + pcs cluster setup --name cluster_pgsql --wait --force "${NODES[@]}" +fi + +pcs stonith sbd enable + +pcs quorum device add model net host=qd algorithm=ffsplit + +pcs cluster start --all --wait + +pcs cluster cib cluster1.xml + +pcs -f cluster1.xml resource defaults migration-threshold=5 +pcs -f cluster1.xml resource defaults resource-stickiness=10 +pcs -f cluster1.xml property set stonith-watchdog-timeout=10s + +# for VM in "${NODES[@]}"; do +# FENCE_ID="fence_vm_${VM}" +# VM_PORT="${VM_PREFIX}_${VM}" +# pcs -f cluster1.xml stonith create "${FENCE_ID}" fence_virsh \ +# pcmk_host_check=static-list "pcmk_host_list=${VM}" \ +# "port=${VM_PORT}" "ipaddr=${HOST_IP}" "login=${SSH_LOGIN}" \ +# "identity_file=/root/.ssh/id_rsa" +# pcs -f cluster1.xml constraint location "fence_vm_${VM}" \ +# avoids "${VM}=INFINITY" +# done + +PGSQLD_RSC_OPTS=( + "ocf:heartbeat:pgsqlms" + "bindir=/usr/pgsql-${PGVER}/bin" + "pgdata=${PGDATA}" + "recovery_template=${CUSTOMDIR}/recovery.conf.pcmk" + "op" "start" "timeout=60s" + "op" "stop" "timeout=60s" + "op" "promote" "timeout=30s" + "op" "demote" "timeout=120s" + "op" "monitor" "interval=15s" "timeout=10s" "role=Master" + "op" "monitor" "interval=16s" "timeout=10s" "role=Slave" + "op" "notify" "timeout=60s" +) + +# NB: pcs 0.10.2 doesn't support to set the id of the clone XML node +# the id is built from the rsc id to clone using "-clone" +# As a matter of cohesion and code simplicity, we use the same +# convention to create the master resource with pcs 0.9.x for +# Pacemaker 1.1 +if [ "$PCMK_VER" -ge 2 ]; then + PGSQLD_RSC_OPTS+=( "promotable" "notify=true" ) +fi + +pcs -f cluster1.xml resource create pgsqld "${PGSQLD_RSC_OPTS[@]}" + +if [ "$PCMK_VER" -eq 1 ]; then + pcs -f cluster1.xml resource master pgsqld-clone pgsqld notify=true +fi + +pcs -f cluster1.xml resource create pgsql-master-ip \ + "ocf:heartbeat:IPaddr2" "ip=${MASTER_IP}" cidr_netmask=24 \ + op monitor interval=10s + +pcs -f cluster1.xml constraint colocation add pgsql-master-ip with master pgsqld-clone INFINITY +pcs -f cluster1.xml constraint order promote pgsqld-clone "then" start pgsql-master-ip symmetrical=false +pcs -f cluster1.xml constraint order demote pgsqld-clone "then" stop pgsql-master-ip symmetrical=false + +pcs cluster cib-push scope=configuration cluster1.xml --wait + +crm_mon -Dn1 diff --git a/extra/vagrant/2nodes-qdevice-vip/provision/pgsql-primary.bash b/extra/vagrant/2nodes-qdevice-vip/provision/pgsql-primary.bash new file mode 100755 index 0000000..e7da79c --- /dev/null +++ b/extra/vagrant/2nodes-qdevice-vip/provision/pgsql-primary.bash @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +PGVER="$1" +PGDATA="$2" +MASTER_IP="$3" +NODENAME="$4" + +CUSTOMDIR="${PGDATA}/conf.d" + +# cleanup +systemctl --quiet --now disable "postgresql-${PGVER}" +rm -rf "${PGDATA}" + +# init instance +"/usr/pgsql-${PGVER}/bin/postgresql-${PGVER}-setup" initdb + +# pg_hba setup +cat< "${PGDATA}/pg_hba.conf" +local all all trust +host all all 0.0.0.0/0 trust + +# forbid self-replication +host replication postgres ${MASTER_IP}/32 reject +host replication postgres ${NODENAME} reject + +# allow any standby connection +host replication postgres 0.0.0.0/0 trust +EOC + +# postgresql.conf setup +mkdir -p "$CUSTOMDIR" +echo "include_dir = 'conf.d'" >> "${PGDATA}/postgresql.conf" + +cat <<'EOC' > "${CUSTOMDIR}/custom.conf" +listen_addresses = '*' +wal_level = replica +max_wal_senders = 10 +hot_standby = on +hot_standby_feedback = on +wal_keep_segments = 256 +log_destination = 'syslog,stderr' +log_checkpoints = on +log_min_duration_statement = 0 +log_autovacuum_min_duration = 0 +log_replication_commands = on +EOC + +if [ "${PGVER%%.*}" -lt 12 ]; then + # recovery.conf setup + cat<<-EOC > "${CUSTOMDIR}/recovery.conf.pcmk" + standby_mode = on + primary_conninfo = 'host=${MASTER_IP} application_name=${NODENAME}' + recovery_target_timeline = 'latest' + EOC +else + cat <<-EOC > "${CUSTOMDIR}/repli.conf" + primary_conninfo = 'host=${MASTER_IP} application_name=${NODENAME}' + EOC + + # standby_mode disappear in v12 + # no need to add recovery_target_timeline as its default is 'latest' since v12 +fi + +# backing up files +cp "${PGDATA}/pg_hba.conf" "${PGDATA}/.." +cp "${PGDATA}/postgresql.conf" "${PGDATA}/.." +cp "${CUSTOMDIR}"/* "${PGDATA}/.." + +chown -R postgres:postgres "$PGDATA" + +# create master ip +ip -o addr show to "${MASTER_IP}" | if ! grep -q "${MASTER_IP}" +then + DEV=$(ip route show to "${MASTER_IP}/24"|grep -Eo 'dev \w+') + ip addr add "${MASTER_IP}/24" dev "${DEV/dev }" +fi + +# restart master pgsql +systemctl --quiet start "postgresql-${PGVER}" diff --git a/extra/vagrant/2nodes-qdevice-vip/provision/pgsql-replicas.bash b/extra/vagrant/2nodes-qdevice-vip/provision/pgsql-replicas.bash new file mode 100755 index 0000000..6b74801 --- /dev/null +++ b/extra/vagrant/2nodes-qdevice-vip/provision/pgsql-replicas.bash @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +PGVER="$1" +PGDATA="$2" +MASTER_IP="$3" +NODENAME="$4" + +CUSTOMDIR="${PGDATA}/conf.d" + +# cleanup +systemctl --quiet --now disable "postgresql-${PGVER}" +rm -rf "${PGDATA}" + +# build standby +"/usr/pgsql-${PGVER}/bin/pg_basebackup" -h "${MASTER_IP}" -U postgres -D "${PGDATA}" -X stream + +# set pg_hba +cat< "${PGDATA}/pg_hba.conf" +local all all trust +host all all 0.0.0.0/0 trust + +# forbid self-replication +host replication postgres ${MASTER_IP}/32 reject +host replication postgres ${NODENAME} reject + +# allow any standby connection +host replication postgres 0.0.0.0/0 trust +EOC +cp "${PGDATA}/pg_hba.conf" "${PGDATA}/.." + +if [ "${PGVER%%.*}" -lt 12 ]; then + # recovery.conf setup + cat<<-EOC > "${CUSTOMDIR}/recovery.conf.pcmk" + standby_mode = on + primary_conninfo = 'host=${MASTER_IP} application_name=${NODENAME}' + recovery_target_timeline = 'latest' + EOC + cp "${CUSTOMDIR}/recovery.conf.pcmk" "${PGDATA}/recovery.conf" +else + cat <<-EOC > "${CUSTOMDIR}/repli.conf" + primary_conninfo = 'host=${MASTER_IP} application_name=${NODENAME}' + EOC + + # standby_mode disappear in v12 + # no need to add recovery_target_timeline as its default is 'latest' since v12 + touch "${PGDATA}/standby.signal" +fi + +# backing up files +cp "${PGDATA}/pg_hba.conf" "${PGDATA}/.." +cp "${PGDATA}/postgresql.conf" "${PGDATA}/.." +cp "${CUSTOMDIR}"/* "${PGDATA}/.." + +chown -R "postgres:postgres" "${PGDATA}/.." + +# start +systemctl --quiet start "postgresql-${PGVER}" diff --git a/extra/vagrant/2nodes-qdevice-vip/provision/qdevice.bash b/extra/vagrant/2nodes-qdevice-vip/provision/qdevice.bash new file mode 100755 index 0000000..6dbe6e0 --- /dev/null +++ b/extra/vagrant/2nodes-qdevice-vip/provision/qdevice.bash @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +YUM_INSTALL="yum install --nogpgcheck --quiet -y -e 0" + +$YUM_INSTALL pcs corosync-qnetd + +echo "$1"|passwd --stdin hacluster > /dev/null 2>&1 + +systemctl --quiet --now enable pcsd.service + +if ! pcs qdevice status net cluster_pgsql|grep -q cluster_pgsql; then + pcs qdevice setup model net --enable --start +fi + +firewall-cmd --quiet --permanent --add-service=high-availability +firewall-cmd --quiet --reload diff --git a/extra/vagrant/2nodes-qdevice-vip/provision/system.bash b/extra/vagrant/2nodes-qdevice-vip/provision/system.bash new file mode 100755 index 0000000..1cf5cbc --- /dev/null +++ b/extra/vagrant/2nodes-qdevice-vip/provision/system.bash @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +NODENAME="$1" +RHEL_USER="$2" +RHEL_PASS="$3" +shift +NODES=( "$@" ) + +hostnamectl set-hostname "${NODENAME}" + +for N in "${NODES[@]}"; do + NG=$(sed -n "/${N%=*}\$/p" /etc/hosts|wc -l) + if [ "$NG" -eq 0 ]; then + echo "${N##*=} ${N%=*}" >> /etc/hosts + fi +done + +# shellcheck disable=SC1091 +source "/etc/os-release" +OS_ID="$ID" + +PACKAGES=( vim bash-completion yum-utils ) + +if [ "$OS_ID" = "rhel" ]; then + subscription-manager register --force --username "${RHEL_USER:?}" --password "${RHEL_PASS:?}" --auto-attach + PACKAGES+=("tmux") +else + PACKAGES+=("screen") +fi + +yum install --nogpgcheck --quiet -y -e 0 "${PACKAGES[@]}" + +cat <<'EOF' > "/home/vagrant/.ssh/config" +Host * + CheckHostIP no + StrictHostKeyChecking no +EOF + +cp "/vagrant/extra/vagrant/2nodes-qdevice-vip/provision/id_rsa" "/home/vagrant/.ssh" +cp "/vagrant/extra/vagrant/2nodes-qdevice-vip/provision/id_rsa.pub" "/home/vagrant/.ssh" + +chown -R "vagrant:" "/home/vagrant/.ssh" +chmod 0700 "/home/vagrant/.ssh" +chmod 0600 "/home/vagrant/.ssh/id_rsa" +chmod 0644 "/home/vagrant/.ssh/id_rsa.pub" +chmod 0600 "/home/vagrant/.ssh/config" +chmod 0600 "/home/vagrant/.ssh/authorized_keys" + +cp -R "/home/vagrant/.ssh" "/root" + +# force proper permissions on .ssh files +chown -R "root:" "/root/.ssh" +chmod 0700 "/root/.ssh" +chmod 0600 "/root/.ssh/id_rsa" +chmod 0644 "/root/.ssh/id_rsa.pub" +chmod 0600 "/root/.ssh/config" +chmod 0600 "/root/.ssh/authorized_keys" + +# enable firewall +systemctl --quiet --now enable firewalld diff --git a/extra/vagrant/2nodes-qdevice-vip/vagrant.yml-dist b/extra/vagrant/2nodes-qdevice-vip/vagrant.yml-dist new file mode 100644 index 0000000..4de43fe --- /dev/null +++ b/extra/vagrant/2nodes-qdevice-vip/vagrant.yml-dist @@ -0,0 +1,15 @@ +# boxname: "centos/7" # vagrant box to use +# pgver: "10" # pg version to use +# hapass: "hapass" # password for sys user hacluster +# ssh_login: "user" # ssh login to connect to the host when fencing a VM. +# # put "./provision/id_rsa.pub" in your "~/.ssh/authorized_keys" +# master_ip: "10.20.30.5" # vIP assigned to master +# pg_nodes: # servers to create. +# - "srv1" # First one will be master +# - "srv2" +# - "srv3" +# qd_node: "qd" # QDevice node +# log_node: "log-sink" # log collector node +# vm_prefix: "paf_vm" +# rhel_user: "" # RHEL user account +# rhel_pass: "" # RHEL user account password diff --git a/extra/vagrant/README.md b/extra/vagrant/README.md index f56ef1d..80627cb 100644 --- a/extra/vagrant/README.md +++ b/extra/vagrant/README.md @@ -6,14 +6,14 @@ This `Vagrantfile` is bootstrapping a fresh cluster with: * pgsql primary is on `srv1` and the two standby are on `srv2` and `srv3` * server `log-sink` where all logs from `srv1`, `srv2` and `srv3` are collected under `/var/log/` * pacemaker stack is setup on `srv1`, `srv2` and `srv3` -* fencing using fence_virsh +* fencing using `fence_virsh` * watchdog enabled Note that NTP is enabled by default (using chrony) in the vagrant box used (`centos/7`). No need to set it up ourselves. This README takes `3nodes-vip` as example. Replace with the cluster name you -want. +want: `3nodes-vip`, `3nodes-haproxy` or `2nodes-qdevice-vip`. ## Prerequisites