Vagrant files to build a 2nodes+qdevice cluster

extra/vagrant/2nodes-qdevice-vip/Makefile

@@ -0,0 +1,44 @@
+export VAGRANT_BOX_UPDATE_CHECK_DISABLE=1
+export VAGRANT_CHECKPOINT_DISABLE=1
+
+.PHONY: all create_vm pgsql-primary pgsql-replicas qdevice pacemaker cts prov clean validate
+
+
+all: create_vm pgsql-replicas pacemaker
+
+create_vm:
+	vagrant up
+
+pgsql-replicas: pcmk-stop pgsql-primary
+	vagrant up --provision-with=pgsql-replicas
+
+qdevice:
+	vagrant up --provision-with=qdevice
+
+pacemaker: qdevice
+	vagrant up --provision-with=pacemaker
+
+pgsql-primary: pcmk-stop
+	vagrant up --provision-with=pgsql-primary
+
+prov:
+	vagrant up --provision
+
+clean:
+	vagrant destroy -f
+
+check: validate
+
+validate:
+	@vagrant validate
+	@if which shellcheck >/dev/null                                          ;\
+	then shellcheck provision/*bash                                          ;\
+	else echo "WARNING: shellcheck is not in PATH, not checking bash syntax" ;\
+	fi
+
+cts:
+	vagrant up --provision-with=cts
+
+pcmk-stop:
+	vagrant ssh -c 'if [ -f "/etc/corosync/corosync.conf" ]; then sudo pcs cluster stop --all --wait; fi'
+

extra/vagrant/2nodes-qdevice-vip/Vagrantfile

@@ -0,0 +1,126 @@
+require 'ipaddr'
+require 'yaml'
+
+#ENV['VAGRANT_NO_PARALLEL'] = 'yes' # uncomment to forbid parallel execution
+ENV["LANG"]           = "C"
+ENV["LC_ALL"]         = "C"
+
+boxname   = 'centos/7'     # vagrant box to use
+pgver     = '11'     # pg version to use
+hapass    = 'hapass' # password for sys user hacluster
+ssh_login = 'root' # ssh login to connect to the host when fencing a VM.
+                     # put "./provision/id_rsa.pub" in your "~<ssh_login>/.ssh/authorized_keys"
+master_ip = '10.20.30.105' # vIP assigned to master
+pg_nodes  = 'srv1', 'srv2' # first will be primary
+qd_node   = 'qd'       # name of the node receiving logs
+log_node  = 'log-sink' # name of the node receiving logs
+vm_prefix = 'paf_2nqd' # VM prefix in libvrit
+rhel_user = ''         # RHEL user account
+rhel_pass = ''         # RHEL user account password
+
+if File.file?('vagrant.yml') and ( custom = YAML.load_file('vagrant.yml') )
+    boxname   = custom['boxname']   if custom.has_key?('boxname')
+    pgver     = custom['pgver']     if custom.has_key?('pgver')
+    hapass    = custom['hapass']    if custom.has_key?('hapass')
+    ssh_login = custom['ssh_login'] if custom.has_key?('ssh_login')
+    master_ip = custom['master_ip'] if custom.has_key?('master_ip')
+    pg_nodes  = custom['pg_nodes']  if custom.has_key?('pg_nodes')
+    log_node  = custom['log_node']  if custom.has_key?('log_node')
+    vm_prefix = custom['vm_prefix'] if custom.has_key?('vm_prefix')
+    rhel_user = custom['rhel_user'] if custom.has_key?('rhel_user')
+    rhel_pass = custom['rhel_pass'] if custom.has_key?('rhel_pass')
+end
+
+Vagrant.configure(2) do |config|
+
+    pgdata     = "/var/lib/pgsql/#{pgver}/data"
+    next_ip    = IPAddr.new(master_ip).succ
+    host_ip    = (IPAddr.new(master_ip) & "255.255.255.0").succ.to_s
+    nodes_ips  = {}
+
+    ( pg_nodes + [ qd_node, log_node ] ).each do |node|
+        nodes_ips[node] = next_ip.to_s
+        next_ip = next_ip.succ
+    end
+
+    # don't mind about insecure ssh key
+    config.ssh.insert_key = false
+
+    # https://vagrantcloud.com/search.
+    config.vm.box = boxname
+
+    # hardware and host settings
+    config.vm.provider 'libvirt' do |lv|
+        lv.cpus = 1
+        lv.memory = 512
+        lv.watchdog model: 'i6300esb'
+        lv.default_prefix = vm_prefix
+        lv.qemu_use_session = false
+    end
+
+    # disable default share
+    config.vm.synced_folder ".", "/vagrant", disabled: true
+
+    config.vm.synced_folder "../../..", "/vagrant", type: "rsync",
+        rsync__exclude: [ ".git/" ]
+
+    # system setup for all nodes
+    (pg_nodes + [qd_node, log_node]).each do |node|
+        config.vm.define node do |conf|
+            conf.vm.network 'private_network', ip: nodes_ips[node]
+            conf.vm.provision 'system-setup', type: 'shell', path: 'provision/system.bash',
+                args: [ node, rhel_user, rhel_pass ] + nodes_ips.keys.map {|n| "#{n}=#{nodes_ips[n]}"},
+                preserve_order: true
+        end
+    end
+
+    # setup rsyslog to collect logs from other node on log-sink
+    config.vm.define log_node do |conf|
+        conf.vm.provision 'rsyslog-setup', type: 'shell', path: 'provision/log_sink.bash'
+    end
+
+    # common postgresql+pacemaker installation and setup
+    pg_nodes.each do |node|
+        config.vm.define node do |conf|
+            conf.vm.provision 'cluster-common', type: 'shell', path: 'provision/cluster-common.bash',
+                args: [ pgver, hapass, master_ip ],
+                preserve_order: true
+        end
+    end
+
+    # build primary pgsql instance
+    config.vm.define pg_nodes.first, primary:true do |conf|
+        conf.vm.provision 'pgsql-primary', type: 'shell',
+            path: 'provision/pgsql-primary.bash',
+            args: [ pgver, pgdata, master_ip, pg_nodes.first ],
+            run: 'never'
+    end
+
+    # replicas setup. Use "vagrant up --provision-with=pgsql-replicas"
+    pg_nodes[1..-1].each do |node|
+        config.vm.define node do |conf|
+            conf.vm.provision 'pgsql-replicas', type: 'shell', path: 'provision/pgsql-replicas.bash',
+                args: [ pgver, pgdata, master_ip, node ], run: 'never'
+        end
+    end
+
+    # cluster setup. Use "vagrant up --provision-with=qdevice"
+    config.vm.define qd_node do |conf|
+        conf.vm.provision 'qdevice', type: 'shell', path:'provision/qdevice.bash',
+            args: [ hapass ],
+            run: 'never'
+    end
+
+    # cluster setup. Use "vagrant up --provision-with=pacemaker"
+    pg_nodes.each do |node|
+        config.vm.define node do |conf|
+            conf.vm.provision 'pacemaker', type: 'shell', path:'provision/pacemaker.bash',
+                args: [ pgver, hapass, master_ip, ssh_login,
+                        vm_prefix, host_ip, pgdata, qd_node ] + pg_nodes,
+                run: 'never'
+        end
+    end
+
+    # cluster test suite setup. Use "vagrant up --provision-with=cts"
+    config.vm.provision 'cts', type: 'shell', path: 'provision/cts.bash', run: 'never'
+end

extra/vagrant/2nodes-qdevice-vip/provision/cluster-common.bash

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+PGVER="$1"
+HAPASS="$2"
+MASTER_IP="$3"
+
+# shellcheck disable=SC1091
+source "/etc/os-release"
+OS_ID="$ID"
+OS_VER="$VERSION_ID"
+YUM_INSTALL="yum install --nogpgcheck --quiet -y -e 0"
+
+# install required packages
+if [ "$OS_ID" = "rhel" ]; then
+    # use yum instead of dnf for compatibility between EL 7 and 8
+    yum-config-manager --enable "*highavailability-rpms"
+fi
+
+if ! rpm --quiet -q "pgdg-redhat-repo"; then
+    if [ "${OS_VER:0:2}" = "8." ]; then
+        $YUM_INSTALL "https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm"
+    else
+        $YUM_INSTALL "https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm"
+    fi
+fi
+
+# disable postgresql upstream module conflicting with pgdg packages in RHEL8
+if [ "${OS_VER:0:2}" = "8." ]; then
+    yum -qy module disable postgresql
+fi
+
+PACKAGES=(
+    pacemaker corosync-qdevice pcs resource-agents fence-agents-virsh sbd perl-Module-Build
+    "postgresql${PGVER}"
+    "postgresql${PGVER}-server"
+    "postgresql${PGVER}-contrib"
+)
+
+$YUM_INSTALL "${PACKAGES[@]}"
+
+# firewall setup
+firewall-cmd --quiet --permanent --add-service=high-availability
+firewall-cmd --quiet --permanent --add-service=postgresql
+firewall-cmd --quiet --reload
+
+# cluster stuffs
+systemctl --quiet --now enable pcsd
+echo "${HAPASS}"|passwd --stdin hacluster > /dev/null 2>&1
+cp /etc/sysconfig/pacemaker /etc/sysconfig/pacemaker.dist
+cat<<'EOF' > /etc/sysconfig/pacemaker
+PCMK_debug=yes
+PCMK_logpriority=debug
+EOF
+
+# cleanup master ip everywhere
+HAS_MASTER_IP=$(ip -o addr show to "${MASTER_IP}"|wc -l)
+
+if [ "$HAS_MASTER_IP" -gt 0 ]; then
+    DEV=$(ip route show to "${MASTER_IP}/24"|grep -Eom1 'dev \w+')
+    ip addr del "${MASTER_IP}/24" dev "${DEV/dev }"
+fi
+
+# send logs to log-sinks
+cat <<'EOF' >/etc/rsyslog.d/fwd_log_sink.conf
+*.* action(type="omfwd"
+queue.type="LinkedList"
+queue.filename="log_sink_fwd"
+action.resumeRetryCount="-1"
+queue.saveonshutdown="on"
+target="log-sink" Port="514" Protocol="tcp")
+EOF
+
+systemctl --quiet restart rsyslog
+
+# cleanup pre-existing IP address
+ip -o addr show to "${MASTER_IP}" | if grep -q "${MASTER_IP}"
+then
+    DEV=$(ip route show to "${MASTER_IP}/24"|grep -Eo 'dev \w+')
+    ip addr del "${MASTER_IP}/24" dev "${DEV/dev }"
+fi
+
+# install PAF
+cd /vagrant
+[ -f Build ] && perl Build distclean
+sudo -u vagrant perl Build.PL --quiet >/dev/null 2>&1
+sudo -u vagrant perl Build --quiet
+perl Build --quiet install

extra/vagrant/2nodes-qdevice-vip/provision/cts.bash

@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# install packages
+
+PACKAGES=(
+    pacemaker-cts patch
+)
+
+yum install --nogpgcheck --quiet -y -e 0 "${PACKAGES[@]}"
+
+# do not drop any log messages from rsyslog
+cat <<'EOF'>/etc/rsyslog.d/rateLimit.conf
+$imjournalRatelimitInterval 0
+$imjournalRatelimitBurst 0
+EOF
+
+systemctl --quiet restart rsyslog
+
+# make journald logs persistent
+mkdir -p /var/log/journal
+
+# do not drop any log messages from journald
+mkdir -p /etc/systemd/journald.conf.d
+cat <<'EOF'>/etc/systemd/journald.conf.d/rateLimit.conf
+RateLimitInterval=0
+RateLimitBurst=0
+EOF
+
+systemctl --quiet restart systemd-journald
+
+
+# shellcheck disable=SC1091
+source "/etc/os-release"
+OS_VER="$VERSION_ID"
+if [ "${OS_VER:0:2}" != "7." ]; then exit; fi
+
+# fix bug in the log watcher for EL7
+cat <<'EOF' | patch /usr/lib64/python2.7/site-packages/cts/watcher.py
+*** /tmp/watcher.py.orig	2019-02-07 16:25:32.836265277 +0100
+--- /tmp/watcher.py	2019-02-07 16:27:03.296926885 +0100
+***************
+*** 124,130 ****
+          self.offset = "EOF"
+  
+          if host == None:
+!             host = "localhost"
+  
+      def __str__(self):
+          if self.host:
+--- 124,130 ----
+          self.offset = "EOF"
+  
+          if host == None:
+!             self.host = "localhost"
+  
+      def __str__(self):
+          if self.host:
+***************
+*** 155,179 ****
+  class FileObj(SearchObj):
+      def __init__(self, filename, host=None, name=None):
+          global has_log_watcher
+!         SearchObj.__init__(self, filename, host, name)
+! 
+!         if host is not None:
+!             if not host in has_log_watcher:
+  
+!                 global log_watcher
+!                 global log_watcher_bin
+  
+!                 self.debug("Installing %s on %s" % (log_watcher_file, host))
+  
+!                 os.system("cat << END >> %s\n%s\nEND" %(log_watcher_file, log_watcher))
+!                 os.system("chmod 755 %s" %(log_watcher_file))
+  
+!                 self.rsh.cp(log_watcher_file, "root@%s:%s" % (host, log_watcher_bin))
+!                 has_log_watcher[host] = 1
+  
+!                 os.system("rm -f %s" %(log_watcher_file))
+  
+!             self.harvest()
+  
+      def async_complete(self, pid, returncode, outLines, errLines):
+          for line in outLines:
+--- 155,176 ----
+  class FileObj(SearchObj):
+      def __init__(self, filename, host=None, name=None):
+          global has_log_watcher
+!         global log_watcher
+!         global log_watcher_bin
+  
+!         SearchObj.__init__(self, filename, host, name)
+  
+!         self.debug("Installing %s on %s" % (log_watcher_file, self.host))
+  
+!         os.system("cat << END >> %s\n%s\nEND" %(log_watcher_file, log_watcher))
+!         os.system("chmod 755 %s" %(log_watcher_file))
+  
+!         self.rsh.cp(log_watcher_file, "root@%s:%s" % (self.host, log_watcher_bin))
+!         has_log_watcher[self.host] = 1
+  
+!         os.system("rm -f %s" %(log_watcher_file))
+  
+!         self.harvest()
+  
+      def async_complete(self, pid, returncode, outLines, errLines):
+          for line in outLines:
+
+EOF
+