From a07617e644d4a07527b16c7057a37be08caf9ec6 Mon Sep 17 00:00:00 2001 From: Jehan-Guillaume de Rorthais Date: Mon, 2 Dec 2019 00:15:16 +0100 Subject: [PATCH] Vagrant files to build a 3nodes+haproxy cluster --- extra/vagrant/3nodes-haproxy/Makefile | 44 +++++ extra/vagrant/3nodes-haproxy/Vagrantfile | 152 ++++++++++++++++++ .../3nodes-haproxy/provision/cluster.bash | 102 ++++++++++++ .../3nodes-haproxy}/provision/cts.bash | 0 .../3nodes-haproxy/provision/haproxy.bash | 78 +++++++++ .../3nodes-haproxy}/provision/id_rsa | 0 .../3nodes-haproxy}/provision/id_rsa.pub | 0 .../3nodes-haproxy/provision/pacemaker.bash | 46 ++++++ .../provision/pgsql-replicas.bash | 63 ++++++++ .../3nodes-haproxy/provision/postgresql.bash | 151 +++++++++++++++++ .../3nodes-haproxy/provision/rsyslog.bash | 51 ++++++ .../3nodes-haproxy/provision/system.bash | 65 ++++++++ extra/vagrant/3nodes-haproxy/vagrant.yml-dist | 14 ++ extra/{ => vagrant/3nodes-vip}/Makefile | 0 extra/{ => vagrant/3nodes-vip}/Vagrantfile | 0 .../3nodes-vip}/provision/cluster-common.bash | 0 extra/vagrant/3nodes-vip/provision/cts.bash | 114 +++++++++++++ extra/vagrant/3nodes-vip/provision/id_rsa | 27 ++++ extra/vagrant/3nodes-vip/provision/id_rsa.pub | 1 + .../3nodes-vip}/provision/log_sink.bash | 0 .../3nodes-vip}/provision/pacemaker.bash | 0 .../3nodes-vip}/provision/pgsql-primary.bash | 0 .../3nodes-vip}/provision/pgsql-replicas.bash | 0 .../3nodes-vip}/provision/system.bash | 0 .../{ => vagrant/3nodes-vip}/vagrant.yml-dist | 0 extra/{ => vagrant}/README.md | 8 +- 26 files changed, 913 insertions(+), 3 deletions(-) create mode 100644 extra/vagrant/3nodes-haproxy/Makefile create mode 100644 extra/vagrant/3nodes-haproxy/Vagrantfile create mode 100755 extra/vagrant/3nodes-haproxy/provision/cluster.bash rename extra/{ => vagrant/3nodes-haproxy}/provision/cts.bash (100%) create mode 100644 extra/vagrant/3nodes-haproxy/provision/haproxy.bash rename extra/{ => vagrant/3nodes-haproxy}/provision/id_rsa (100%) rename extra/{ => vagrant/3nodes-haproxy}/provision/id_rsa.pub (100%) create mode 100755 extra/vagrant/3nodes-haproxy/provision/pacemaker.bash create mode 100755 extra/vagrant/3nodes-haproxy/provision/pgsql-replicas.bash create mode 100755 extra/vagrant/3nodes-haproxy/provision/postgresql.bash create mode 100755 extra/vagrant/3nodes-haproxy/provision/rsyslog.bash create mode 100755 extra/vagrant/3nodes-haproxy/provision/system.bash create mode 100644 extra/vagrant/3nodes-haproxy/vagrant.yml-dist rename extra/{ => vagrant/3nodes-vip}/Makefile (100%) rename extra/{ => vagrant/3nodes-vip}/Vagrantfile (100%) rename extra/{ => vagrant/3nodes-vip}/provision/cluster-common.bash (100%) create mode 100755 extra/vagrant/3nodes-vip/provision/cts.bash create mode 100644 extra/vagrant/3nodes-vip/provision/id_rsa create mode 100644 extra/vagrant/3nodes-vip/provision/id_rsa.pub rename extra/{ => vagrant/3nodes-vip}/provision/log_sink.bash (100%) rename extra/{ => vagrant/3nodes-vip}/provision/pacemaker.bash (100%) rename extra/{ => vagrant/3nodes-vip}/provision/pgsql-primary.bash (100%) rename extra/{ => vagrant/3nodes-vip}/provision/pgsql-replicas.bash (100%) rename extra/{ => vagrant/3nodes-vip}/provision/system.bash (100%) rename extra/{ => vagrant/3nodes-vip}/vagrant.yml-dist (100%) rename extra/{ => vagrant}/README.md (94%) diff --git a/extra/vagrant/3nodes-haproxy/Makefile b/extra/vagrant/3nodes-haproxy/Makefile new file mode 100644 index 0000000..5b2a70c --- /dev/null +++ b/extra/vagrant/3nodes-haproxy/Makefile @@ -0,0 +1,44 @@ +export VAGRANT_BOX_UPDATE_CHECK_DISABLE=1 +export VAGRANT_CHECKPOINT_DISABLE=1 + +.PHONY: all create_vm postgresql pgsql_replicas pacemaker prov clean check validate cts pcmk-stop + + +all: create_vm postgresql pgsql_replicas pacemaker cluster + +create_vm: + vagrant up + +postgresql: pcmk-stop + vagrant up --provision-with=postgresql + +pgsql_replicas: pcmk-stop + vagrant up --provision-with=pgsql-replicas + +pacemaker: + vagrant up --provision-with=pacemaker + +cluster: + vagrant up --provision-with=cluster-setup + +prov: + vagrant up --provision + +clean: + vagrant destroy -f + +check: validate + +validate: + @vagrant validate + @if which shellcheck >/dev/null ;\ + then shellcheck provision/*bash ;\ + else echo "WARNING: shellcheck is not in PATH, not checking bash syntax" ;\ + fi + +cts: + vagrant up --provision-with=cts + +pcmk-stop: + vagrant ssh -c 'if [ -f "/etc/corosync/corosync.conf" ]; then sudo pcs cluster stop --all --wait; fi' + diff --git a/extra/vagrant/3nodes-haproxy/Vagrantfile b/extra/vagrant/3nodes-haproxy/Vagrantfile new file mode 100644 index 0000000..3d904c9 --- /dev/null +++ b/extra/vagrant/3nodes-haproxy/Vagrantfile @@ -0,0 +1,152 @@ +require 'ipaddr' +require 'yaml' + +#ENV['VAGRANT_NO_PARALLEL'] = 'yes' # uncomment to forbid parallel execution +ENV["LANG"] = "C" +ENV["LC_ALL"] = "C" + +boxname = 'centos/7' # vagrant box to use +pgver = '11' # pg version to use +hapass = 'hapass' # password for sys user hacluster +ssh_login = 'root' # ssh login to connect to the host when fencing a VM. + # put "./provision/id_rsa.pub" in your "~/.ssh/authorized_keys" +base_ip = '10.20.30.5' # Base IP address to compute other ones +pg_nodes = 'srv1', 'srv2', 'srv3' # first will be primary +log_node = 'log-sink' # name of the node receiving logs +vm_prefix = 'paf_vm' # VM prefix in libvrit +rhel_user = '' # RHEL user account +rhel_pass = '' # RHEL user account password + +if File.file?('vagrant.yml') and ( custom = YAML.load_file('vagrant.yml') ) + boxname = custom['boxname'] if custom.has_key?('boxname') + pgver = custom['pgver'] if custom.has_key?('pgver') + hapass = custom['hapass'] if custom.has_key?('hapass') + ssh_login = custom['ssh_login'] if custom.has_key?('ssh_login') + base_ip = custom['base_ip'] if custom.has_key?('base_ip') + pg_nodes = custom['pg_nodes'] if custom.has_key?('pg_nodes') + log_node = custom['log_node'] if custom.has_key?('log_node') + vm_prefix = custom['vm_prefix'] if custom.has_key?('vm_prefix') + rhel_user = custom['rhel_user'] if custom.has_key?('rhel_user') + rhel_pass = custom['rhel_pass'] if custom.has_key?('rhel_pass') +end + +Vagrant.configure(2) do |config| + + ############################################################################ + # computes variables + + pgdata = "/var/lib/pgsql/#{pgver}/data" + next_ip = IPAddr.new(base_ip).succ + host_ip = (IPAddr.new(base_ip) & "255.255.255.0").succ.to_s + nodes_ips = {} + + ( pg_nodes + [ log_node ] ).each do |node| + nodes_ips[node] = next_ip.to_s + next_ip = next_ip.succ + end + + ############################################################################ + # general vagrant setup + + # don't mind about insecure ssh key + config.ssh.insert_key = false + + # https://vagrantcloud.com/search. + config.vm.box = boxname + + # hardware and host settings + config.vm.provider 'libvirt' do |lv| + lv.cpus = 1 + lv.memory = 512 + lv.watchdog model: 'i6300esb' + lv.default_prefix = vm_prefix + end + + # disable default share + config.vm.synced_folder ".", "/vagrant", disabled: true + + config.vm.synced_folder "../../..", "/vagrant", type: "rsync", + rsync__exclude: [ ".git/" ] + + ############################################################################ + # system setup for all nodes + + config.vm.define pg_nodes.first, primary: true + + (pg_nodes + [log_node]).each do |node| + config.vm.define node do |conf| + conf.vm.network 'private_network', ip: nodes_ips[node] + conf.vm.provision 'system-setup', type: 'shell', + path: 'provision/system.bash', + args: [ node, rhel_user, rhel_pass ] + nodes_ips.keys.map {|n| "#{n}=#{nodes_ips[n]}"}, + preserve_order: true + end + end + + + ############################################################################ + # setup rsyslog to collect logs + (pg_nodes + [log_node]).each do |node| + config.vm.define node do |conf| + conf.vm.provision 'rsyslog-setup', type: 'shell', + path: 'provision/rsyslog.bash', + args: [ log_node ], + preserve_order: true + end + end + + ############################################################################ + # setup haproxy + pg_nodes.each do |node| + config.vm.define node do |conf| + conf.vm.provision 'haproxy-setup', type: 'shell', + path: 'provision/haproxy.bash', + preserve_order: true + end + end + + ############################################################################ + # postgresql installation and setup + pg_nodes.each do |node| + config.vm.define node do |conf| + conf.vm.provision 'postgresql', type: 'shell', + path: 'provision/postgresql.bash', + args: [ pgver, pg_nodes.first, pgdata ], + preserve_order: true + end + end + + # replicas setup. Use "vagrant up --provision-with=pgsql-replicas" + pg_nodes[1..-1].each do |node| + config.vm.define node do |conf| + conf.vm.provision 'pgsql-replicas', type: 'shell', + path: 'provision/pgsql-replicas.bash', + args: [ pgver, node, pgdata ], + run: 'never' + end + end + + ############################################################################ + # cluster setup. Use "vagrant up --provision-with=pacemaker" + pg_nodes.each do |node| + config.vm.define node do |conf| + conf.vm.provision 'pacemaker', type: 'shell', + path: 'provision/pacemaker.bash', + args: [ hapass ], + run: 'never' + end + end + + # create the cluster. Use "vagrant up --provision-with=cluster-setup" + pg_nodes.each do |node| + config.vm.define node do |conf| + conf.vm.provision 'cluster-setup', type: 'shell', + path: 'provision/cluster.bash', + args: [ pgver, ssh_login, vm_prefix, host_ip, pgdata, hapass ] + pg_nodes, + run: 'never' + end + end + + # cluster test suite setup. Use "vagrant up --provision-with=cts" + config.vm.provision 'cts', type: 'shell', path: 'provision/cts.bash', run: 'never' +end diff --git a/extra/vagrant/3nodes-haproxy/provision/cluster.bash b/extra/vagrant/3nodes-haproxy/provision/cluster.bash new file mode 100755 index 0000000..a25a309 --- /dev/null +++ b/extra/vagrant/3nodes-haproxy/provision/cluster.bash @@ -0,0 +1,102 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +PGVER="$1" +SSH_LOGIN="$2" +VM_PREFIX="$3" +HOST_IP="$4" +PGDATA="$5" +HAPASS="$6" +shift 6 +NODES=( "$@" ) + +CUSTOMDIR="${PGDATA}/conf.d" + +# psd authent +PCMK_VER=$(yum info --quiet pacemaker|grep ^Version) +PCMK_VER="${PCMK_VER#*: }" # extract x.y.z +PCMK_VER="${PCMK_VER:0:1}" # extract x + +if [ "$PCMK_VER" -ge 2 ]; then + # if pacemaker version is 2.x, we suppose pcs support it (pcs >= 0.10) + # from pcs 0.10, pcs host auth must be exec'ed on each node + pcs host auth -u hacluster -p "${HAPASS}" "${NODES[@]}" +else + # this could be run on one node, but it doesn't hurt if it runs everywhere, + # so we keep this piece of code with the one dedicated to pacemaker 2.x + pcs cluster auth -u hacluster -p "${HAPASS}" "${NODES[@]}" +fi + +# Stop PostgreSQL everywhere +systemctl --quiet stop "postgresql-${PGVER}" + +if [ "$(hostname -s)" != "${NODES[0]}" ]; then + exit 0 +fi + +# WARNING: +# Starting from here, everything is executed on first node only! + +if [ "$PCMK_VER" -ge 2 ]; then + pcs cluster setup cluster_pgsql --force "${NODES[@]}" +else + pcs cluster setup --name cluster_pgsql --wait --force "${NODES[@]}" +fi + +# pcs stonith sbd enable + +pcs cluster start --all --wait + +pcs cluster cib cluster1.xml + +pcs -f cluster1.xml resource defaults migration-threshold=5 +pcs -f cluster1.xml resource defaults resource-stickiness=10 +#pcs -f cluster1.xml property set stonith-watchdog-timeout=10s + +for VM in "${NODES[@]}"; do + FENCE_ID="fence_vm_${VM}" + VM_PORT="${VM_PREFIX}_${VM}" + pcs -f cluster1.xml stonith create "${FENCE_ID}" fence_virsh \ + pcmk_host_check=static-list "pcmk_host_list=${VM}" \ + "port=${VM_PORT}" "ipaddr=${HOST_IP}" "login=${SSH_LOGIN}" \ + "identity_file=/root/.ssh/id_rsa" + pcs -f cluster1.xml constraint location "fence_vm_${VM}" \ + avoids "${VM}=INFINITY" +done + +PGSQLD_RSC_OPTS=( + "ocf:heartbeat:pgsqlms" + "pgport=5434" + "bindir=/usr/pgsql-${PGVER}/bin" + "pgdata=${PGDATA}" + "recovery_template=${CUSTOMDIR}/recovery.conf.pcmk" + "op" "start" "timeout=60s" + "op" "stop" "timeout=60s" + "op" "promote" "timeout=30s" + "op" "demote" "timeout=120s" + "op" "monitor" "interval=15s" "timeout=10s" "role=Master" + "op" "monitor" "interval=16s" "timeout=10s" "role=Slave" + "op" "notify" "timeout=60s" +) + +# NB: pcs 0.10.2 doesn't support to set the id of the clone XML node +# the id is built from the rsc id to clone using "-clone" +# As a matter of cohesion and code simplicity, we use the same +# convention to create the master resource with pcs 0.9.x for +# Pacemaker 1.1 +if [ "$PCMK_VER" -ge 2 ]; then + PGSQLD_RSC_OPTS+=( "promotable" "notify=true" ) +fi + +pcs -f cluster1.xml resource create pgsqld "${PGSQLD_RSC_OPTS[@]}" + +if [ "$PCMK_VER" -eq 1 ]; then + pcs -f cluster1.xml resource master pgsqld-clone pgsqld notify=true +fi + +pcs cluster cib-push scope=configuration cluster1.xml --wait + +crm_mon -Dn1 diff --git a/extra/provision/cts.bash b/extra/vagrant/3nodes-haproxy/provision/cts.bash similarity index 100% rename from extra/provision/cts.bash rename to extra/vagrant/3nodes-haproxy/provision/cts.bash diff --git a/extra/vagrant/3nodes-haproxy/provision/haproxy.bash b/extra/vagrant/3nodes-haproxy/provision/haproxy.bash new file mode 100644 index 0000000..edf6539 --- /dev/null +++ b/extra/vagrant/3nodes-haproxy/provision/haproxy.bash @@ -0,0 +1,78 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +YUM_INSTALL="yum install --nogpgcheck --quiet -y -e 0" + +$YUM_INSTALL haproxy + +systemctl --quiet --now disable haproxy + +cp /etc/haproxy/haproxy.cfg /etc/haproxy/haproxy.cfg-dist +cat <<'EOF' > /etc/haproxy/haproxy.cfg +global + log 127.0.0.1:514 local2 + chroot /var/lib/haproxy + pidfile /var/run/haproxy.pid + maxconn 4000 + user haproxy + group haproxy + daemon + + stats socket /var/lib/haproxy/stats + +defaults + mode tcp + log global + option tcplog + retries 3 + timeout connect 10s + timeout client 10m + timeout server 10m + timeout check 1s + maxconn 300 + +listen stats + mode http + bind *:7000 + stats enable + stats uri / + timeout connect 15s + timeout client 15s + timeout server 15s + +listen prd + bind *:5432 + option tcp-check + tcp-check connect port 5431 + tcp-check expect string production + default-server inter 2s fastinter 1s rise 2 fall 1 on-marked-down shutdown-sessions + server srv1 srv1:5434 check + server srv2 srv2:5434 check + server srv3 srv3:5434 check + +listen stb + bind *:5433 + balance leastconn + option tcp-check + tcp-check connect port 5431 + tcp-check expect string standby + default-server inter 2s fastinter 1s rise 2 fall 1 on-marked-down shutdown-sessions + server srv1 srv1:5434 check + server srv2 srv2:5434 check + server srv3 srv3:5434 check +EOF + +setsebool -P haproxy_connect_any=1 + +systemctl --quiet --now enable haproxy + +if ! firewall-cmd --get-services|grep -q haproxy-stats; then + firewall-cmd --quiet --permanent --new-service="haproxy-stats" + firewall-cmd --quiet --permanent --service="haproxy-stats" --set-description="HAProxy statistics" + firewall-cmd --quiet --permanent --service="haproxy-stats" --add-port="7000/tcp" +fi +firewall-cmd --quiet --permanent --add-service="haproxy-stats" +firewall-cmd --quiet --reload diff --git a/extra/provision/id_rsa b/extra/vagrant/3nodes-haproxy/provision/id_rsa similarity index 100% rename from extra/provision/id_rsa rename to extra/vagrant/3nodes-haproxy/provision/id_rsa diff --git a/extra/provision/id_rsa.pub b/extra/vagrant/3nodes-haproxy/provision/id_rsa.pub similarity index 100% rename from extra/provision/id_rsa.pub rename to extra/vagrant/3nodes-haproxy/provision/id_rsa.pub diff --git a/extra/vagrant/3nodes-haproxy/provision/pacemaker.bash b/extra/vagrant/3nodes-haproxy/provision/pacemaker.bash new file mode 100755 index 0000000..4d4d57a --- /dev/null +++ b/extra/vagrant/3nodes-haproxy/provision/pacemaker.bash @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +HAPASS="$1" + +# shellcheck disable=SC1091 +source "/etc/os-release" +OS_ID="$ID" +YUM_INSTALL="yum install --nogpgcheck --quiet -y -e 0" + +# install required packages +if [ "$OS_ID" = "rhel" ]; then + # use yum instead of dnf for compatibility between EL 7 and 8 + yum-config-manager --enable "*highavailability-rpms" +fi + +PACKAGES=( + pacemaker pcs resource-agents fence-agents-virsh sbd perl-Module-Build +) + +$YUM_INSTALL "${PACKAGES[@]}" + +# install PAF +cd /vagrant +[ -f Build ] && perl Build distclean +sudo -u vagrant perl Build.PL --quiet >/dev/null 2>&1 +sudo -u vagrant perl Build --quiet +perl Build --quiet install + +# firewall setup +firewall-cmd --quiet --permanent --add-service=high-availability +firewall-cmd --quiet --reload + +# pcsd setup +systemctl --quiet --now enable pcsd +echo "${HAPASS}"|passwd --stdin hacluster > /dev/null 2>&1 + +# Pacemaker setup +cp /etc/sysconfig/pacemaker /etc/sysconfig/pacemaker.dist +cat<<'EOF' > /etc/sysconfig/pacemaker +PCMK_debug=yes +PCMK_logpriority=debug +EOF diff --git a/extra/vagrant/3nodes-haproxy/provision/pgsql-replicas.bash b/extra/vagrant/3nodes-haproxy/provision/pgsql-replicas.bash new file mode 100755 index 0000000..e0b7df2 --- /dev/null +++ b/extra/vagrant/3nodes-haproxy/provision/pgsql-replicas.bash @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +PGVER="$1" +NODENAME="$2" +PGDATA="$3" + +CUSTOMDIR="${PGDATA}/conf.d" + +# cleanup +systemctl --quiet --now disable "postgresql-${PGVER}" +rm -rf "${PGDATA}" + +# build standby +"/usr/pgsql-${PGVER}/bin/pg_basebackup" -h 127.0.0.1 -U postgres -D "${PGDATA}" -X stream + +# set pg_hba +cat< "${PGDATA}/pg_hba.conf" +local all all trust +host all all 0.0.0.0/0 trust + +local replication all reject +host replication all $NODENAME reject +host replication all 127.0.0.1/32 reject +host replication all ::1/128 reject +# allow any standby connection +host replication all 0.0.0.0/0 trust +EOC + +cat < "${CUSTOMDIR}/cluster_name.conf" +cluster_name = 'pgsql-$NODENAME' +EOC + +if [ "${PGVER%%.*}" -lt 12 ]; then + # recovery.conf setup + cat<<-EOC > "${CUSTOMDIR}/recovery.conf.pcmk" + standby_mode = on + primary_conninfo = 'host=127.0.0.1 application_name=${NODENAME}' + recovery_target_timeline = 'latest' + EOC + cp "${CUSTOMDIR}/recovery.conf.pcmk" "${PGDATA}/recovery.conf" +else + cat <<-EOC > "${CUSTOMDIR}/repli.conf" + primary_conninfo = 'host=127.0.0.1 application_name=${NODENAME}' + EOC + + # standby_mode disappear in v12 + # no need to add recovery_target_timeline as its default is 'latest' since v12 + touch "${PGDATA}/standby.signal" +fi + +# backing up files +cp "${PGDATA}/pg_hba.conf" "${PGDATA}/.." +cp "${PGDATA}/postgresql.conf" "${PGDATA}/.." +cp "${CUSTOMDIR}"/* "${PGDATA}/.." + +chown -R "postgres:postgres" "${PGDATA}/.." + +# start +systemctl --quiet start "postgresql-${PGVER}" diff --git a/extra/vagrant/3nodes-haproxy/provision/postgresql.bash b/extra/vagrant/3nodes-haproxy/provision/postgresql.bash new file mode 100755 index 0000000..9aecaa5 --- /dev/null +++ b/extra/vagrant/3nodes-haproxy/provision/postgresql.bash @@ -0,0 +1,151 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +PGVER="$1" +NODENAME="$2" +PGDATA="$3" + +# shellcheck disable=SC1091 +source "/etc/os-release" +OS_VER="$VERSION_ID" +YUM_INSTALL="yum install --nogpgcheck --quiet -y -e 0" + +if ! rpm --quiet -q "pgdg-redhat-repo"; then + if [ "${OS_VER:0:2}" = "8." ]; then + $YUM_INSTALL "https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm" + else + $YUM_INSTALL "https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm" + fi +fi + +# disable postgresql upstream module conflicting with pgdg packages in RHEL8 +if [ "${OS_VER:0:2}" = "8." ]; then + yum -qy module disable postgresql +fi + +PACKAGES=( + "postgresql${PGVER}" + "postgresql${PGVER}-server" + "postgresql${PGVER}-contrib" +) + +$YUM_INSTALL "${PACKAGES[@]}" + +# PostgreSQL state +cat<<'EOF' > /etc/systemd/system/pgsql-state@.service +[Unit] +Description=Local PostgreSQL state + +[Service] +User=postgres +Group=postgres +ExecStart=/usr/pgsql-12/bin/psql -d postgres -U postgres -p 5434 -Atc "select CASE pg_is_in_recovery() WHEN true THEN 'standby' ELSE 'production' END" +StandardOutput=socket +EOF + +cat<<'EOF' > /etc/systemd/system/pgsql-state.socket +[Unit] +Description=Local PostgreSQL state + +[Socket] +ListenStream=5431 +Accept=yes + +[Install] +WantedBy=sockets.target +EOF + +systemctl --quiet --now enable pgsql-state.socket + +# firewall setup +firewall-cmd --quiet --permanent --service=postgresql --add-port="5433/tcp" +firewall-cmd --quiet --permanent --service=postgresql --add-port="5434/tcp" +firewall-cmd --quiet --permanent --remove-service=postgresql +firewall-cmd --quiet --permanent --add-service=postgresql +if ! firewall-cmd --get-services|grep -q pgsql-state; then + firewall-cmd --quiet --permanent --new-service="pgsql-state" + firewall-cmd --quiet --permanent --service="pgsql-state" --set-description="Local PostgreSQL state" + firewall-cmd --quiet --permanent --service="pgsql-state" --add-port="5431/tcp" +fi +firewall-cmd --quiet --permanent --add-service="pgsql-state" +firewall-cmd --quiet --reload + +if [ "$(hostname -s)" != "$NODENAME" ]; then + exit 0 +fi + +# Build the primary +CUSTOMDIR="${PGDATA}/conf.d" + +# cleanup +systemctl --quiet --now disable "postgresql-${PGVER}" +rm -rf "${PGDATA}" + +# init instance +"/usr/pgsql-${PGVER}/bin/postgresql-${PGVER}-setup" initdb + +# pg_hba setup +cat< "${PGDATA}/pg_hba.conf" +local all all trust +host all all 0.0.0.0/0 trust + +local replication all reject +host replication all $NODENAME reject +host replication all 127.0.0.1/32 reject +host replication all ::1/128 reject +# allow any standby connection +host replication postgres 0.0.0.0/0 trust +EOC + +# postgresql.conf setup +mkdir -p "$CUSTOMDIR" +echo "include_dir = 'conf.d'" >> "${PGDATA}/postgresql.conf" + +cat < "${CUSTOMDIR}/cluster_name.conf" +cluster_name = 'pgsql-$NODENAME' +EOC + +cat <<'EOC' > "${CUSTOMDIR}/custom.conf" +listen_addresses = '*' +port = 5434 +wal_level = replica +max_wal_senders = 10 +hot_standby = on +hot_standby_feedback = on +wal_keep_segments = 256 +log_destination = 'syslog,stderr' +log_checkpoints = on +log_min_duration_statement = 0 +log_autovacuum_min_duration = 0 +log_replication_commands = on +log_line_prefix = '%m [%p] host=%h ' +EOC + +if [ "${PGVER%%.*}" -lt 12 ]; then + # recovery.conf setup + cat<<-EOC > "${CUSTOMDIR}/recovery.conf.pcmk" + standby_mode = on + primary_conninfo = 'host=127.0.0.1 application_name=${NODENAME}' + recovery_target_timeline = 'latest' + EOC +else + cat <<-EOC > "${CUSTOMDIR}/repli.conf" + primary_conninfo = 'host=127.0.0.1 application_name=${NODENAME}' + EOC + + # standby_mode disappear in v12 + # no need to add recovery_target_timeline as its default is 'latest' since v12 +fi + +# backing up files +cp "${PGDATA}/pg_hba.conf" "${PGDATA}/.." +cp "${PGDATA}/postgresql.conf" "${PGDATA}/.." +cp "${CUSTOMDIR}"/* "${PGDATA}/.." + +chown -R postgres:postgres "$PGDATA" + +# restart master pgsql +systemctl --quiet start "postgresql-${PGVER}" diff --git a/extra/vagrant/3nodes-haproxy/provision/rsyslog.bash b/extra/vagrant/3nodes-haproxy/provision/rsyslog.bash new file mode 100755 index 0000000..024055d --- /dev/null +++ b/extra/vagrant/3nodes-haproxy/provision/rsyslog.bash @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +LOG_SINK="$1" + +if [ "$(hostname -s)" == "$LOG_SINK" ]; then + # setup log sink + cat <<-'EOF' > /etc/rsyslog.d/log_sink.conf + $ModLoad imtcp + $InputTCPServerRun 514 + + $template RemoteLogsMerged,"/var/log/%HOSTNAME%/messages.log" + *.* ?RemoteLogsMerged + + $template RemoteLogs,"/var/log/%HOSTNAME%/%PROGRAMNAME%.log" + *.* ?RemoteLogs + #& ~ + EOF + + if ! firewall-cmd --get-services|grep -q rsyslog-tcp; then + firewall-cmd --quiet --permanent --new-service="rsyslog-tcp" + firewall-cmd --quiet --permanent --service="rsyslog-tcp" --set-description="RSyslog TCP port" + firewall-cmd --quiet --permanent --service="rsyslog-tcp" --add-port="514/tcp" + fi + firewall-cmd --quiet --permanent --add-service="rsyslog-tcp" + firewall-cmd --quiet --reload + + semanage port -m -t syslogd_port_t -p tcp 514 +else + # send logs to log-sinks + cat <<-'EOF' >/etc/rsyslog.d/20-fwd_log_sink.conf + *.* action(type="omfwd" + queue.type="LinkedList" + queue.filename="log_sink_fwd" + action.resumeRetryCount="-1" + queue.saveonshutdown="on" + target="log-sink" Port="514" Protocol="tcp") + EOF + + # listen for haproxy logs locally + cat <<-'EOF' >/etc/rsyslog.d/10-haproxy.conf + $ModLoad imudp + $UDPServerAddress 127.0.0.1 + $UDPServerRun 514 + EOF +fi + +systemctl --quiet restart rsyslog diff --git a/extra/vagrant/3nodes-haproxy/provision/system.bash b/extra/vagrant/3nodes-haproxy/provision/system.bash new file mode 100755 index 0000000..9ab5d68 --- /dev/null +++ b/extra/vagrant/3nodes-haproxy/provision/system.bash @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +NODENAME="$1" +RHEL_USER="$2" +RHEL_PASS="$3" +shift 3 +NODES=( "$@" ) + +hostnamectl set-hostname "${NODENAME}" + +for N in "${NODES[@]}"; do + NG=$(sed -n "/${N%=*}\$/p" /etc/hosts|wc -l) + if [ "$NG" -eq 0 ]; then + echo "${N##*=} ${N%=*}" >> /etc/hosts + fi +done + +# shellcheck disable=SC1091 +source "/etc/os-release" +OS_ID="$ID" +YUM_INSTALL="yum install --nogpgcheck --quiet -y -e 0" + +PACKAGES=( vim bash-completion yum-utils policycoreutils policycoreutils-python ) + +if [ "$OS_ID" = "rhel" ]; then + subscription-manager register --force --username "${RHEL_USER:?}" --password "${RHEL_PASS:?}" --auto-attach + PACKAGES+=("tmux") +else + PACKAGES+=("screen") +fi + +$YUM_INSTALL "${PACKAGES[@]}" + +cat <<'EOF' > "/home/vagrant/.ssh/config" +Host * + CheckHostIP no + StrictHostKeyChecking no +EOF + +cp "/vagrant/extra/vagrant/3nodes-haproxy/provision/id_rsa" "/home/vagrant/.ssh" +cp "/vagrant/extra/vagrant/3nodes-haproxy/provision/id_rsa.pub" "/home/vagrant/.ssh" + +chown -R "vagrant:" "/home/vagrant/.ssh" +chmod 0700 "/home/vagrant/.ssh" +chmod 0600 "/home/vagrant/.ssh/id_rsa" +chmod 0644 "/home/vagrant/.ssh/id_rsa.pub" +chmod 0600 "/home/vagrant/.ssh/config" +chmod 0600 "/home/vagrant/.ssh/authorized_keys" + +cp -R "/home/vagrant/.ssh" "/root" + +# force proper permissions on .ssh files +chown -R "root:" "/root/.ssh" +chmod 0700 "/root/.ssh" +chmod 0600 "/root/.ssh/id_rsa" +chmod 0644 "/root/.ssh/id_rsa.pub" +chmod 0600 "/root/.ssh/config" +chmod 0600 "/root/.ssh/authorized_keys" + +# enable firewall +systemctl --quiet --now enable firewalld diff --git a/extra/vagrant/3nodes-haproxy/vagrant.yml-dist b/extra/vagrant/3nodes-haproxy/vagrant.yml-dist new file mode 100644 index 0000000..fd19591 --- /dev/null +++ b/extra/vagrant/3nodes-haproxy/vagrant.yml-dist @@ -0,0 +1,14 @@ +# boxname: "centos/7" # vagrant box to use +# pgver: "10" # pg version to use +# hapass: "hapass" # password for sys user hacluster +# ssh_login: "user" # ssh login to connect to the host when fencing a VM. +# # put "./provision/id_rsa.pub" in your "~/.ssh/authorized_keys" +# base_ip: "10.20.30.5" # Base IP address to compute other ones +# pg_nodes: # servers to create. +# - "srv1" # First one will be master +# - "srv2" +# - "srv3" +# log_node: "log-sink" +# vm_prefix: "paf_vm" +# rhel_user: "" # RHEL user account +# rhel_pass: "" # RHEL user account password diff --git a/extra/Makefile b/extra/vagrant/3nodes-vip/Makefile similarity index 100% rename from extra/Makefile rename to extra/vagrant/3nodes-vip/Makefile diff --git a/extra/Vagrantfile b/extra/vagrant/3nodes-vip/Vagrantfile similarity index 100% rename from extra/Vagrantfile rename to extra/vagrant/3nodes-vip/Vagrantfile diff --git a/extra/provision/cluster-common.bash b/extra/vagrant/3nodes-vip/provision/cluster-common.bash similarity index 100% rename from extra/provision/cluster-common.bash rename to extra/vagrant/3nodes-vip/provision/cluster-common.bash diff --git a/extra/vagrant/3nodes-vip/provision/cts.bash b/extra/vagrant/3nodes-vip/provision/cts.bash new file mode 100755 index 0000000..b55b98d --- /dev/null +++ b/extra/vagrant/3nodes-vip/provision/cts.bash @@ -0,0 +1,114 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +# install packages + +PACKAGES=( + pacemaker-cts patch +) + +yum install --nogpgcheck --quiet -y -e 0 "${PACKAGES[@]}" + +# do not drop any log messages from rsyslog +cat <<'EOF'>/etc/rsyslog.d/rateLimit.conf +$imjournalRatelimitInterval 0 +$imjournalRatelimitBurst 0 +EOF + +systemctl --quiet restart rsyslog + +# make journald logs persistent +mkdir -p /var/log/journal + +# do not drop any log messages from journald +mkdir -p /etc/systemd/journald.conf.d +cat <<'EOF'>/etc/systemd/journald.conf.d/rateLimit.conf +RateLimitInterval=0 +RateLimitBurst=0 +EOF + +systemctl --quiet restart systemd-journald + + +# shellcheck disable=SC1091 +source "/etc/os-release" +OS_VER="$VERSION_ID" +if [ "${OS_VER:0:2}" != "7." ]; then exit; fi + +# fix bug in the log watcher for EL7 +cat <<'EOF' | patch /usr/lib64/python2.7/site-packages/cts/watcher.py +*** /tmp/watcher.py.orig 2019-02-07 16:25:32.836265277 +0100 +--- /tmp/watcher.py 2019-02-07 16:27:03.296926885 +0100 +*************** +*** 124,130 **** + self.offset = "EOF" + + if host == None: +! host = "localhost" + + def __str__(self): + if self.host: +--- 124,130 ---- + self.offset = "EOF" + + if host == None: +! self.host = "localhost" + + def __str__(self): + if self.host: +*************** +*** 155,179 **** + class FileObj(SearchObj): + def __init__(self, filename, host=None, name=None): + global has_log_watcher +! SearchObj.__init__(self, filename, host, name) +! +! if host is not None: +! if not host in has_log_watcher: + +! global log_watcher +! global log_watcher_bin + +! self.debug("Installing %s on %s" % (log_watcher_file, host)) + +! os.system("cat << END >> %s\n%s\nEND" %(log_watcher_file, log_watcher)) +! os.system("chmod 755 %s" %(log_watcher_file)) + +! self.rsh.cp(log_watcher_file, "root@%s:%s" % (host, log_watcher_bin)) +! has_log_watcher[host] = 1 + +! os.system("rm -f %s" %(log_watcher_file)) + +! self.harvest() + + def async_complete(self, pid, returncode, outLines, errLines): + for line in outLines: +--- 155,176 ---- + class FileObj(SearchObj): + def __init__(self, filename, host=None, name=None): + global has_log_watcher +! global log_watcher +! global log_watcher_bin + +! SearchObj.__init__(self, filename, host, name) + +! self.debug("Installing %s on %s" % (log_watcher_file, self.host)) + +! os.system("cat << END >> %s\n%s\nEND" %(log_watcher_file, log_watcher)) +! os.system("chmod 755 %s" %(log_watcher_file)) + +! self.rsh.cp(log_watcher_file, "root@%s:%s" % (self.host, log_watcher_bin)) +! has_log_watcher[self.host] = 1 + +! os.system("rm -f %s" %(log_watcher_file)) + +! self.harvest() + + def async_complete(self, pid, returncode, outLines, errLines): + for line in outLines: + +EOF + diff --git a/extra/vagrant/3nodes-vip/provision/id_rsa b/extra/vagrant/3nodes-vip/provision/id_rsa new file mode 100644 index 0000000..7d6a083 --- /dev/null +++ b/extra/vagrant/3nodes-vip/provision/id_rsa @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzI +w+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoP +kcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2 +hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NO +Td0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcW +yLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQIBIwKCAQEA4iqWPJXtzZA68mKd +ELs4jJsdyky+ewdZeNds5tjcnHU5zUYE25K+ffJED9qUWICcLZDc81TGWjHyAqD1 +Bw7XpgUwFgeUJwUlzQurAv+/ySnxiwuaGJfhFM1CaQHzfXphgVml+fZUvnJUTvzf +TK2Lg6EdbUE9TarUlBf/xPfuEhMSlIE5keb/Zz3/LUlRg8yDqz5w+QWVJ4utnKnK +iqwZN0mwpwU7YSyJhlT4YV1F3n4YjLswM5wJs2oqm0jssQu/BT0tyEXNDYBLEF4A +sClaWuSJ2kjq7KhrrYXzagqhnSei9ODYFShJu8UWVec3Ihb5ZXlzO6vdNQ1J9Xsf +4m+2ywKBgQD6qFxx/Rv9CNN96l/4rb14HKirC2o/orApiHmHDsURs5rUKDx0f9iP +cXN7S1uePXuJRK/5hsubaOCx3Owd2u9gD6Oq0CsMkE4CUSiJcYrMANtx54cGH7Rk +EjFZxK8xAv1ldELEyxrFqkbE4BKd8QOt414qjvTGyAK+OLD3M2QdCQKBgQDtx8pN +CAxR7yhHbIWT1AH66+XWN8bXq7l3RO/ukeaci98JfkbkxURZhtxV/HHuvUhnPLdX +3TwygPBYZFNo4pzVEhzWoTtnEtrFueKxyc3+LjZpuo+mBlQ6ORtfgkr9gBVphXZG +YEzkCD3lVdl8L4cw9BVpKrJCs1c5taGjDgdInQKBgHm/fVvv96bJxc9x1tffXAcj +3OVdUN0UgXNCSaf/3A/phbeBQe9xS+3mpc4r6qvx+iy69mNBeNZ0xOitIjpjBo2+ +dBEjSBwLk5q5tJqHmy/jKMJL4n9ROlx93XS+njxgibTvU6Fp9w+NOFD/HvxB3Tcz +6+jJF85D5BNAG3DBMKBjAoGBAOAxZvgsKN+JuENXsST7F89Tck2iTcQIT8g5rwWC +P9Vt74yboe2kDT531w8+egz7nAmRBKNM751U/95P9t88EDacDI/Z2OwnuFQHCPDF +llYOUI+SpLJ6/vURRbHSnnn8a/XG+nzedGH5JGqEJNQsz+xT2axM0/W/CRknmGaJ +kda/AoGANWrLCz708y7VYgAtW2Uf1DPOIYMdvo6fxIB5i9ZfISgcJ/bbCUkFrhoH ++vq/5CIWxCPp0f85R4qxxQ5ihxJ0YDQT9Jpx4TMss4PSavPaBH3RXow5Ohe+bYoQ +NE5OgEXk2wVfZczCZpigBKbKZHNYcelXtTt/nP3rsCuGcM4h53s= +-----END RSA PRIVATE KEY----- diff --git a/extra/vagrant/3nodes-vip/provision/id_rsa.pub b/extra/vagrant/3nodes-vip/provision/id_rsa.pub new file mode 100644 index 0000000..18a9c00 --- /dev/null +++ b/extra/vagrant/3nodes-vip/provision/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key diff --git a/extra/provision/log_sink.bash b/extra/vagrant/3nodes-vip/provision/log_sink.bash similarity index 100% rename from extra/provision/log_sink.bash rename to extra/vagrant/3nodes-vip/provision/log_sink.bash diff --git a/extra/provision/pacemaker.bash b/extra/vagrant/3nodes-vip/provision/pacemaker.bash similarity index 100% rename from extra/provision/pacemaker.bash rename to extra/vagrant/3nodes-vip/provision/pacemaker.bash diff --git a/extra/provision/pgsql-primary.bash b/extra/vagrant/3nodes-vip/provision/pgsql-primary.bash similarity index 100% rename from extra/provision/pgsql-primary.bash rename to extra/vagrant/3nodes-vip/provision/pgsql-primary.bash diff --git a/extra/provision/pgsql-replicas.bash b/extra/vagrant/3nodes-vip/provision/pgsql-replicas.bash similarity index 100% rename from extra/provision/pgsql-replicas.bash rename to extra/vagrant/3nodes-vip/provision/pgsql-replicas.bash diff --git a/extra/provision/system.bash b/extra/vagrant/3nodes-vip/provision/system.bash similarity index 100% rename from extra/provision/system.bash rename to extra/vagrant/3nodes-vip/provision/system.bash diff --git a/extra/vagrant.yml-dist b/extra/vagrant/3nodes-vip/vagrant.yml-dist similarity index 100% rename from extra/vagrant.yml-dist rename to extra/vagrant/3nodes-vip/vagrant.yml-dist diff --git a/extra/README.md b/extra/vagrant/README.md similarity index 94% rename from extra/README.md rename to extra/vagrant/README.md index f972db7..f56ef1d 100644 --- a/extra/README.md +++ b/extra/vagrant/README.md @@ -12,6 +12,8 @@ This `Vagrantfile` is bootstrapping a fresh cluster with: Note that NTP is enabled by default (using chrony) in the vagrant box used (`centos/7`). No need to set it up ourselves. +This README takes `3nodes-vip` as example. Replace with the cluster name you +want. ## Prerequisites @@ -35,7 +37,7 @@ vagrant plugin install vagrant-libvirt Pacemaker must be able to ssh to the libvirt host with no password using a user able to `virsh destroy $other_vm`. Here are the steps: -* copy `/test/provision/id_rsa.pub` inside `user@host:~/.ssh/authorized_keys` +* copy `/extra/vagrant/3nodes-vip/provision/id_rsa.pub` inside `user@host:~/.ssh/authorized_keys` * edit `ssh_login` in the `vagrant.yml` configuration file * user might need to be in group `libvirt` * user might need to add `uri_default='qemu:///system'` in its @@ -53,7 +55,7 @@ root$ su - $MYUSER myuser$ mkdir -p "${HOME}/.config/libvirt" myuser$ echo "uri_default='qemu:///system'" > "${HOME}/.config/libvirt/libvirt.conf" myuser$ git clone https://github.com/ClusterLabs/PAF.git -myuser$ cd PAF/test +myuser$ cd PAF/extra/vagrant/3nodes-vip myuser$ cat "provision/id_rsa.pub" >> "${HOME}/.ssh/authorized_keys" myuser$ echo "ssh_login: \"$USER\"" >> vagrant.yml ~~~ @@ -63,7 +65,7 @@ myuser$ echo "ssh_login: \"$USER\"" >> vagrant.yml To create the cluster, run: ~~~ -cd PAF/test +cd PAF/extra/vagrant/3nodes-vip make all ~~~