Merge pull request #342 from appuio/improve-probes

[mariadb-galera] Add failover testing scripts

appuio/mariadb-galera/.helmignore

@@ -19,3 +19,5 @@
 .project
 .idea/
 *.tmproj
+
+hack/

appuio/mariadb-galera/Chart.yaml

@@ -27,4 +27,4 @@ sources:
   - https://github.com/bitnami/bitnami-docker-mariadb-galera
   - https://github.com/prometheus/mysqld_exporter
   - https://mariadb.org
-version: 1.0.0
+version: 1.0.1

appuio/mariadb-galera/README.md

@@ -1,6 +1,6 @@
 # mariadb-galera
 
-![Version: 1.0.0](https://img.shields.io/badge/Version-1.0.0-informational?style=flat-square) ![AppVersion: 10.5.12](https://img.shields.io/badge/AppVersion-10.5.12-informational?style=flat-square)
+![Version: 1.0.1](https://img.shields.io/badge/Version-1.0.1-informational?style=flat-square) ![AppVersion: 10.5.12](https://img.shields.io/badge/AppVersion-10.5.12-informational?style=flat-square)
 
 MariaDB Galera is a multi-master database cluster solution for synchronous replication and high availability.
 

appuio/mariadb-galera/hack/failover-testing/Readme.md

@@ -0,0 +1,16 @@
+# MariaDB Galera Failover Experiments
+
+Scripts and random notes for galera failover experiments.
+
+## Files
+
+- `client.sh (ip)` Start a mysql client shell to the cluster.
+- `monitor.sh` Reports status changes for all nodes in a cluster.
+- `values.yaml` Helm values for a simple galera cluster.
+- `fill.sql` Adds a table with 100'000'000 (one hundred million) rows.
+- `deny-traffic-to-node.sh (remove) node_id` Deny network traffic to a single node.
+
+## Resources
+
+- https://mariadb.com/kb/en/what-is-mariadb-galera-cluster/
+- https://github.com/bitnami/charts/pull/6769 Optimize mariadb galera cluster split-brain detection

appuio/mariadb-galera/hack/failover-testing/client.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -eu
+
+kubectl run mariadb-client \
+    --rm --tty -i --restart='Never' \
+    --namespace default \
+    --labels="role=client" \
+    --image docker.io/bitnami/mariadb-galera:10.5.12-debian-10-r1 \
+    --command -- \
+        mysql -h ${1:-mariadb} -P 3306 -uroot \
+        -p$(kubectl get secret --namespace default mariadb -o jsonpath="{.data.mariadb-root-password}" | base64 --decode) \
+        my_database \
+||:
+
+kubectl delete pod/mariadb-client ||:

appuio/mariadb-galera/hack/failover-testing/deny-traffic-to-node.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+set -eu
+
+action=apply
+
+if [[ $# -eq 0 ]] ; then
+    echo 'Usage: $0 (remove) NODE_ID'
+    exit 3
+fi
+
+if [[ $# -eq 2 && "$1" == "remove" ]] ; then
+    action=delete
+    shift
+fi
+
+kubectl $action -f- <<- YAML
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: deny-mariadb-traffic-node-$1
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: mariadb-galera
+      statefulset.kubernetes.io/pod-name: mariadb-$1
+  # Exception for test-client
+  ingress:
+    - from:
+      - podSelector:
+          matchLabels:
+            role: client
+  egress:
+    - to:
+      - podSelector:
+          matchLabels:
+            role: client
+
+  policyTypes:
+  - Ingress
+  - Egress
+YAML

appuio/mariadb-galera/hack/failover-testing/fill.sql

@@ -0,0 +1,43 @@
+CREATE TABLE if not exists rand_numbers(number INT);
+
+insert into rand_numbers ( number ) select rand() from (
+    select 0 as i
+    union select 1 union select 2 union select 3
+    union select 4 union select 5 union select 6
+    union select 7 union select 8 union select 9
+) as t1, (
+    select 0 as i
+    union select 1 union select 2 union select 3
+    union select 4 union select 5 union select 6
+    union select 7 union select 8 union select 9
+) as t2, (
+    select 0 as i
+    union select 1 union select 2 union select 3
+    union select 4 union select 5 union select 6
+    union select 7 union select 8 union select 9
+) as t3, (
+    select 0 as i
+    union select 1 union select 2 union select 3
+    union select 4 union select 5 union select 6
+    union select 7 union select 8 union select 9
+) as t4, (
+    select 0 as i
+    union select 1 union select 2 union select 3
+    union select 4 union select 5 union select 6
+    union select 7 union select 8 union select 9
+) as t5, (
+    select 0 as i
+    union select 1 union select 2 union select 3
+    union select 4 union select 5 union select 6
+    union select 7 union select 8 union select 9
+) as t6, (
+    select 0 as i
+    union select 1 union select 2 union select 3
+    union select 4 union select 5 union select 6
+    union select 7 union select 8 union select 9
+) as t7, (
+    select 0 as i
+    union select 1 union select 2 union select 3
+    union select 4 union select 5 union select 6
+    union select 7 union select 8 union select 9
+) as t8;

appuio/mariadb-galera/hack/failover-testing/monitor.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+echo Monitoring nodes
+
+declare -A last_state=()
+
+while :
+do
+    for i in {0..2}
+    do
+        node=mariadb-$i
+        response=$(
+            # kubectl exec ${node} -- sh -c 'mysqladmin status -u"${MARIADB_ROOT_USER}" -p"${MARIADB_ROOT_PASSWORD}"' 2>&1
+            (kubectl exec ${node} -- sh -c 'mysql -Nb -u"${MARIADB_ROOT_USER}" -p"${MARIADB_ROOT_PASSWORD}" -e "select node_name from mysql.wsrep_cluster_members;"' | tr '\n' ',') 2>&1
+        )
+        if [ "$response" != "${last_state[$node]}" ]
+        then
+            echo "### $(date +%R:%S): Node $node state changed"
+            echo "'${last_state[$node]}' -> '$response'"
+            last_state[$node]=$response
+        fi
+    done
+done

appuio/mariadb-galera/hack/failover-testing/values.yaml

@@ -0,0 +1,37 @@
+fullnameOverride: mariadb
+extraEnvVars:
+  - name: MARIADB_GALERA_FORCE_SAFETOBOOTSTRAP
+    value: "no"
+startupProbe:
+  enabled: true
+  # Allow up to 20 minutes for DB to initialize
+  failureThreshold: 108
+  initialDelaySeconds: 10
+livenessProbe:
+  initialDelaySeconds: 10
+metrics:
+  enabled: false
+  extraFlags:
+    - --exporter.log_slow_filter
+    - --collect.engine_innodb_status
+  serviceMonitor:
+    enabled: true
+    interval: 10s
+    scrapeTimeout: 3s
+    relabelings:
+      - sourceLabels: [__address__]
+        targetLabel: service_level
+        replacement: standard
+persistence:
+  enabled: true
+podDisruptionBudget:
+  create: true
+  minAvailable: ""
+  maxUnavailable: 1
+rootUser:
+  forcePassword: true
+  password: "G076VpoTDOOFLnEQ0ns"
+galera:
+  mariabackup:
+    password: "G076VpoTDOOFLnEQ0ns"
+    forcePassword: true