diff --git a/.gitignore b/.gitignore index b6f0879..c41c637 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ deployment +rs-key +google-credentials diff --git a/README.md b/README.md index 98d9abf..17a59c0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # kubernetes-mongodb-cluster -A scalable kubernetes cluster for SSL secured mongodb. +A scalable kubernetes cluster for SSL secured mongodb on GKE with backups. ![issues](https://img.shields.io/github/issues/AlexsJones/kubernetes-mongodb-cluster.svg) ![forks](https://img.shields.io/github/forks/AlexsJones/kubernetes-mongodb-cluster.svg) @@ -9,50 +9,99 @@ A scalable kubernetes cluster for SSL secured mongodb. ![twitter](https://img.shields.io/twitter/url/https/github.com/AlexsJones/kubernetes-mongodb-cluster.svg?style=social) -Built on the great work of others, brought together in k8s manifests. - +- GKE local disks +- Backups with FUSE to Google storage - Statefulset +- Node/Pod affinity keys - Configmap for mongo.conf, boot options and per env tuning - Service discovery with sidecars - Supports auto scaling - Example built with generated SSL cert -Influenced and inspired by: -- https://github.com/MichaelScript/kubernetes-mongodb -- https://github.com/cvallance/mongo-k8s-sidecar -- My own experience with trying to implement this.. https://kubernetes.io/blog/2017/01/running-mongodb-on-kubernetes-with-statefulsets/ - ## Dependencies ``` - golang - go get github.com/AlexsJones/vortex -- google cloud platform (for a few annotations e.g. load balancer and pvc) +- google cloud platform (for a few annotations e.g. load balancer and pvc) and GKE cluster ``` ## Get me started +If you want to start from absolute zero here is the command to build the cluster on GKE: + +1. + +``` +gcloud container clusters create mongodbcluster --num-nodes 1 --node-locations=europe-west2-a,europe-west2-b,europe-west2-c --local-ssd-count 3 --region=europe-west2 --labels=type=mongodb --node-labels=node-type=mongodb --machine-type=n1-standard-8 + +kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin --user=$(gcloud config get-value core/account) ``` -kubectl create ns mongodb -./build_environment.sh dev -./generate_pem.sh -kubectl apply -f deployment/mongo + +2. + +- Provide a service account with access to the `storage_bucket` as defined in `environments/` + e.g `storage_bucket: mybucketintheus` *It must have access to storage object get/list/create* + +- Download the secret for this service account locally e.g `gcloud iam service-accounts keys create google-credentials --iam-account ` + +- `kubectl create secret generic google-credentials --from-file=google-credentials -n mongodb` + +3. + +Followed by the deployment (production-gke) + +``` +- `./build_environment.sh production-gke ` +- ./generate_pem.sh +- `kubectl apply -f deployment/gke-storage -n mongodb` +- `kubectl apply -f deployment/mongo -n mongodb` +``` + +_To confirm the local-disks are attached run the following_ + +``` +❯ kubectl get pvc -n mongodb +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +data-mongod-0 Bound local-pv-46a6870e 368Gi RWO local-scsi 1m +data-mongod-1 Bound local-pv-93823dd3 368Gi RWO local-scsi 40s +data-mongod-2 Bound local-pv-69642ae6 368Gi RWO local-scsi 17s + ``` +4. + + + + +## But I don't like GKE and/or I'm on another provider + +If you do not wish to use GKE nor local-scsi do the following deployment + + +``` +- Check your environment/ does not have local-scsi storage class set +- `./build_environment.sh ` +- ./generate_pem.sh +- `kubectl apply -f deployment/mongo -n mongodb` +``` +./build + ## Test it works The mongo-job runs the following command ``` -kubectl exec -it mongod-0 -c mongod -- mongo --host 127.0.0.1:27017 --authenticationDatabase admin --username root --password root --eval "rs.status()" +kubectl exec -it mongod-0 -n mongodb -c mongod -- mongo --host 127.0.0.1:27017 --authenticationDatabase admin --username root --password root --eval "rs.status()" ``` Execute the job with ``` -kubectl apply -f deployment/utils/job.yaml +kubectl apply -f deployment/utils/job.yaml -n mongodb ``` ## Configuration + - Primary mongodb configuration is within `templates/mongo/configmap.yaml` for wiredtiger settings and log paths - Within `templates/mongo/statefulset.yaml` the mongod boot options can be changed to suit requirements. @@ -70,14 +119,15 @@ mongodb: Can be changed in the environment folder file - -### Restoring a database backup - -Since 1.10 you can now upload mongodumps into configmaps or in `utils/pod-mongorestore.yaml` you can just use `kubectl cp` -then execute the backup file - - ## Using UI tools Tools such as mongochef/robochef can be used with their direct connection mode on localhost:27017 and `kubectl port-forward mongod-0 27017:27017` + + +### Creditations + +Influenced and inspired by: +- https://github.com/MichaelScript/kubernetes-mongodb +- https://github.com/cvallance/mongo-k8s-sidecar +- My own experience with trying to implement this.. https://kubernetes.io/blog/2017/01/running-mongodb-on-kubernetes-with-statefulsets/ diff --git a/environments/default.yaml b/environments/default.yaml index cd6a351..3f9b22b 100644 --- a/environments/default.yaml +++ b/environments/default.yaml @@ -20,10 +20,13 @@ dev: "true" image: "{{.image}}" namespace: "{{.namespace}}" replica: "3" -affinity_key: "{{.affinitykey}}" -affinity_selector: "{{.affinityselector}}" +with_affinity: "true" +pod_affinity_key: "app" +pod_affinity_selector: "mongodb" +node_affinity_key: "{{.affinitykey}}" +node_affinity_selector: "{{.affinityselector}}" storage_size: "{{.storagesize}}" -storage_class: "{{.storageclass}}" +storage_class: "{{.storageclass}}" #local-scsi can be selected on GKE with the ./templates/gke-storage applied resources: requests: cpu: "{{.cpurequest}}" diff --git a/environments/dev.yaml b/environments/dev.yaml index 09cd6f2..3b9ba6d 100644 --- a/environments/dev.yaml +++ b/environments/dev.yaml @@ -1,11 +1,15 @@ -dev: "true" -image: "mongo" +image: "mongo:3.6" namespace: "mongodb" replica: "3" -affinity_key: "app" -affinity_selector: "mongodb" +with_affinity: "false" +# These wll not apply ----------------- +pod_affinity_key: "app" +pod_affinity_selector: "mongodb" +node_affinity_key: "node-type" +node_affinity_selector: "mongodb" +# -------------------------------------- storage_size: "1Gi" -storage_class: "fast-retain" +storage_class: "fast-retain" #local-scsi can be selected on GKE with the ./templates/gke-storage applied resources: requests: cpu: "0.2m" diff --git a/environments/production-gke.yaml b/environments/production-gke.yaml new file mode 100644 index 0000000..c935111 --- /dev/null +++ b/environments/production-gke.yaml @@ -0,0 +1,27 @@ +image: "mongo:3.6" +namespace: "mongodb" +replica: "3" +with_affinity: "true" +pod_affinity_key: "app" +pod_affinity_selector: "mongodb" +node_affinity_key: "node-type" +node_affinity_selector: "mongodb" +with_gcs_backups: "true" +gcsfuse: + storage_bucket: mongodb-gcs-backups +storage_size: "250Gi" +storage_class: "local-scsi" #local-scsi can be selected on GKE with the ./templates/gke-storage applied +resources: + requests: + cpu: "2" + memory: "1000Mi" + limits: + cpu: "3" + memory: "30000Mi" +mongodb: + rootusername: "root" + rootpassword: "root" + replsetname: "MainRepSet" + sslmode: "preferSSL" +mongosidecar: + sslenabled: "true" diff --git a/gcsfuse/Dockerfile b/gcsfuse/Dockerfile new file mode 100644 index 0000000..11dad6a --- /dev/null +++ b/gcsfuse/Dockerfile @@ -0,0 +1,6 @@ +FROM ubuntu:xenial + +RUN apt-get update && apt-get install curl -y + +RUN echo "deb http://packages.cloud.google.com/apt gcsfuse-xenial main" | tee /etc/apt/sources.list.d/gcsfuse.list && \ + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && apt-get update && apt-get install gcsfuse -y diff --git a/gcsfuse/build_and_push.sh b/gcsfuse/build_and_push.sh new file mode 100755 index 0000000..3774068 --- /dev/null +++ b/gcsfuse/build_and_push.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +docker build -t tibbar/gcsfuse:latest . +docker push tibbar/gcsfuse:latest diff --git a/generate_pem.sh b/generate_pem.sh index bbfd2cc..31015f1 100755 --- a/generate_pem.sh +++ b/generate_pem.sh @@ -1,3 +1,4 @@ +kubectl create ns mongodb || true echo 'Generating self signed certificate' KEY=$1 openssl genrsa -des3 -passout pass:$KEY -out server.pass.key 2048 @@ -15,4 +16,3 @@ openssl rand -base64 741 > rs-key chmod 0400 rs-key kubectl --namespace=mongodb delete secret mongodb-rs-key || true kubectl --namespace=mongodb create secret generic mongodb-rs-key --from-file=rs-key -rm rs-key diff --git a/mongo-k8s-sidecar b/mongo-k8s-sidecar deleted file mode 160000 index 770b1cd..0000000 --- a/mongo-k8s-sidecar +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 770b1cd4f772aa105c6303398478bc1e52f27e80 diff --git a/templates/gke-storage/scsi-storage.yaml b/templates/gke-storage/scsi-storage.yaml new file mode 100644 index 0000000..7e4b06a --- /dev/null +++ b/templates/gke-storage/scsi-storage.yaml @@ -0,0 +1,128 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: local-scsi +spec: + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: node-type + operator: In + values: + - {{.namespace}} + capacity: + storage: 375Gi + accessModes: + - "ReadWriteOnce" + persistentVolumeReclaimPolicy: "Retain" + storageClassName: "local-storage" + local: + path: "/mnt/disks/ssd0" + +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: "local-scsi" +provisioner: "kubernetes.io/no-provisioner" +volumeBindingMode: "WaitForFirstConsumer" +--- +# Source: provisioner/templates/provisioner.yaml + +apiVersion: v1 +kind: ConfigMap + +metadata: + name: local-provisioner-config +data: + useNodeNameOnly: "true" + storageClassMap: | + local-scsi: + hostDir: /mnt/disks + mountDir: /mnt/disks +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: local-volume-provisioner + namespace: {{.namespace}} + labels: + app: local-volume-provisioner +spec: + selector: + matchLabels: + app: local-volume-provisioner + template: + metadata: + labels: + app: local-volume-provisioner + spec: + serviceAccountName: local-storage-admin + containers: + - image: "quay.io/external_storage/local-volume-provisioner:v2.2.0" + imagePullPolicy: "Always" + name: provisioner + securityContext: + privileged: true + env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - mountPath: /etc/provisioner/config + name: provisioner-config + readOnly: true + - mountPath: /mnt/disks + name: local-scsi + volumes: + - name: provisioner-config + configMap: + name: local-provisioner-config + - name: local-scsi + hostPath: + path: /mnt/disks +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: local-storage-admin +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: local-storage-provisioner-pv-binding + namespace: {{.namespace}} +subjects: +- kind: ServiceAccount + name: local-storage-admin + namespace: {{.namespace}} +roleRef: + kind: ClusterRole + name: system:persistent-volume-provisioner + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: local-storage-provisioner-node-clusterrole + namespace: {{.namespace}} +rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: local-storage-provisioner-node-binding + namespace: {{.namespace}} +subjects: +- kind: ServiceAccount + name: local-storage-admin + namespace: {{.namespace}} +roleRef: + kind: ClusterRole + name: local-storage-provisioner-node-clusterrole + apiGroup: rbac.authorization.k8s.io diff --git a/templates/mongo/configmap.yaml b/templates/mongo/configmap.yaml index 42589cf..907e8ac 100644 --- a/templates/mongo/configmap.yaml +++ b/templates/mongo/configmap.yaml @@ -4,6 +4,33 @@ metadata: namespace: mongodb apiVersion: v1 data: + setup.sh: | + err_report() { + echo "errexit on line $(caller)" >&2 + } + trap err_report ERR + apt-get update; + apt-get install rsync curl -y; + echo "deb http://packages.cloud.google.com/apt gcsfuse-xenial main" > /etc/apt/sources.list.d/gcsfuse.list; + curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -; + apt-get update; + apt-get install -y gcsfuse; + # Mount folder + mkdir -p /mnt/bucket; + gcsfuse -o nonempty,allow_other,rw $STORAGE_BUCKET /mnt/bucket + # Create Job + sleep 3000 + /etc/configure_backups.sh + configure_backups.sh: | + MONGO_IS_PRIMARY=`mongo localhost:27017 --quiet --eval 'db.isMaster().ismaster'`; + if [ "${MONGO_IS_PRIMARY}" == "true" ]; then + echo "No backups will be taken against a primary node, exiting!" + exit 0; + fi + D=$(/bin/date -u "+%Y%m%d-%H%M%S") + mkdir -p /mnt/bucket/$D + mkdir -p /mnt/bucket/$D/$HOSTNAME + rsync -a --ignore-missing-args --delete-during /data/db /mnt/bucket/$D/$HOSTNAME mongodb.conf: | # mongod.conf # for documentation of all options, see: diff --git a/templates/mongo/statefulset.yaml b/templates/mongo/statefulset.yaml index b20fc52..a4979b0 100644 --- a/templates/mongo/statefulset.yaml +++ b/templates/mongo/statefulset.yaml @@ -25,18 +25,28 @@ spec: spec: serviceAccount: kubernetes-mongodb-service-account {{- if not .dev }} -{{- if .affinity_key }} affinity: +{{- if .node_affinity_key }} + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: {{.node_affinity_key}} + operator: In + values: + - {{.node_affinity_selector}} +{{- end }} +{{- if .pod_affinity_key }} podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 podAffinityTerm: labelSelector: matchExpressions: - - key: {{.affinity_key}} + - key: {{.pod_affinity_key}} operator: In values: - - {{.affinity_selector}} + - {{.pod_affinity_selector}} topologyKey: kubernetes.io/hostname {{- end }} {{- end }} @@ -44,7 +54,13 @@ spec: initContainers: - name: copy-ro-scripts image: busybox - command: ['sh', '-c', 'cp /tmp/rs-key /mongotemp/rs-key && chmod 0400 /mongotemp/rs-key && chown 999:999 /mongotemp/rs-key'] + command: + - /bin/sh + - -c + - > + cp /tmp/rs-key /mongotemp/rs-key; + chmod 0400 /mongotemp/rs-key; + chown 999:999 /mongotemp/rs-key; volumeMounts: - name: mongodb-rs-key mountPath: /tmp/rs-key @@ -53,7 +69,7 @@ spec: mountPath: /mongotemp containers: - name: mongo-sidecar - image: tibbar/mongo-k8s-sidecar + image: tibbar/mongo-k8s-sidecar:latest env: - name: MONGO_SIDECAR_POD_LABELS value: "app=mongodb,role=services,component=mongodb" @@ -74,27 +90,25 @@ spec: - name: KUBE_NAMESPACE value: "{{.namespace}}" - name: mongod +{{- if .with_gcs_backups }} + securityContext: + privileged: true + capabilities: + add: + - SYS_ADMIN +{{- end }} image: {{.image}} - args: - - "mongod" - - "--wiredTigerCacheSizeGB" - - "1.5" - - "--bind_ip" - - "0.0.0.0" - - "--replSet" - - "{{.mongodb.replsetname}}" - - "--smallfiles" - - "--noprealloc" - - "--auth" - - "--sslMode" - - "{{.mongodb.sslmode}}" - - "--sslPEMKeyFile" - - "/etc/mongodb-secret/mongodb.pem" - - "--sslAllowConnectionsWithoutCertificates" - - "--sslAllowInvalidCertificates" - - "--sslAllowInvalidHostnames" - - "--config" - - "/etc/mongodb-config/mongodb.conf" + imagePullPolicy: IfNotPresent + command: + - /bin/bash + - -c + - > +{{- if .with_gcs_backups }} + /etc/setup.sh +{{- end }} + mongod --wiredTigerCacheSizeGB 1.5 --bind_ip 0.0.0.0 --replSet {{.mongodb.replsetname}} \ + --smallfiles --noprealloc --auth --sslMode {{.mongodb.sslmode}} --sslPEMKeyFile /etc/mongodb-secret/mongodb.pem \ + --sslAllowConnectionsWithoutCertificates --sslAllowInvalidCertificates --config /etc/mongodb-config/mongodb.conf resources: requests: {{- if ne .resources.requests.memory "" }} @@ -111,6 +125,12 @@ spec: cpu: "{{ .resources.limits.cpu }}" {{- end }} env: +{{- if .with_gcs_backups }} + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /etc/google-credentials + - name: STORAGE_BUCKET + value: {{ .gcsfuse.storage_bucket }} +{{- end }} - name: MONGO_INITDB_ROOT_USERNAME value: "{{.mongodb.rootusername}}" - name: MONGO_INITDB_ROOT_PASSWORD @@ -129,6 +149,17 @@ spec: mountPath: /mongotemp - name: data mountPath: /data/db +{{- if .with_gcs_backups }} + - name: mongodb-config + mountPath: /etc/setup.sh + subPath: setup.sh + - name: mongodb-config + mountPath: /etc/configure_backups.sh + subPath: configure_backups.sh + - name: google-credentials + mountPath: /etc/google-credentials + subPath: google-credentials +{{- end }} volumes: - name: tempvol emptyDir: {} @@ -141,6 +172,12 @@ spec: - name: mongodb-config configMap: name: mongodb-config + defaultMode: 0744 +{{- if .with_gcs_backups }} + - name: google-credentials + secret: + secretName: google-credentials +{{- end }} volumeClaimTemplates: - metadata: name: data