diff --git a/charts/ozone/templates/_helpers.tpl b/charts/ozone/templates/_helpers.tpl index e5f56a8..c457004 100644 --- a/charts/ozone/templates/_helpers.tpl +++ b/charts/ozone/templates/_helpers.tpl @@ -51,22 +51,116 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- $pods | join "," }} {{- end }} -{{/* Common configuration environment variables */}} -{{- define "ozone.configuration.env" -}} +{{/* List of comma separated om ids */}} +{{- define "ozone.om.cluster.ids" -}} + {{- $pods := list }} + {{- $replicas := .Values.om.replicas | int }} + {{- range $i := until $replicas }} + {{- $pods = append $pods (printf "%s-om-%d" $.Release.Name $i) }} + {{- end }} + {{- $pods | join "," }} +{{- end }} + +{{/* List of comma separated scm ids */}} +{{- define "ozone.scm.cluster.ids" -}} + {{- $pods := list }} + {{- $replicas := .Values.scm.replicas | int }} + {{- range $i := until $replicas }} + {{- $pods = append $pods (printf "%s-scm-%d" $.Release.Name $i) }} + {{- end }} + {{- $pods | join "," }} +{{- end }} + +{{/* List of decommission om nodes */}} +{{- define "ozone.om.decommissioned.nodes" -}} + {{- $nodes := list }} + {{- $statefulset := lookup "apps/v1" "StatefulSet" $.Release.Namespace (printf "%s-om" $.Release.Name) -}} + {{- if $statefulset }} + {{- $oldCount := $statefulset.spec.replicas | int -}} + {{- $newCount := .Values.om.replicas | int }} + {{- range $i := until $oldCount }} + {{- $minCount := max $newCount 1 -}} + {{- if ge $i $minCount }} + {{- $nodes = append $nodes (printf "%s-om-%d" $.Release.Name $i) }} + {{- end }} + {{- end }} + {{- end }} + {{- $nodes | join "," }} +{{- end }} + +{{/* List of bootstrap om nodes */}} +{{- define "ozone.om.bootstrap.nodes" -}} + {{- $nodes := list }} + {{- $statefulset := lookup "apps/v1" "StatefulSet" $.Release.Namespace (printf "%s-om" $.Release.Name) -}} + {{- if $statefulset }} + {{- $oldCount := $statefulset.spec.replicas | int -}} + {{- $newCount := .Values.om.replicas | int }} + {{- range $i := until $newCount }} + {{- if ge $i $oldCount }} + {{- $nodes = append $nodes (printf "%s-om-%d" $.Release.Name $i) }} + {{- end }} + {{- end }} + {{- end }} + {{- $nodes | join ","}} +{{- end }} + +{{/* List of decommission scm nodes */}} +{{- define "ozone.scm.decommissioned.nodes" -}} + {{- $nodes := list }} + {{- $statefulset := lookup "apps/v1" "StatefulSet" $.Release.Namespace (printf "%s-scm" $.Release.Name) -}} + {{- if $statefulset }} + {{- $oldCount := $statefulset.spec.replicas | int -}} + {{- $newCount := .Values.scm.replicas | int }} + {{- range $i := until $oldCount }} + {{- if ge $i $newCount }} + {{- $nodes = append $nodes (printf "%s-scm-%d" $.Release.Name $i) }} + {{- end }} + {{- end }} + {{- end }} + {{- $nodes | join "," -}} +{{- end }} + +{{/* List of decommission data nodes */}} +{{- define "ozone.data.decommissioned.hosts" -}} + {{- $hosts := list }} + {{- $statefulset := lookup "apps/v1" "StatefulSet" $.Release.Namespace (printf "%s-datanode" $.Release.Name) -}} + {{- if $statefulset }} + {{- $oldCount := $statefulset.spec.replicas | int -}} + {{- $newCount := .Values.datanode.replicas | int }} + {{- range $i := until $oldCount }} + {{- if ge $i $newCount }} + {{- $hosts = append $hosts (printf "%s-datanode-%d.%s-datanode-headless.%s.svc.cluster.local" $.Release.Name $i $.Release.Name $.Release.Namespace) }} + {{- end }} + {{- end }} + {{- end }} + {{- $hosts | join "," -}} +{{- end }} + +{{- define "ozone.configuration.env.common" -}} - name: OZONE-SITE.XML_hdds.datanode.dir value: /data/storage - name: OZONE-SITE.XML_ozone.scm.datanode.id.dir value: /data - name: OZONE-SITE.XML_ozone.metadata.dirs value: /data/metadata -- name: OZONE-SITE.XML_ozone.scm.block.client.address - value: {{ include "ozone.scm.pods" . }} -- name: OZONE-SITE.XML_ozone.scm.client.address - value: {{ include "ozone.scm.pods" . }} -- name: OZONE-SITE.XML_ozone.scm.names - value: {{ include "ozone.scm.pods" . }} -- name: OZONE-SITE.XML_ozone.om.address - value: {{ include "ozone.om.pods" . }} +- name: OZONE-SITE.XML_ozone.scm.ratis.enable + value: "true" +- name: OZONE-SITE.XML_ozone.scm.service.ids + value: cluster1 +- name: OZONE-SITE.XML_ozone.scm.nodes.cluster1 + value: {{ include "ozone.scm.cluster.ids" . }} + {{/*- name: OZONE-SITE.XML_ozone.scm.skip.bootstrap.validation*/}} + {{/* value: {{ quote .Values.scm.skipBootstrapValidation }}*/}} +{{- range $i, $val := until ( .Values.scm.replicas | int ) }} +- name: {{ printf "OZONE-SITE.XML_ozone.scm.address.cluster1.%s-scm-%d" $.Release.Name $i }} + value: {{ printf "%s-scm-%d.%s-scm-headless.%s.svc.cluster.local" $.Release.Name $i $.Release.Name $.Release.Namespace }} +{{- end }} +- name: OZONE-SITE.XML_ozone.scm.primordial.node.id + value: {{ printf "%s-scm-0" $.Release.Name }} +- name: OZONE-SITE.XML_ozone.om.ratis.enable + value: "true" +- name: OZONE-SITE.XML_ozone.om.service.ids + value: cluster1 - name: OZONE-SITE.XML_hdds.scm.safemode.min.datanode value: "3" - name: OZONE-SITE.XML_ozone.datanode.pipeline.limit @@ -74,3 +168,49 @@ app.kubernetes.io/instance: {{ .Release.Name }} - name: OZONE-SITE.XML_dfs.datanode.use.datanode.hostname value: "true" {{- end }} + +{{/* Common configuration environment variables */}} +{{- define "ozone.configuration.env" -}} +{{- $bOmNodes := ternary (splitList "," (include "ozone.om.bootstrap.nodes" .)) (list) (ne "" (include "ozone.om.bootstrap.nodes" .)) }} +{{- $dOmNodes := ternary (splitList "," (include "ozone.om.decommissioned.nodes" .)) (list) (ne "" (include "ozone.om.decommissioned.nodes" .)) }} +{{- $activeOmNodes := ternary (splitList "," (include "ozone.om.cluster.ids" .)) (list) (ne "" (include "ozone.om.cluster.ids" .)) }} +{{ include "ozone.configuration.env.common" . }} +{{- if gt (len $dOmNodes) 0 }} +{{- $decomIds := $dOmNodes | join "," }} +- name: OZONE-SITE.XML_ozone.om.decommissioned.nodes.cluster1 + value: {{ $decomIds }} +{{- else}} +- name: OZONE-SITE.XML_ozone.om.decommissioned.nodes.cluster1 + value: "" +{{- end }} +- name: OZONE-SITE.XML_ozone.om.nodes.cluster1 + value: {{ $activeOmNodes | join "," }} +{{- range $tempId := $activeOmNodes }} +- name: {{ printf "OZONE-SITE.XML_ozone.om.address.cluster1.%s" $tempId }} + value: {{ printf "%s.%s-om-headless.%s.svc.cluster.local" $tempId $.Release.Name $.Release.Namespace }} +{{- end }} +{{- range $tempId := $dOmNodes }} +- name: {{ printf "OZONE-SITE.XML_ozone.om.address.cluster1.%s" $tempId }} + value: {{ printf "%s-helm-manager-decommission-%s-svc.%s.svc.cluster.local" $.Release.Name $tempId $.Release.Namespace }} +{{- end }} +{{- end }} + +{{/* Common configuration environment variables for pre hook */}} +{{- define "ozone.configuration.env.prehook" -}} +{{- $bOmNodes := ternary (splitList "," (include "ozone.om.bootstrap.nodes" .)) (list) (ne "" (include "ozone.om.bootstrap.nodes" .)) }} +{{- $dOmNodes := ternary (splitList "," (include "ozone.om.decommissioned.nodes" .)) (list) (ne "" (include "ozone.om.decommissioned.nodes" .)) }} +{{- $activeOmNodes := ternary (splitList "," (include "ozone.om.cluster.ids" .)) (list) (ne "" (include "ozone.om.cluster.ids" .)) }} +{{- $allOmNodes := concat $activeOmNodes $dOmNodes }} +{{ include "ozone.configuration.env.common" . }} +- name: OZONE-SITE.XML_ozone.om.decommissioned.nodes.cluster1 + value: "" +{{- range $tempId := $allOmNodes }} +- name: {{ printf "OZONE-SITE.XML_ozone.om.address.cluster1.%s" $tempId }} + value: {{ printf "%s.%s-om-headless.%s.svc.cluster.local" $tempId $.Release.Name $.Release.Namespace }} +{{- end }} +{{ $allOmNodes = append $allOmNodes "om-leader-transfer"}} +- name: OZONE-SITE.XML_ozone.om.nodes.cluster1 + value: {{ $allOmNodes | join "," }} +- name: "OZONE-SITE.XML_ozone.om.address.cluster1.om-leader-transfer" + value: localhost +{{- end }} \ No newline at end of file diff --git a/charts/ozone/templates/datanode/datanode-service-headless.yaml b/charts/ozone/templates/datanode/datanode-service-headless.yaml index 375abb1..1f023ca 100644 --- a/charts/ozone/templates/datanode/datanode-service-headless.yaml +++ b/charts/ozone/templates/datanode/datanode-service-headless.yaml @@ -28,6 +28,10 @@ spec: ports: - name: ui port: {{ .Values.datanode.service.port }} + - name: ratis-ipc + port: 9858 + - name: ipc + port: 9859 selector: {{- include "ozone.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: datanode diff --git a/charts/ozone/templates/datanode/datanode-statefulset.yaml b/charts/ozone/templates/datanode/datanode-statefulset.yaml index a9320d1..3ea6b6a 100644 --- a/charts/ozone/templates/datanode/datanode-statefulset.yaml +++ b/charts/ozone/templates/datanode/datanode-statefulset.yaml @@ -65,6 +65,10 @@ spec: ports: - name: ui containerPort: {{ .Values.datanode.service.port }} + - name: ratis-ipc + containerPort: 9858 + - name: ipc + containerPort: 9859 livenessProbe: httpGet: path: / diff --git a/charts/ozone/templates/helm/om-manager.yaml b/charts/ozone/templates/helm/om-manager.yaml new file mode 100644 index 0000000..1e2d4a2 --- /dev/null +++ b/charts/ozone/templates/helm/om-manager.yaml @@ -0,0 +1,239 @@ +{{- if or .Values.om.persistence.enabled }} +{{- $dnodes := ternary (splitList "," (include "ozone.om.decommissioned.nodes" .)) (list) (ne "" (include "ozone.om.decommissioned.nodes" .)) }} +{{- $env := concat .Values.env .Values.helm.env }} +{{- $envFrom := concat .Values.envFrom .Values.helm.envFrom }} +{{- $nodeSelector := or .Values.helm.nodeSelector .Values.nodeSelector }} +{{- $affinity := or .Values.helm.affinity .Values.affinity }} +{{- $tolerations := or .Values.helm.tolerations .Values.tolerations }} +{{- $securityContext := or .Values.helm.securityContext .Values.securityContext }} +{{- if and (gt (len $dnodes) 0) ( .Values.om.persistence.enabled) }} + +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ printf "%s-helm-manager-leader-transfer" $.Release.Name }} + labels: + {{- include "ozone.labels" $ | nindent 4 }} + app.kubernetes.io/component: helm-manager + annotations: + "helm.sh/hook": pre-upgrade + "helm.sh/hook-weight": "0" + "helm.sh/hook-delete-policy": hook-succeeded, hook-failed +spec: + backoffLimit: {{ $.Values.helm.backoffLimit }} + template: + metadata: + labels: + {{- include "ozone.selectorLabels" $ | nindent 8 }} + app.kubernetes.io/component: helm-manager + spec: + containers: + - name: om-leader-transfer + image: "{{ $.Values.image.repository }}:{{ $.Values.image.tag | default $.Chart.AppVersion }}" + imagePullPolicy: {{ $.Values.image.pullPolicy }} + {{- with $.Values.om.command }} + command: {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} + args: + - sh + - -c + - | + set -e + exec ozone admin om transfer -id=cluster1 -n={{ $.Release.Name }}-om-0 + env: + {{- include "ozone.configuration.env.prehook" $ | nindent 12 }} + {{- with $env }} + {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} + {{- with $envFrom }} + envFrom: {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} + ports: + - name: data-ratis-ipc + containerPort: 9858 + - name: data-ipc + containerPort: 9859 + - name: scm-rpc-client + containerPort: 9860 + - name: scm-block-cl + containerPort: 9863 + - name: scm-rpc-data + containerPort: 9861 + - name: scm-ratis + containerPort: 9894 + - name: scm-grpc + containerPort: 9895 + - name: om-rpc + containerPort: 9862 + - name: om-ratis + containerPort: 9872 + volumeMounts: + - name: config + mountPath: {{ $.Values.configuration.dir }} + - name: om-data + mountPath: {{ $.Values.om.persistence.path }} + {{- with $nodeSelector }} + nodeSelector: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $securityContext }} + securityContext: {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: om-data + emptyDir: { } + - name: config + projected: + sources: + - configMap: + name: {{ $.Release.Name }}-ozone + {{- with $.Values.configuration.filesFrom }} + {{- tpl (toYaml .) $ | nindent 14 }} + {{- end }} + restartPolicy: Never + +{{- range $dnode := $dnodes }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ printf "%s-helm-manager-decommission-%s" $.Release.Name $dnode }} + labels: + {{- include "ozone.labels" $ | nindent 4 }} + app.kubernetes.io/component: helm-manager + annotations: + "helm.sh/hook": post-upgrade + "helm.sh/hook-weight": "0" + "helm.sh/hook-delete-policy": hook-succeeded, hook-failed +spec: + backoffLimit: {{ $.Values.helm.backoffLimit }} + template: + metadata: + labels: + {{- include "ozone.selectorLabels" $ | nindent 8 }} + app.kubernetes.io/component: helm-manager + spec: + containers: + - name: om-decommission + image: "{{ $.Values.image.repository }}:{{ $.Values.image.tag | default $.Chart.AppVersion }}" + imagePullPolicy: {{ $.Values.image.pullPolicy }} + {{- with $.Values.om.command }} + command: {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} + args: + - sh + - -c + - | + set -e + decommission_finalizer() { + echo "Init decommission finalizer process..." + while true; do + IFS= read -r line; + echo "$line" + if echo "$line" | grep -q "Successfully decommissioned OM {{ $dnode }}"; then + echo "{{ $dnode }} was successfully decommissioned!" + if [ -d /old{{ $.Values.om.persistence.path }} ]; then + echo "Delete old data on pvc to enable rescheduling without manual PVC deletion!" + rm -rf /old{{ $.Values.om.persistence.path }}/* + echo "Data deleted!" + fi + break; + fi + done + echo "Decommission finalizer process finished!" + exit 0 + } + exec ozone admin om decommission -id=cluster1 -nodeid={{ $dnode }} -hostname={{ printf "%s-helm-manager-decommission-%s-svc.%s.svc.cluster.local" $.Release.Name $dnode $.Release.Namespace }} | decommission_finalizer + env: + {{- include "ozone.configuration.env" $ | nindent 12 }} + {{- with $env }} + {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} + {{- with $envFrom }} + envFrom: {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} + ports: + - name: data-ratis-ipc + containerPort: 9858 + - name: data-ipc + containerPort: 9859 + - name: scm-rpc-client + containerPort: 9860 + - name: scm-block-cl + containerPort: 9863 + - name: scm-rpc-data + containerPort: 9861 + - name: scm-ratis + containerPort: 9894 + - name: scm-grpc + containerPort: 9895 + - name: om-rpc + containerPort: 9862 + - name: om-ratis + containerPort: 9872 + volumeMounts: + - name: config + mountPath: {{ $.Values.configuration.dir }} + - name: om-data + mountPath: {{ $.Values.om.persistence.path }} + - name: om-data-old + mountPath: /old{{ $.Values.om.persistence.path }} + {{- with $nodeSelector }} + nodeSelector: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $securityContext }} + securityContext: {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: om-data-old + persistentVolumeClaim: + claimName: {{ $.Release.Name}}-om-{{ $dnode }} + - name: om-data + emptyDir: { } + - name: config + projected: + sources: + - configMap: + name: {{ $.Release.Name }}-ozone + {{- with $.Values.configuration.filesFrom }} + {{- tpl (toYaml .) $ | nindent 14 }} + {{- end }} + restartPolicy: Never + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ printf "%s-helm-manager-decommission-%s-svc" $.Release.Name $dnode }} + labels: + {{- include "ozone.labels" $ | nindent 4 }} + app.kubernetes.io/component: helm-manager + annotations: + "helm.sh/hook": post-upgrade + "helm.sh/hook-weight": "-10" + "helm.sh/hook-delete-policy": hook-succeeded, hook-failed +spec: + selector: + job-name: {{ printf "%s-helm-manager-decommission-%s" $.Release.Name $dnode }} + ports: + - name: rpc + port: 9862 + targetPort: 9862 + - name: ratis + port: 9872 + targetPort: 9872 + type: ClusterIP +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/ozone/templates/om/om-service-headless.yaml b/charts/ozone/templates/om/om-service-headless.yaml index d16659b..df9ce55 100644 --- a/charts/ozone/templates/om/om-service-headless.yaml +++ b/charts/ozone/templates/om/om-service-headless.yaml @@ -28,6 +28,8 @@ spec: ports: - name: ui port: {{ .Values.om.service.port }} + - name: ratis + port: 9872 selector: {{- include "ozone.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: om diff --git a/charts/ozone/templates/om/om-statefulset.yaml b/charts/ozone/templates/om/om-statefulset.yaml index b71c58c..43f86ae 100644 --- a/charts/ozone/templates/om/om-statefulset.yaml +++ b/charts/ozone/templates/om/om-statefulset.yaml @@ -22,6 +22,8 @@ {{- $affinity := or .Values.om.affinity .Values.affinity }} {{- $tolerations := or .Values.om.tolerations .Values.tolerations }} {{- $securityContext := or .Values.om.securityContext .Values.securityContext }} +{{- $bnodes := ternary (splitList "," (include "ozone.om.bootstrap.nodes" .)) (list) (ne "" (include "ozone.om.bootstrap.nodes" .)) }} +{{- $activeNodes := ternary (splitList "," (include "ozone.om.cluster.ids" .)) (list) (ne "" (include "ozone.om.cluster.ids" .)) }} apiVersion: apps/v1 kind: StatefulSet metadata: @@ -39,7 +41,7 @@ spec: template: metadata: annotations: - checksum/config: {{ include (print $.Template.BasePath "/ozone-configmap.yaml") . | sha256sum }} + checksum/config: {{ include (print $.Template.BasePath "/ozone-configmap.yaml") . | cat (include "ozone.configuration.env" .) | sha256sum }} labels: {{- include "ozone.selectorLabels" . | nindent 8 }} app.kubernetes.io/component: om @@ -51,9 +53,66 @@ spec: {{- with .Values.om.command }} command: {{- tpl (toYaml .) $ | nindent 12 }} {{- end }} - {{- with .Values.om.args }} - args: {{- tpl (toYaml .) $ | nindent 12 }} - {{- end }} + args: + - sh + - -c + - | + set -e + HELM_MANAGER_PATH="{{ .Values.om.persistence.path }}{{ .Values.helm.persistence.path }}" + HELM_MANAGER_BOOTSTRAPPED_FILE="$HELM_MANAGER_PATH/bootstrapped" + {{- $flattenedArgs := join " " $.Values.om.args }} + {{- if and (.Values.om.persistence.enabled) (gt (len $bnodes) 0) }} + joinArr() { + local IFS="," + echo "$*" + } + bootstrap_finalizer() { + echo "Init bootrap finalizer process..." + while true; do + IFS= read -r line; + echo "$line" + if echo "$line" | grep -q "Successfully bootstrapped OM $HOSTNAME and joined the Ratis group"; then + echo "$HOSTNAME was successfully bootstrapped!" + mkdir -p "$HELM_MANAGER_PATH" + touch "$HELM_MANAGER_BOOTSTRAPPED_FILE" + break; + fi + done + echo "Bootstrap finalizer process finished!" + exit 0 + } + bootstrapHosts="{{ join "," $bnodes }}" + echo "Need to handle bootstrap for nodes $bootstrapHosts" + IFS=',' read -r -a hostArray <<< "$bootstrapHosts" + doBootstrap=false + nodesConfigOverwriteList=() + for host in "${hostArray[@]}"; do + if [[ "$host" == "$HOSTNAME" ]]; then + doBootstrap=true + activeNodesConfig="{{ join "," $activeNodes }}" + IFS=',' read -r -a overwriteArray <<< "$activeNodesConfig" + for overwriteHost in "${overwriteArray[@]}"; do + nodesConfigOverwriteList+=("$overwriteHost") + if [[ "$overwriteHost" == "$HOSTNAME" ]]; then + break; + fi + done + break + fi + done + if [ "$doBootstrap" = true ] && [ ! -f "$HELM_MANAGER_BOOTSTRAPPED_FILE" ]; then + echo "$HOSTNAME must be started with bootstrap arg!" + overwriteCmd="$(joinArr "${nodesConfigOverwriteList[@]}")" + echo "Bootstrapping node config for this node: $overwriteCmd" + exec {{ printf "%s --set ozone.om.nodes.cluster1=" $flattenedArgs }}"$overwriteCmd" --bootstrap | bootstrap_finalizer + else + echo "$HOSTNAME must not be started with bootstrap arg!" + exec {{ join " " $.Values.om.args }} + fi + {{- else }} + echo "No bootstrap handling needed!" + exec {{ join " " $.Values.om.args }} + {{- end }} env: {{- include "ozone.configuration.env" . | nindent 12 }} - name: WAITFOR @@ -71,6 +130,8 @@ spec: containerPort: 9862 - name: ui containerPort: {{ .Values.om.service.port }} + - name: ratis + containerPort: 9872 livenessProbe: httpGet: path: / @@ -104,7 +165,7 @@ spec: {{- end }} {{- if not .Values.om.persistence.enabled }} - name: {{ .Release.Name }}-om - emptyDir: {} + emptyDir: { } {{- end }} {{- if .Values.om.persistence.enabled }} volumeClaimTemplates: diff --git a/charts/ozone/templates/ozone-configmap.yaml b/charts/ozone/templates/ozone-configmap.yaml index dbee026..f5b12d8 100644 --- a/charts/ozone/templates/ozone-configmap.yaml +++ b/charts/ozone/templates/ozone-configmap.yaml @@ -21,5 +21,9 @@ kind: ConfigMap metadata: name: {{ .Release.Name }}-ozone labels: {{- include "ozone.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": pre-upgrade, post-install + "helm.sh/hook-weight": "-10" + "helm.sh/resource-policy": keep data: {{- tpl (toYaml .Values.configuration.files) $ | nindent 4 }} diff --git a/charts/ozone/templates/scm/scm-service-headless.yaml b/charts/ozone/templates/scm/scm-service-headless.yaml index dce5857..8890ffd 100644 --- a/charts/ozone/templates/scm/scm-service-headless.yaml +++ b/charts/ozone/templates/scm/scm-service-headless.yaml @@ -28,6 +28,18 @@ spec: ports: - name: ui port: {{ .Values.scm.service.port }} + - name: rpc-datanode + port: 9861 + - name: block-client + port: 9863 + - name: rpc-client + port: 9860 + {{- if gt (int .Values.scm.replicas) 1 }} + - name: ratis + port: 9894 + - name: grpc + port: 9895 + {{- end }} selector: {{- include "ozone.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: scm diff --git a/charts/ozone/templates/scm/scm-statefulset.yaml b/charts/ozone/templates/scm/scm-statefulset.yaml index 457ba26..6f10f5e 100644 --- a/charts/ozone/templates/scm/scm-statefulset.yaml +++ b/charts/ozone/templates/scm/scm-statefulset.yaml @@ -31,6 +31,7 @@ metadata: app.kubernetes.io/component: scm spec: replicas: {{ .Values.scm.replicas }} + podManagementPolicy: Parallel serviceName: {{ .Release.Name }}-scm-headless selector: matchLabels: @@ -61,6 +62,24 @@ spec: mountPath: {{ .Values.configuration.dir }} - name: {{ .Release.Name }}-scm mountPath: {{ .Values.scm.persistence.path }} + {{- if gt (int .Values.scm.replicas) 1 }} + - name: bootstrap + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + args: ["ozone", "scm", "--bootstrap"] + env: + {{- include "ozone.configuration.env" . | nindent 12 }} + {{- with $env }} + {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} + {{- with $envFrom }} + envFrom: {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: {{ .Values.configuration.dir }} + - name: {{ .Release.Name }}-scm + mountPath: {{ .Values.scm.persistence.path }} + {{- end }} containers: - name: scm image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" @@ -82,10 +101,18 @@ spec: ports: - name: rpc-client containerPort: 9860 + - name: block-client + containerPort: 9863 - name: rpc-datanode containerPort: 9861 - name: ui containerPort: {{ .Values.scm.service.port }} + {{- if gt (int .Values.scm.replicas) 1 }} + - name: ratis + containerPort: 9894 + - name: grpc + containerPort: 9895 + {{- end }} livenessProbe: httpGet: path: / diff --git a/charts/ozone/values.yaml b/charts/ozone/values.yaml index 9c2dc4a..2eefb4e 100644 --- a/charts/ozone/values.yaml +++ b/charts/ozone/values.yaml @@ -214,3 +214,28 @@ scm: size: 10Gi # The name of a specific storage class name to use storageClassName: ~ + +# Helm Manager configuration +helm: + # Additional Helm Manager environment variables (templated) + env: [] + # Additional Helm Manager envFrom items to set up environment variables (templated) + envFrom: [] + # Constrain Helm Manager pods to nodes with specific node labels + nodeSelector: {} + # Constrain Helm Manager pods to nodes by affinity/anti-affinity rules + affinity: {} + # Allow to schedule Helm Manager pods on nodes with matching taints + tolerations: [] + # Helm Manager security context (overwrites common security context) + securityContext: {} + # Decommissioning is handled with a post-upgrade helm hook job. + # To avoid endless retries of decommissioning, this limit is set. + # This can happen if PVC has been deleted or is not reachable. + # This is used for decommissioning OM + backoffLimit: 5 + # Helm Manager persistence (this is enabled automatically if al least one + # of datanode, scm or om is enabled) + persistence: + # Path for Helm Manager volume mount + path: /metadata/helm \ No newline at end of file