From 44b8f904db415558d83074d94a22ff700d023985 Mon Sep 17 00:00:00 2001 From: Viktor Zhemchuzhnikov Date: Mon, 22 Apr 2024 13:47:33 +0900 Subject: [PATCH 1/8] feat: add ingress for processors and plain http setup --- Chart.yaml | 10 +++---- charts/ca/Chart.yaml | 4 +-- charts/ca/values.yaml | 2 +- configs/nifi.properties | 23 +++++++++++++-- templates/NOTES.txt | 12 ++++++-- templates/ingress-processors.yaml | 47 +++++++++++++++++++++++++++++++ templates/ingress.yaml | 4 +-- templates/service.yaml | 11 ++++++++ templates/statefulset.yaml | 45 ++++++++++++++++++++++++++++- values.yaml | 30 ++++++++++++++++---- 10 files changed, 168 insertions(+), 20 deletions(-) create mode 100644 templates/ingress-processors.yaml diff --git a/Chart.yaml b/Chart.yaml index b3fd5a59..77b8c18a 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -1,8 +1,8 @@ --- apiVersion: v2 name: nifi -version: 1.2.1 -appVersion: 1.23.2 +version: 1.3.0 +appVersion: 1.25.0 description: Apache NiFi is a software project from the Apache Software Foundation designed to automate the flow of data between software systems. keywords: - nifi @@ -27,16 +27,16 @@ maintainers: url: https://github.com/zakaria2905 dependencies: - name: zookeeper - version: 9.2.7 + version: 13.1.1 repository: https://charts.bitnami.com/bitnami condition: zookeeper.enabled - name: nifi-registry alias: registry - version: 1.0.0 + version: 1.1.5 repository: https://dysnix.github.io/charts/ condition: registry.enabled - name: ca - version: 1.0.1 + version: 1.0.2 condition: ca.enabled - name: openldap version: ~1.2.4 diff --git a/charts/ca/Chart.yaml b/charts/ca/Chart.yaml index 3d17b7f7..1ff365d4 100644 --- a/charts/ca/Chart.yaml +++ b/charts/ca/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 name: ca -version: 1.0.1 +version: 1.0.2 # We are using the nifi version as appVersion -appVersion: 1.11.4 +appVersion: 1.25.0 description: A Helm chart to deploy ca server to generate self-signed certificates using nifi-toolkit. keywords: - nifi-toolkit diff --git a/charts/ca/values.yaml b/charts/ca/values.yaml index 936e2702..45b01a9d 100644 --- a/charts/ca/values.yaml +++ b/charts/ca/values.yaml @@ -7,7 +7,7 @@ replicaCount: 1 image: repository: apache/nifi-toolkit pullPolicy: IfNotPresent - tag: "1.12.1" + tag: "1.25.0" service: type: ClusterIP diff --git a/configs/nifi.properties b/configs/nifi.properties index 35cacebc..6ca97e4c 100644 --- a/configs/nifi.properties +++ b/configs/nifi.properties @@ -122,7 +122,11 @@ nifi.components.status.snapshot.frequency=1 min # Site to Site properties nifi.remote.input.host= +{{ if .Values.properties.httpsPort }} nifi.remote.input.secure=true +{{ else }} +nifi.remote.input.secure=false +{{ end }} nifi.remote.input.socket.port={{.Values.properties.siteToSite.port}} nifi.remote.input.http.enabled=true nifi.remote.input.http.transaction.ttl=30 sec @@ -132,7 +136,8 @@ nifi.remote.contents.cache.expiration=30 secs nifi.web.war.directory=./lib nifi.web.proxy.host={{.Values.properties.webProxyHost}} nifi.web.https.port={{.Values.properties.httpsPort}} -nifi.web.http.host= +nifi.web.http.host={{.Values.properties.webHttpHost}} +nifi.web.http.port={{.Values.properties.httpPort}} nifi.web.http.network.interface.default= nifi.web.https.host={{.Values.properties.webHttpsHost}} nifi.web.https.network.interface.default= @@ -177,7 +182,7 @@ nifi.security.truststore=/opt/nifi/nifi-current/conf/truststore.p12 nifi.security.truststoreType=PKCS12 nifi.security.truststorePasswd= nifi.security.user.authorizer=managed-authorizer -{{ else }} +{{ else if .Values.properties.httpsPort }} nifi.security.keystore=./conf/keystore.p12 nifi.security.keystoreType=PKCS12 nifi.security.keystorePasswd= @@ -187,6 +192,16 @@ nifi.security.truststoreType=PKCS12 nifi.security.truststorePasswd= nifi.security.user.login.identity.provider=single-user-provider nifi.security.user.authorizer=single-user-authorizer +{{else}} +nifi.security.keystore= +nifi.security.keystoreType= +nifi.security.keystorePasswd= +nifi.security.keyPasswd= +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.user.login.identity.provider= +nifi.security.user.authorizer= {{end}} nifi.security.needClientAuth={{.Values.properties.needClientAuth}} @@ -221,7 +236,11 @@ nifi.security.user.knox.audiences= # cluster common properties (all nodes must have same values) # nifi.cluster.protocol.heartbeat.interval=5 sec +{{ if .Values.properties.httpsPort}} nifi.cluster.protocol.is.secure=true +{{ else }} +nifi.cluster.protocol.is.secure=false +{{ end }} # cluster node properties (only configure for cluster nodes) # nifi.cluster.is.node={{.Values.properties.isNode}} diff --git a/templates/NOTES.txt b/templates/NOTES.txt index e1ddf03a..7f098122 100644 --- a/templates/NOTES.txt +++ b/templates/NOTES.txt @@ -1,6 +1,14 @@ +{{- $port := (.Values.service.httpsPort | default .Values.service.httpPort) }} To access the NiFi UI via kubectl port forwarding, wait until the cluster is ready and then run: -kubectl port-forward -n {{ .Release.Namespace}} svc/{{ .Release.Name }} {{ .Values.service.httpsPort }}:{{ .Values.service.httpsPort }} +kubectl port-forward -n {{ .Release.Namespace}} svc/{{ .Release.Name }} {{ $port }}:{{ $port }} -...and point your web browser to https://localhost:{{ .Values.service.httpsPort }}/nifi/ +...and point your web browser to http{{ if .Values.properties.httpsPort }}s{{ end }}://localhost:{{ $port }}/nifi/ + +{{- if .Values.ingress.enabled }} +...and since you enabled the ingress, you can access NiFi at: +{{- range .Values.ingress.hosts }} +http://{{ . }}/nifi +{{ end }} +{{ end }} \ No newline at end of file diff --git a/templates/ingress-processors.yaml b/templates/ingress-processors.yaml new file mode 100644 index 00000000..08522e62 --- /dev/null +++ b/templates/ingress-processors.yaml @@ -0,0 +1,47 @@ +--- +{{- if .Values.ingress.processors.enabled -}} +{{- $fullName := include "apache-nifi.fullname" $ -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ template "apache-nifi.fullname" $ }}-ingress-processors + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "apache-nifi.name" . | quote }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + release: {{ .Release.Name | quote }} + heritage: {{ .Release.Service | quote }} +{{- with .Values.ingress.processors.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: +{{- if .Values.ingress.processors.className }} + ingressClassName: {{ .Values.ingress.processors.className | quote }} +{{- end }} +{{- if .Values.ingress.processors.tls }} + tls: + {{- range .Values.ingress.processors.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} +{{- end }} + rules: + {{- range .Values.ingress.processors.hosts }} + - host: {{ .host }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .type }} + backend: + service: + name: {{ .serviceName }} + port: + number: {{ .port }} + {{- end }} + {{- end }} +{{- end }} diff --git a/templates/ingress.yaml b/templates/ingress.yaml index e398fa7a..7bd62649 100644 --- a/templates/ingress.yaml +++ b/templates/ingress.yaml @@ -2,7 +2,7 @@ {{- if .Values.ingress.enabled -}} {{- $fullName := include "apache-nifi.fullname" $ -}} {{- $ingressPath := .Values.ingress.path -}} -{{- $ingressHttpsPort := .Values.service.httpsPort -}} +{{- $ingressHttpPort := (.Values.service.httpsPort | default .Values.service.httpPort) -}} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -42,6 +42,6 @@ spec: service: name: {{ $fullName }} port: - number: {{ $ingressHttpsPort }} + number: {{ $ingressHttpPort }} {{- end }} {{- end }} diff --git a/templates/service.yaml b/templates/service.yaml index 5de375b7..be225683 100644 --- a/templates/service.yaml +++ b/templates/service.yaml @@ -16,8 +16,13 @@ spec: type: {{ .Values.headless.type }} clusterIP: None ports: + {{- if .Values.properties.httpsPort }} - port: {{ .Values.properties.httpsPort }} name: https + {{- else }} + - port: {{ .Values.properties.httpPort }} + name: http + {{- end }} - port: {{ .Values.properties.clusterPort }} name: cluster - port: {{ .Values.properties.siteToSite.port }} @@ -60,9 +65,15 @@ spec: {{- end }} {{- end }} ports: + {{- if .Values.properties.httpsPort }} - port: {{ .Values.service.httpsPort }} name: https targetPort: {{ .Values.properties.httpsPort }} + {{- else }} + - port: {{ .Values.service.httpPort }} + name: http + targetPort: {{ .Values.properties.httpPort }} + {{- end }} nodePort: {{ .Values.service.nodePort }} - port: {{ .Values.properties.siteToSite.port }} name: site-to-site diff --git a/templates/statefulset.yaml b/templates/statefulset.yaml index 97d4b27a..1ff633bc 100644 --- a/templates/statefulset.yaml +++ b/templates/statefulset.yaml @@ -299,7 +299,11 @@ spec: function offloadNode() { FQDN=$(hostname -f) echo "disconnecting node '$FQDN'" + {{- if .Values.properties.httpsPort }} baseUrl=https://${FQDN}:{{ .Values.properties.httpsPort }} + {{- else }} + baseUrl=http://${FQDN}:{{ .Values.properties.httpPort }} + {{- end }} echo "keystoreType=$(prop nifi.security.keystoreType)" > secure.properties echo "keystore=$(prop nifi.security.keystore)" >> secure.properties @@ -320,7 +324,11 @@ spec: echo "" echo "get a connected node" connectedNode=$(jq -r 'first(.cluster.nodes|=sort_by(.address)| .cluster.nodes[] | select(.status=="CONNECTED")) | .address' nodes.json) + {{- if .Values.properties.httpsPort }} baseUrl=https://${connectedNode}:{{ .Values.properties.httpsPort }} + {{- else }} + baseUrl=http://${connectedNode}:{{ .Values.properties.httpPort }} + {{- end }} echo baseUrl ${baseUrl} echo "" echo "wait until node has state 'DISCONNECTED'" @@ -377,6 +385,7 @@ spec: name: metrics protocol: TCP {{- end }} +{{- if .Values.properties.httpsPort }} - containerPort: {{ .Values.properties.httpsPort }} {{- if .Values.sts.hostPort }} hostPort: {{ .Values.sts.hostPort }} @@ -386,6 +395,32 @@ spec: - containerPort: {{ .Values.properties.clusterPort }} name: cluster protocol: TCP +{{- else }} + - containerPort: {{ .Values.properties.httpPort }} +{{- if .Values.sts.hostPort }} + hostPort: {{ .Values.sts.hostPort }} +{{- end }} + name: http + protocol: TCP +{{- end }} +{{- if .Values.properties.siteToSite.port }} + - containerPort: {{ .Values.properties.siteToSite.port }} +{{- if .Values.properties.siteToSite.hostPort }} + hostPort: {{ .Values.properties.siteToSite.hostPort }} +{{- end }} + name: site-to-site + protocol: TCP +{{- end }} +{{- if .Values.containerPorts }} +{{ toYaml .Values.containerPorts | indent 8 }} +{{- end }} + env: + - name: NIFI_ZOOKEEPER_CONNECT_STRING + value: {{ template "zookeeper.url" . }} +{{- if not (or (.Values.auth.ldap.enabled) (.Values.auth.oidc.enabled)) }} + - name: NIFI_WEB_HTTPS_HOST + value: +{{- end }} {{- if .Values.containerPorts }} {{ toYaml .Values.containerPorts | indent 8 }} {{- end }} @@ -416,7 +451,7 @@ spec: {{- end }} periodSeconds: 20 tcpSocket: - port: {{ .Values.properties.httpsPort }} + port: {{ .Values.service.httpsPort | default .Values.service.httpPort }} # exec: # command: # - bash @@ -444,7 +479,11 @@ spec: failureThreshold: {{ .Values.sts.startupProbe.failureThreshold }} periodSeconds: {{ .Values.sts.startupProbe.periodSeconds }} tcpSocket: + {{- if .Values.properties.httpsPort }} port: {{ .Values.properties.httpsPort }} + {{- else }} + port: {{ .Values.properties.httpPort }} + {{- end }} {{- end }} livenessProbe: {{- if not .Values.sts.startupProbe.enabled }} @@ -452,7 +491,11 @@ spec: {{- end }} periodSeconds: 60 tcpSocket: + {{- if .Values.properties.httpsPort }} port: {{ .Values.properties.httpsPort }} + {{- else }} + port: {{ .Values.properties.httpPort }} + {{- end }} volumeMounts: - mountPath: /opt/nifi/nifi-current/logs {{- if and .Values.persistence.enabled .Values.persistence.subPath.enabled }} diff --git a/values.yaml b/values.yaml index 5daec4f7..13b32162 100644 --- a/values.yaml +++ b/values.yaml @@ -7,7 +7,7 @@ replicaCount: 1 ## image: repository: apache/nifi - tag: "1.23.2" + tag: "1.25.0" pullPolicy: "IfNotPresent" ## Optionally specify an imagePullSecret. @@ -91,7 +91,8 @@ properties: # use externalSecure for when inbound SSL is provided by nginx-ingress or other external mechanism externalSecure: false isNode: false - httpsPort: 8443 + httpsPort: 8443 # Set this to null to disable secure configuration. + httpPort: 8080 webProxyHost: # : (If Nifi service is NodePort or LoadBalancer) clusterPort: 6007 zkClientEnsembleTraker: false # https://issues.apache.org/jira/browse/NIFI-10481 @@ -192,7 +193,7 @@ headless: # ui service service: type: ClusterIP - httpsPort: 8443 + httpsPort: 8443 # Make sure to change it to httpPort if you set httpsPort to null # nodePort: 30236 annotations: {} # loadBalancerIP: @@ -233,8 +234,27 @@ ingress: annotations: {} tls: [] hosts: [] - path: / + path: / # Change it to /nifi if you want to have a ingress for processors on the same host # If you want to change the default path, see this issue https://github.com/cetic/helm-nifi/issues/22 + processors: + enabled: false + # className: nginx + annotations: {} + tls: [] + # - hosts: + # - nifi.example.com + # secretName: nifi-tls + hosts: [] + # - host: nifi.example.com + # paths: + # - path: /processor01 + # port: 7001 + # serviceName: processor01 + # type: Prefix + # - path: /processor02 + # port: 7002 + # serviceName: processor02 + # type: Prefix # Amount of memory to give the NiFi java heap jvmMemory: 2g @@ -249,7 +269,7 @@ sidecar: ## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ ## persistence: - enabled: false + enabled: true # When creating persistent storage, the NiFi helm chart can either reference an already-defined # storage class by name, such as "standard" or can define a custom storage class by specifying From 68f01c0736a9196cd5150961911de76f1f83e77f Mon Sep 17 00:00:00 2001 From: Viktor Zhemchuzhnikov Date: Mon, 22 Apr 2024 15:04:26 +0900 Subject: [PATCH 2/8] fix: rename service.httpsPort to service.httpPort --- templates/NOTES.txt | 2 +- templates/service.yaml | 2 +- templates/statefulset.yaml | 2 +- values.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/NOTES.txt b/templates/NOTES.txt index 7f098122..e040538e 100644 --- a/templates/NOTES.txt +++ b/templates/NOTES.txt @@ -1,4 +1,4 @@ -{{- $port := (.Values.service.httpsPort | default .Values.service.httpPort) }} +{{- $port := .Values.service.httpPort }} To access the NiFi UI via kubectl port forwarding, wait until the cluster is ready and then run: diff --git a/templates/service.yaml b/templates/service.yaml index be225683..be89a87a 100644 --- a/templates/service.yaml +++ b/templates/service.yaml @@ -66,7 +66,7 @@ spec: {{- end }} ports: {{- if .Values.properties.httpsPort }} - - port: {{ .Values.service.httpsPort }} + - port: {{ .Values.service.httpPort }} name: https targetPort: {{ .Values.properties.httpsPort }} {{- else }} diff --git a/templates/statefulset.yaml b/templates/statefulset.yaml index 1ff633bc..d697525a 100644 --- a/templates/statefulset.yaml +++ b/templates/statefulset.yaml @@ -451,7 +451,7 @@ spec: {{- end }} periodSeconds: 20 tcpSocket: - port: {{ .Values.service.httpsPort | default .Values.service.httpPort }} + port: {{ .Values.service.httpPort }} # exec: # command: # - bash diff --git a/values.yaml b/values.yaml index 13b32162..846d6bc7 100644 --- a/values.yaml +++ b/values.yaml @@ -193,7 +193,7 @@ headless: # ui service service: type: ClusterIP - httpsPort: 8443 # Make sure to change it to httpPort if you set httpsPort to null + httpPort: 8443 # Make sure to change it to properties.httpPort if you set properties.httpsPort to null # nodePort: 30236 annotations: {} # loadBalancerIP: From b804c4deef9b4b9c9001e0f9e43a6507c36c4d91 Mon Sep 17 00:00:00 2001 From: Viktor Zhemchuzhnikov Date: Mon, 22 Apr 2024 15:10:22 +0900 Subject: [PATCH 3/8] fix: use the same port for service --- templates/service.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/templates/service.yaml b/templates/service.yaml index be89a87a..49ec0a9b 100644 --- a/templates/service.yaml +++ b/templates/service.yaml @@ -65,15 +65,14 @@ spec: {{- end }} {{- end }} ports: - {{- if .Values.properties.httpsPort }} - port: {{ .Values.service.httpPort }} + {{- if .Values.properties.httpsPort }} name: https targetPort: {{ .Values.properties.httpsPort }} - {{- else }} - - port: {{ .Values.service.httpPort }} + {{- else }} name: http targetPort: {{ .Values.properties.httpPort }} - {{- end }} + {{- end }} nodePort: {{ .Values.service.nodePort }} - port: {{ .Values.properties.siteToSite.port }} name: site-to-site From ee4c2cb0aa672fe8505b7720d047a4993457e56d Mon Sep 17 00:00:00 2001 From: Viktor Zhemchuzhnikov Date: Mon, 30 Sep 2024 17:16:55 +0900 Subject: [PATCH 4/8] fix: adjust to support secure configuration with traefik --- configs/logback.xml | 261 +++++++++++++++++++++++++++++++ configs/nifi.properties | 6 + templates/servers_transport.yaml | 10 ++ templates/service.yaml | 5 +- templates/statefulset.yaml | 63 ++------ 5 files changed, 292 insertions(+), 53 deletions(-) create mode 100644 configs/logback.xml create mode 100644 templates/servers_transport.yaml diff --git a/configs/logback.xml b/configs/logback.xml new file mode 100644 index 00000000..46de56b8 --- /dev/null +++ b/configs/logback.xml @@ -0,0 +1,261 @@ + + + + + + + + true + + + + ${org.apache.nifi.bootstrap.config.log.dir}/nifi-app.log + + + ${org.apache.nifi.bootstrap.config.log.dir}/nifi-app_%d{yyyy-MM-dd_HH}.%i.log + 100MB + + 30 + + true + + %date %level [%thread] %logger{40} %msg%n + + + + + ${org.apache.nifi.bootstrap.config.log.dir}/nifi-user.log + + + ${org.apache.nifi.bootstrap.config.log.dir}/nifi-user_%d.log + + 30 + + + %date %level [%thread] %logger{40} %msg%n + + + + + ${org.apache.nifi.bootstrap.config.log.dir}/nifi-request.log + + ${org.apache.nifi.bootstrap.config.log.dir}/nifi-request_%d.log + 30 + + + %msg%n + + + + + ${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap.log + + + ${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap_%d.log + + 5 + + + %date %level [%thread] %logger{40} %msg%n + + + + + ${org.apache.nifi.bootstrap.config.log.dir}/nifi-deprecation.log + + ${org.apache.nifi.bootstrap.config.log.dir}/nifi-deprecation_%d.%i.log + 10MB + 10 + 100MB + + + %date %level [%thread] %logger %msg%n + + + + + + + + ${org.apache.nifi.bootstrap.config.log.dir}/nifi-app-${logFileSuffix}.log + + + ${org.apache.nifi.bootstrap.config.log.dir}/nifi-app-${logFileSuffix}_%d{yyyy-MM-dd_HH}.%i.log + 100MB + + 30 + + true + + %date %level [%thread] %logger{40} %msg%n + + + + + + + + %date %level [%thread] %logger{40} %msg%n + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/configs/nifi.properties b/configs/nifi.properties index 6ca97e4c..ffb796ad 100644 --- a/configs/nifi.properties +++ b/configs/nifi.properties @@ -14,7 +14,13 @@ # limitations under the License. # Core Properties # +{{ $isVersion2 := hasPrefix "2." .Values.image.tag -}} + +{{ if $isVersion2 -}} +nifi.flow.configuration.file=../data/flow.json.gz +{{ else -}} nifi.flow.configuration.file=../data/flow.xml.gz +{{ end -}} nifi.flow.configuration.archive.enabled=true nifi.flow.configuration.archive.dir=../data/archive/ nifi.flow.configuration.archive.max.time={{.Values.properties.flowArchiveMaxTime}} diff --git a/templates/servers_transport.yaml b/templates/servers_transport.yaml new file mode 100644 index 00000000..4b8a5cb4 --- /dev/null +++ b/templates/servers_transport.yaml @@ -0,0 +1,10 @@ +{{- if .Values.traefik.serversTransport.enabled }} +apiVersion: traefik.io/v1alpha1 +kind: ServersTransport +metadata: + name: {{ template "apache-nifi.fullname" . }}-transport + namespace: {{ .Release.Namespace }} + +spec: + insecureSkipVerify: {{ .Values.traefik.serversTransport.insecureSkipVerify }} +{{- end }} diff --git a/templates/service.yaml b/templates/service.yaml index 49ec0a9b..c0093943 100644 --- a/templates/service.yaml +++ b/templates/service.yaml @@ -40,10 +40,13 @@ metadata: chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" release: {{ .Release.Name | quote }} heritage: {{ .Release.Service | quote }} -{{- if .Values.service.annotations }} +{{- if or .Values.service.annotations .Values.traefik.serversTransport.enabled }} annotations: {{ toYaml .Values.service.annotations | indent 4 }} {{- end }} +{{- if .Values.traefik.serversTransport.enabled }} + traefik.ingress.kubernetes.io/service.serversTransport: {{ .Release.Namespace }}-{{ template "apache-nifi.fullname" . }}-transport@kubernetescrd +{{- end }} spec: type: {{ .Values.service.type }} {{- if .Values.service.loadBalancerIP }} diff --git a/templates/statefulset.yaml b/templates/statefulset.yaml index d697525a..4c8d550a 100644 --- a/templates/statefulset.yaml +++ b/templates/statefulset.yaml @@ -137,45 +137,11 @@ spec: {{- end }} cat "${NIFI_HOME}/conf/nifi.temp" > "${NIFI_HOME}/conf/nifi.properties" + cat "${NIFI_HOME}/conf/authorizers.temp" > "${NIFI_HOME}/conf/authorizers.xml" {{- if .Values.auth.ldap.enabled }} cat "${NIFI_HOME}/conf/authorizers.temp" > "${NIFI_HOME}/conf/authorizers.xml" cat "${NIFI_HOME}/conf/login-identity-providers-ldap.xml" > "${NIFI_HOME}/conf/login-identity-providers.xml" -{{- else if .Values.auth.oidc.enabled }} - prop_replace nifi.security.user.login.identity.provider '' - prop_replace nifi.security.user.authorizer managed-authorizer - prop_replace nifi.security.user.oidc.discovery.url {{ .Values.auth.oidc.discoveryUrl }} - prop_replace nifi.security.user.oidc.client.id {{ .Values.auth.oidc.clientId }} - prop_replace nifi.security.user.oidc.client.secret {{ .Values.auth.oidc.clientSecret }} - prop_replace nifi.security.user.oidc.claim.identifying.user {{ .Values.auth.oidc.claimIdentifyingUser }} - xmlstarlet ed --inplace --delete "//authorizers/authorizer[identifier='single-user-authorizer']" "${NIFI_HOME}/conf/authorizers.xml" - xmlstarlet ed --inplace --update "//authorizers/userGroupProvider/property[@name='Users File']" -v './auth-conf/users.xml' "${NIFI_HOME}/conf/authorizers.xml" - xmlstarlet ed --inplace --delete "//authorizers/userGroupProvider/property[@name='Initial User Identity 1']" "${NIFI_HOME}/conf/authorizers.xml" - xmlstarlet ed --inplace \ - --subnode "authorizers/userGroupProvider" --type 'elem' -n 'property' \ - --value {{ .Values.auth.oidc.admin | quote }} \ - --insert "authorizers/userGroupProvider/property[not(@name)]" --type attr -n name \ - --value "Initial User Identity {{ .Values.replicaCount }}" \ - "${NIFI_HOME}/conf/authorizers.xml" - xmlstarlet ed --inplace --update "//authorizers/accessPolicyProvider/property[@name='Initial Admin Identity']" -v {{ .Values.auth.oidc.admin | quote }} "${NIFI_HOME}/conf/authorizers.xml" - xmlstarlet ed --inplace --update "//authorizers/accessPolicyProvider/property[@name='Authorizations File']" -v './auth-conf/authorizations.xml' "${NIFI_HOME}/conf/authorizers.xml" - {{- if .Values.properties.isNode }} - xmlstarlet ed --inplace --delete "authorizers/accessPolicyProvider/property[@name='Node Identity 1']" "${NIFI_HOME}/conf/authorizers.xml" - {{ range untilStep 0 (int .Values.replicaCount) 1 }} - xmlstarlet ed --inplace \ - --subnode "authorizers/accessPolicyProvider" --type 'elem' -n 'property' \ - --value "CN={{ template "apache-nifi.fullname" $ }}-{{ . }}.{{ template "apache-nifi.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc.{{ $.Values.certManager.clusterDomain }}, OU=NIFI" \ - --insert "authorizers/accessPolicyProvider/property[not(@name)]" --type attr -n name \ - --value "Node Identity {{ . }}" \ - "${NIFI_HOME}/conf/authorizers.xml" - xmlstarlet ed --inplace \ - --subnode "authorizers/userGroupProvider" --type 'elem' -n 'property' \ - --value "CN={{ template "apache-nifi.fullname" $ }}-{{ . }}.{{ template "apache-nifi.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc.{{ $.Values.certManager.clusterDomain }}, OU=NIFI" \ - --insert "authorizers/userGroupProvider/property[not(@name)]" --type attr -n name \ - --value "Initial User Identity {{ . }}" \ - "${NIFI_HOME}/conf/authorizers.xml" - {{/* range untilStep 0 (int .Values.replicaCount ) 1 */}}{{ end }} - {{- end }} {{- else if .Values.auth.clientAuth.enabled }} cat "${NIFI_HOME}/conf/authorizers.temp" > "${NIFI_HOME}/conf/authorizers.xml" xmlstarlet ed --inplace --delete "//authorizers/authorizer[identifier='single-user-authorizer']" "${NIFI_HOME}/conf/authorizers.xml" @@ -205,22 +171,6 @@ spec: {{- end }} {{- if .Values.certManager.enabled }} - xmlstarlet ed --inplace --delete "authorizers/accessPolicyProvider/property[@name='Node Identity 1']" "${NIFI_HOME}/conf/authorizers.xml" -{{ range untilStep 0 (int .Values.replicaCount) 1 }} - xmlstarlet ed --inplace \ - --subnode "authorizers/accessPolicyProvider" --type 'elem' -n 'property' \ - --value "CN={{ template "apache-nifi.fullname" $ }}-{{ . }}.{{ template "apache-nifi.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc.{{ $.Values.certManager.clusterDomain }}, OU=NIFI" \ - --insert "authorizers/accessPolicyProvider/property[not(@name)]" --type attr -n name \ - --value "Node Identity {{ . }}" \ - "${NIFI_HOME}/conf/authorizers.xml" - xmlstarlet ed --inplace \ - --subnode "authorizers/userGroupProvider" --type 'elem' -n 'property' \ - --value "CN={{ template "apache-nifi.fullname" $ }}-{{ . }}.{{ template "apache-nifi.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc.{{ $.Values.certManager.clusterDomain }}, OU=NIFI" \ - --insert "authorizers/userGroupProvider/property[not(@name)]" --type attr -n name \ - --value "Initial User Identity {{ . }}" \ - "${NIFI_HOME}/conf/authorizers.xml" -{{/* range untilStep 0 (int .Values.replicaCount ) 1 */}}{{ end }} - prop_replace nifi.security.keystore "${NIFI_HOME}/tls/keystore.jks" prop_replace nifi.security.keystoreType JKS prop_replace nifi.security.keystorePasswd "{{ .Values.certManager.keystorePasswd }}" @@ -451,7 +401,7 @@ spec: {{- end }} periodSeconds: 20 tcpSocket: - port: {{ .Values.service.httpPort }} + port: {{ .Values.service.httpsPort | default .Values.service.httpPort }} # exec: # command: # - bash @@ -570,6 +520,9 @@ spec: - name: "flow-content" mountPath: /opt/nifi/data/flow.xml subPath: "flow.xml" + - name: "logback-xml" + mountPath: /opt/nifi/nifi-current/conf/logback.xml + subPath: "logback.xml" {{- range $secret := .Values.secrets }} {{- if $secret.mountPath }} {{- if $secret.keys }} @@ -841,6 +794,12 @@ spec: items: - key: "flow.xml" path: "flow.xml" + - name: "logback-xml" + configMap: + name: {{ template "apache-nifi.fullname" . }}-config + items: + - key: "logback.xml" + path: "logback.xml" {{- if .Values.certManager.enabled }} - name: secret-reader-token secret: From 82e18bdf69ae544cacb295f98da4973f1cc4f767 Mon Sep 17 00:00:00 2001 From: Viktor Zhemchuzhnikov Date: Mon, 30 Sep 2024 17:26:50 +0900 Subject: [PATCH 5/8] fix: add traefik conf to values --- values.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/values.yaml b/values.yaml index 846d6bc7..9eb94a77 100644 --- a/values.yaml +++ b/values.yaml @@ -256,6 +256,11 @@ ingress: # serviceName: processor02 # type: Prefix +traefik: + serversTransport: + enabled: false + insecureSkipVerify: true + # Amount of memory to give the NiFi java heap jvmMemory: 2g From 0ede5d83b835e2b5744eb65f0fdd391b19b9d8b7 Mon Sep 17 00:00:00 2001 From: Viktor Zhemchuzhnikov Date: Tue, 1 Oct 2024 12:16:18 +0900 Subject: [PATCH 6/8] feat: add traefik tls options --- templates/ingress-processors.yaml | 6 +++++- templates/ingress.yaml | 6 +++++- templates/tls_options.yaml | 11 +++++++++++ values.yaml | 4 ++++ 4 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 templates/tls_options.yaml diff --git a/templates/ingress-processors.yaml b/templates/ingress-processors.yaml index 08522e62..0ad6615c 100644 --- a/templates/ingress-processors.yaml +++ b/templates/ingress-processors.yaml @@ -11,8 +11,12 @@ metadata: chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" release: {{ .Release.Name | quote }} heritage: {{ .Release.Service | quote }} -{{- with .Values.ingress.processors.annotations }} annotations: +{{- if .Values.traefik.tlsOptions.enabled }} + traefik.ingress.kubernetes.io/router.tls: "true" + traefik.ingress.kubernetes.io/router.tls.options: {{ .Release.Namespace}}-{{ $fullName }}-tls-option +{{- end }} +{{- with .Values.ingress.processors.annotations }} {{ toYaml . | indent 4 }} {{- end }} spec: diff --git a/templates/ingress.yaml b/templates/ingress.yaml index 7bd62649..685aeb82 100644 --- a/templates/ingress.yaml +++ b/templates/ingress.yaml @@ -13,8 +13,12 @@ metadata: chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" release: {{ .Release.Name | quote }} heritage: {{ .Release.Service | quote }} -{{- with .Values.ingress.annotations }} annotations: +{{- if .Values.traefik.tlsOptions.enabled }} + traefik.ingress.kubernetes.io/router.tls: "true" + traefik.ingress.kubernetes.io/router.tls.options: {{ .Release.Namespace}}-{{ $fullName }}-tls-option +{{- end }} +{{- with .Values.ingress.annotations }} {{ toYaml . | indent 4 }} {{- end }} spec: diff --git a/templates/tls_options.yaml b/templates/tls_options.yaml new file mode 100644 index 00000000..d098f32f --- /dev/null +++ b/templates/tls_options.yaml @@ -0,0 +1,11 @@ +{{- if .Values.traefik.tlsOptions.enabled }} +apiVersion: traefik.io/v1alpha1 +kind: TLSOption +metadata: + name: {{ template "nifi.fullname" . }}-tls-option + namespace: {{ .Release.Namespace }} + +spec: + maxVersion: {{ .Values.traefik.tlsOptions.maxVersion | default "VersionTLS12" }} + minVersion: {{ .Values.traefik.tlsOptions.minVersion | default "VersionTLS12" }} +{{- end }} \ No newline at end of file diff --git a/values.yaml b/values.yaml index 9eb94a77..d1ce59d0 100644 --- a/values.yaml +++ b/values.yaml @@ -260,6 +260,10 @@ traefik: serversTransport: enabled: false insecureSkipVerify: true + tlsOptions: + enabled: false + minVersion: VersionTLS12 + maxVersion: VersionTLS12 # Amount of memory to give the NiFi java heap jvmMemory: 2g From f2fe93aef7c626198f7890050e4aa2c2fb2c2bab Mon Sep 17 00:00:00 2001 From: Viktor Zhemchuzhnikov Date: Tue, 1 Oct 2024 12:18:09 +0900 Subject: [PATCH 7/8] fix: tls options name --- templates/tls_options.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/tls_options.yaml b/templates/tls_options.yaml index d098f32f..290299e7 100644 --- a/templates/tls_options.yaml +++ b/templates/tls_options.yaml @@ -2,7 +2,7 @@ apiVersion: traefik.io/v1alpha1 kind: TLSOption metadata: - name: {{ template "nifi.fullname" . }}-tls-option + name: {{ template "apache-nifi.fullname" . }}-tls-option namespace: {{ .Release.Namespace }} spec: From beb45c19ab4d309ded82b15bf2db4d6bff5c6569 Mon Sep 17 00:00:00 2001 From: Viktor Zhemchuzhnikov Date: Tue, 1 Oct 2024 12:21:29 +0900 Subject: [PATCH 8/8] fix: tls options annotations in ingress --- templates/ingress-processors.yaml | 2 +- templates/ingress.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/ingress-processors.yaml b/templates/ingress-processors.yaml index 0ad6615c..19cfe4f9 100644 --- a/templates/ingress-processors.yaml +++ b/templates/ingress-processors.yaml @@ -14,7 +14,7 @@ metadata: annotations: {{- if .Values.traefik.tlsOptions.enabled }} traefik.ingress.kubernetes.io/router.tls: "true" - traefik.ingress.kubernetes.io/router.tls.options: {{ .Release.Namespace}}-{{ $fullName }}-tls-option + traefik.ingress.kubernetes.io/router.tls.options: {{ .Release.Namespace}}-{{ $fullName }}-tls-option@kubernetescrd {{- end }} {{- with .Values.ingress.processors.annotations }} {{ toYaml . | indent 4 }} diff --git a/templates/ingress.yaml b/templates/ingress.yaml index 685aeb82..3a0dccd5 100644 --- a/templates/ingress.yaml +++ b/templates/ingress.yaml @@ -16,7 +16,7 @@ metadata: annotations: {{- if .Values.traefik.tlsOptions.enabled }} traefik.ingress.kubernetes.io/router.tls: "true" - traefik.ingress.kubernetes.io/router.tls.options: {{ .Release.Namespace}}-{{ $fullName }}-tls-option + traefik.ingress.kubernetes.io/router.tls.options: {{ .Release.Namespace}}-{{ $fullName }}-tls-option@kubernetescrd {{- end }} {{- with .Values.ingress.annotations }} {{ toYaml . | indent 4 }}