Add JMX Trans deployment (strimzi#2474)

* feat: Add jmxtrans deployment Added a jmxtrans deployment which will obtain JMX metrics automatically from the kafka brokers. It will also automatically roll when security or kafka brokers are updated. Signed-off-by: Julian Goh <[email protected]> * refactor: Address PR comments Added more documentations and removed redundant variables. Signed-off-by: Julian Goh <[email protected]> * refactor: Address PR comments Implement suggested doc changes. Signed-off-by: Julian Goh <[email protected]> * docs: Add references Signed-off-by: Julian Goh <[email protected]> * refactor: Address PR comments Restructure CRD, built custom Dockerfile for new JmxTrans Signed-off-by: Julian Goh <[email protected]> * refactor: Address PR comments Renamed variables and classes and got rid of unnecessary values. Added JmxTrans image in the Makefiles Signed-off-by: Julian Goh <[email protected]> * fix: add jmxtrans image into files Add jmxtrans image into the helm charts. Signed-off-by: Julian Goh <[email protected]> * refactor: Moved inner classes and rewording Moved inner static classes outside of the main class and moved it into its own class. Added unit tests to test deserializing of the classes. Improved wording on classes. Also added more logging Signed-off-by: Julian Goh <[email protected]> * Review commennts Signed-off-by: Jakub Scholz <[email protected]>
carlobongiovanni · Jan 29, 2020 · 7e2014d · 7e2014d
1 parent f033271
commit 7e2014d
Show file tree

Hide file tree

Showing 44 changed files with 1,816 additions and 18 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,7 @@
 
 # CHANGELOG
+## 0.17.0
+* Add Jmxtrans deployment
 
 ## 0.17.0
 

diff --git a/Makefile b/Makefile
@@ -48,6 +48,7 @@ release_version:
 	$(FIND) ./install -name '*.yaml' -type f -exec $(SED) -i '/value: "\?strimzi\/kafka-bridge:[a-zA-Z0-9_.-]\+"\?/s/strimzi\/kafka-bridge:[a-zA-Z0-9_.-]\+/strimzi\/kafka-bridge:$(BRIDGE_VERSION)/g' {} \;
 	$(FIND) ./install -name '*.yaml' -type f -exec $(SED) -i '/value: "\?strimzi\/kafka:[a-zA-Z0-9_.-]\+"\?/s/strimzi\/kafka:[a-zA-Z0-9_.-]\+-kafka-\([0-9.]\+\)/strimzi\/kafka:$(RELEASE_VERSION)-kafka-\1/g' {} \;
 	$(FIND) ./install -name '*.yaml' -type f -exec $(SED) -i '/[0-9.]\+=strimzi\/kafka[a-zA-Z0-9_.-]\?\+:[a-zA-Z0-9_.-]\+-kafka-[0-9.]\+"\?/s/:[a-zA-Z0-9_.-]\+-kafka-\([0-9.]\+\)/:$(RELEASE_VERSION)-kafka-\1/g' {} \;
+	$(FIND) ./install -name '*.yaml' -type f -exec $(SED) -i '/value: "\?strimzi\/jmxtrans:[a-zA-Z0-9_.-]\+"\?/s/strimzi\/jmxtrans:[a-zA-Z0-9_.-]\+/strimzi\/jmxtrans:$(RELEASE_VERSION)/g' {} \;
 	# Set Kafka Bridge version to its own version
 	$(FIND) ./install -name '*.yaml' -type f -exec $(SED) -i '/value: "\?strimzi\/kafka-bridge:[a-zA-Z0-9_.-]\+"\?/s/strimzi\/kafka-bridge:[a-zA-Z0-9_.-]\+/strimzi\/kafka-bridge:$(BRIDGE_VERSION)/g' {} \;
 

diff --git a/api/src/main/java/io/strimzi/api/kafka/model/JmxTransResources.java b/api/src/main/java/io/strimzi/api/kafka/model/JmxTransResources.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright Strimzi authors.
+ * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+ */
+package io.strimzi.api.kafka.model;
+
+/**
+ * Encapsulates the naming scheme used for the resources which the Cluster Operator manages for a
+ * {@code JmxTrans} instance.
+ */
+public class JmxTransResources {
+    protected JmxTransResources() { }
+    /**
+     * Returns the name of the JmxTrans {@code Deployment}.
+     * @param kafkaClusterName  The {@code metadata.name} of the {@code Kafka} resource.
+     * @return The name of the corresponding JmxTrans {@code Deployment}.
+     */
+    public static String deploymentName(String kafkaClusterName) {
+        return kafkaClusterName + "-kafka-jmx-trans";
+    }
+    /**
+     * Returns the name of the JmxTrans {@code ServiceAccount}.
+     * @param kafkaClusterName  The {@code metadata.name} of the {@code Kafka} resource.
+     * @return The name of the corresponding JmxTrans {@code ServiceAccount}.
+     */
+    public static String serviceAccountName(String kafkaClusterName) {
+        return deploymentName(kafkaClusterName);
+    }
+
+}
diff --git a/api/src/main/java/io/strimzi/api/kafka/model/JmxTransSpec.java b/api/src/main/java/io/strimzi/api/kafka/model/JmxTransSpec.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright Strimzi authors.
+ * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+ */
+package io.strimzi.api.kafka.model;
+
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+import io.fabric8.kubernetes.api.model.ResourceRequirements;
+import io.strimzi.api.kafka.model.template.JmxTransOutputDefinitionTemplate;
+import io.strimzi.api.kafka.model.template.JmxTransQueryTemplate;
+import io.strimzi.crdgenerator.annotations.Description;
+import io.sundr.builder.annotations.Buildable;
+import lombok.EqualsAndHashCode;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Representation for options to be passed into setting up the JmxTrans.
+ */
+@Buildable(
+        editableEnabled = false,
+        generateBuilderPackage = false,
+        builderPackage = "io.fabric8.kubernetes.api.builder"
+)
+@JsonInclude(JsonInclude.Include.NON_DEFAULT)
+@JsonPropertyOrder({"image", "outputDefinitions", "logLevel", "kafkaQueries", "resources"})
+@EqualsAndHashCode
+public class JmxTransSpec implements UnknownPropertyPreserving, Serializable {
+    public static final int DEFAULT_HEALTHCHECK_DELAY = 15;
+    public static final int DEFAULT_HEALTHCHECK_TIMEOUT = 5;
+
+    private static final long serialVersionUID = 1L;
+    protected String image;
+    private String logLevel;
+    private List<JmxTransOutputDefinitionTemplate> outputDefinitions = null;
+    private List<JmxTransQueryTemplate> kafkaQueries = null;
+
+    private ResourceRequirements resources;
+
+    private Map<String, Object> additionalProperties = new HashMap<>(0);
+
+    @Description("The image to use for the JmxTrans")
+    public String getImage() {
+        return image;
+    }
+
+    public void setImage(String image) {
+        this.image = image;
+    }
+
+    @Description("Sets the logging level of the JmxTrans deployment." +
+            "For more information see, https://github.com/jmxtrans/jmxtrans-agent/wiki/Troubleshooting[JmxTrans Logging Level]")
+    public String getLogLevel() {
+        return logLevel;
+    }
+
+    public void setLogLevel(String logLevel) {
+        this.logLevel = logLevel;
+    }
+
+    @JsonProperty(required = true)
+    @JsonInclude(JsonInclude.Include.NON_DEFAULT)
+    @Description("Defines the output hosts that will be referenced later on. " +
+            "For more information on these properties see, xref:type-JmxTransOutputDefinitionTemplate-reference[`JmxTransOutputDefinitionTemplate` schema reference].")
+    public List<JmxTransOutputDefinitionTemplate> getOutputDefinitions() {
+        return outputDefinitions;
+    }
+
+    public void setOutputDefinitions(List<JmxTransOutputDefinitionTemplate> outputDefinitions) {
+        this.outputDefinitions = outputDefinitions;
+    }
+
+    @JsonProperty(required = true)
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    @Description("Queries to send to the Kafka brokers to define what data should be read from each broker. " +
+            "For more information on these properties see, xref:type-JmxTransQueryTemplate-reference[`JmxTransQueryTemplate` schema reference].")
+    public List<JmxTransQueryTemplate> getKafkaQueries() {
+        return kafkaQueries;
+    }
+
+    public void setKafkaQueries(List<JmxTransQueryTemplate> kafkaQueries) {
+        this.kafkaQueries = kafkaQueries;
+    }
+
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    @Description("CPU and memory resources to reserve.")
+    public ResourceRequirements getResources() {
+        return resources;
+    }
+
+    public void setResources(ResourceRequirements resources) {
+        this.resources = resources;
+    }
+
+    @Override
+    public Map<String, Object> getAdditionalProperties() {
+        return additionalProperties;
+    }
+
+    @Override
+    public void setAdditionalProperty(String name, Object value) {
+        this.additionalProperties.put(name, value);
+    }
+
+}
diff --git a/api/src/main/java/io/strimzi/api/kafka/model/KafkaJmxOptions.java b/api/src/main/java/io/strimzi/api/kafka/model/KafkaJmxOptions.java
@@ -47,4 +47,3 @@ public void setAdditionalProperty(String name, Object value) {
         this.additionalProperties.put(name, value);
     }
 }
-
diff --git a/api/src/main/java/io/strimzi/api/kafka/model/KafkaSpec.java b/api/src/main/java/io/strimzi/api/kafka/model/KafkaSpec.java
@@ -41,6 +41,7 @@ public class KafkaSpec implements UnknownPropertyPreserving, Serializable {
     private TopicOperatorSpec topicOperator;
     private EntityOperatorSpec entityOperator;
     private CertificateAuthority clusterCa;
+    private JmxTransSpec jmxTrans;
     private KafkaExporterSpec kafkaExporter;
 
     private CertificateAuthority clientsCa;
@@ -113,6 +114,16 @@ public List<String> getMaintenanceTimeWindows() {
         return maintenanceTimeWindows;
     }
 
+    @Description("Configuration for JmxTrans. When the property is present a JmxTrans deployment is created for gathering JMX metrics from each Kafka broker. " +
+            "For more information see https://github.com/jmxtrans/jmxtrans[JmxTrans GitHub]")
+    public JmxTransSpec getJmxTrans() {
+        return jmxTrans;
+    }
+
+    public void setJmxTrans(JmxTransSpec jmxTrans) {
+        this.jmxTrans = jmxTrans;
+    }
+
     public void setMaintenanceTimeWindows(List<String> maintenanceTimeWindows) {
         this.maintenanceTimeWindows = maintenanceTimeWindows;
     }

diff --git a/api/src/main/java/io/strimzi/api/kafka/model/template/JmxTransOutputDefinitionTemplate.java b/api/src/main/java/io/strimzi/api/kafka/model/template/JmxTransOutputDefinitionTemplate.java
@@ -0,0 +1,118 @@
+/*
+ * Copyright Strimzi authors.
+ * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+ */
+package io.strimzi.api.kafka.model.template;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+import io.strimzi.api.kafka.model.UnknownPropertyPreserving;
+import io.strimzi.crdgenerator.annotations.Description;
+import io.sundr.builder.annotations.Buildable;
+import lombok.EqualsAndHashCode;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Representation for options to define where and how information will be pushed to remote sources of information
+ */
+@Buildable(
+        editableEnabled = false,
+        generateBuilderPackage = false,
+        builderPackage = "io.fabric8.kubernetes.api.builder"
+)
+@JsonInclude(JsonInclude.Include.NON_DEFAULT)
+@JsonPropertyOrder({"outputType", "host", "port", "flushDelayInSeconds", "typeNames", "name"})
+@EqualsAndHashCode
+public class JmxTransOutputDefinitionTemplate implements Serializable, UnknownPropertyPreserving {
+
+    private static final long serialVersionUID = 1L;
+
+    private String outputType;
+    private String host;
+    private Integer port;
+    private Integer flushDelayInSeconds;
+    private String name;
+    private List<String> typeNames;
+
+    private Map<String, Object> additionalProperties = new HashMap<>(0);
+
+    @JsonProperty(value = "outputType", required = true)
+    @Description("Template for setting the format of the data that will be pushed." +
+            "For more information see https://github.com/jmxtrans/jmxtrans/wiki/OutputWriters[JmxTrans OutputWriters]")
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    public String getOutputType() {
+        return outputType;
+    }
+
+    public void setOutputType(String outputType) {
+        this.outputType = outputType;
+    }
+
+    @Description("The DNS/hostname of the remote host that the data is pushed to.")
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    public String getHost() {
+        return host;
+    }
+
+    public void setHost(String host) {
+        this.host = host;
+    }
+
+    @Description("The port of the remote host that the data is pushed to.")
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    public Integer getPort() {
+        return port;
+    }
+
+    public void setPort(Integer port) {
+        this.port = port;
+    }
+
+    @Description("How many seconds the JmxTrans waits before pushing a new set of data out.")
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    public Integer getFlushDelayInSeconds() {
+        return flushDelayInSeconds;
+    }
+
+    public void setFlushDelayInSeconds(Integer flushDelayInSeconds) {
+        this.flushDelayInSeconds = flushDelayInSeconds;
+    }
+
+    @Description("Template for filtering data to be included in response to a wildcard query. " +
+            "For more information see https://github.com/jmxtrans/jmxtrans/wiki/Queries[JmxTrans queries]")
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    public List<String> getTypeNames() {
+        return typeNames;
+    }
+
+    public void setTypeNames(List<String> typeNames) {
+        this.typeNames = typeNames;
+    }
+
+    @Description("Template for setting the name of the output definition. This is used to identify where to send " +
+            "the results of queries should be sent.")
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    @JsonProperty(required = true)
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    @Override
+    public Map<String, Object> getAdditionalProperties() {
+        return this.additionalProperties;
+    }
+
+    @Override
+    public void setAdditionalProperty(String name, Object value) {
+        this.additionalProperties.put(name, value);
+    }
+}
diff --git a/api/src/main/java/io/strimzi/api/kafka/model/template/JmxTransQueryTemplate.java b/api/src/main/java/io/strimzi/api/kafka/model/template/JmxTransQueryTemplate.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright Strimzi authors.
+ * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
+ */
+package io.strimzi.api.kafka.model.template;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+import io.strimzi.api.kafka.model.UnknownPropertyPreserving;
+import io.strimzi.crdgenerator.annotations.Description;
+import io.sundr.builder.annotations.Buildable;
+import lombok.EqualsAndHashCode;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Buildable(
+        editableEnabled = false,
+        generateBuilderPackage = false,
+        builderPackage = "io.fabric8.kubernetes.api.builder"
+)
+@JsonInclude(JsonInclude.Include.NON_DEFAULT)
+@JsonPropertyOrder({
+        "targetMBean", "attributes", "outputs"})
+@EqualsAndHashCode
+public class JmxTransQueryTemplate implements Serializable, UnknownPropertyPreserving {
+    private String targetMBean;
+    private List<String> attributes;
+    private List<String> outputs;
+
+    private static final long serialVersionUID = 1L;
+    private Map<String, Object> additionalProperties = new HashMap<>(0);
+
+    @JsonProperty(required = true)
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    @Description("If using wildcards instead of a specific MBean then the data is gathered from multiple MBeans. " +
+            "Otherwise if specifying an MBean then data is gathered from that specified MBean.")
+    public String getTargetMBean() {
+        return targetMBean;
+    }
+
+    public void setTargetMBean(String targetMBean) {
+        this.targetMBean = targetMBean;
+    }
+
+    @JsonProperty(required = true)
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    @Description("Determine which attributes of the targeted MBean should be included")
+    public List<String> getAttributes() {
+        return attributes;
+    }
+
+    public void setAttributes(List<String> attributes) {
+        this.attributes = attributes;
+    }
+
+    @JsonProperty(required = true)
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    @Description("List of the names of output definitions specified in the spec.kafka.jmxTrans.outputDefinitions that have defined where JMX metrics are pushed to, and in which data format")
+    public List<String> getOutputs() {
+        return outputs;
+    }
+
+    public void setOutputs(List<String> outputs) {
+        this.outputs = outputs;
+    }
+
+    @Override
+    public Map<String, Object> getAdditionalProperties() {
+        return this.additionalProperties;
+    }
+
+    @Override
+    public void setAdditionalProperty(String name, Object value) {
+        this.additionalProperties.put(name, value);
+    }
+
+}
Original file line number	Diff line number	Diff line change
Expand Up		@@ -47,4 +47,3 @@ public void setAdditionalProperty(String name, Object value) {
		this.additionalProperties.put(name, value);
		}
		}