wiremind · maxime1907 · May 16, 2024
diff --git a/.github/workflows/build_release.yml b/.github/workflows/build_release.yml
@@ -24,6 +24,8 @@ jobs:
         os:
           - ubuntu-22.04
         context:
+          - target: druid-auto
+            suffix: -auto
           - target: druid-micro
             suffix: -micro
           - target: druid-nano

diff --git a/Containerfile b/Containerfile
@@ -1,10 +1,13 @@
-FROM openjdk:8-jre AS builder
+FROM eclipse-temurin:11 AS builder
 
 ARG DRUID_VERSION=26.0.0
 ENV DRUID_VERSION=$DRUID_VERSION
 
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends bash=5.1-2+deb11u1 perl=5.32.1-4+deb11u2 \
+    && apt-get install -y --no-install-recommends \
+    git=1:2.34* \
+    libfindbin-libs-perl=2* \
+    python3=3.10* \
     && rm -rf /var/lib/apt/lists/*
 
 RUN wget --progress=dot:giga http://apache.crihan.fr/dist/druid/${DRUID_VERSION}/apache-druid-${DRUID_VERSION}-bin.tar.gz \
@@ -17,6 +20,10 @@ RUN mv /apache-druid-${DRUID_VERSION} /app
 
 WORKDIR /app
 
+COPY patches/druid /tmp/patches
+
+RUN git apply /tmp/patches/*
+
 FROM builder as druid-micro
 
 ENTRYPOINT ["/app/bin/start-micro-quickstart"]
@@ -25,4 +32,8 @@ FROM builder as druid-nano
 
 ENTRYPOINT ["/app/bin/start-nano-quickstart"]
 
+FROM builder as druid-auto
+
+ENTRYPOINT ["/app/bin/start-druid"]
+
 EXPOSE 8081 8082 8083 8888 8091
diff --git a/README.md b/README.md
@@ -1,16 +1,13 @@
-Micro & Nano druid, for E2E tests
-=======================================
+# Micro & Nano druid, for E2E tests
 
 Dockerfiles to run nano and micro setups of [druid](https://github.com/apache/druid) for e2e tests. Some configurations are applied to maximize the response time of the Druid docker but should in no way refer to a production cluster.
 
-Arguments
----------
+## Arguments
 
 Arguments that can be set while building:
 - `DRUID_VERSION`: You can also change the default value in the Containerfile.
 
-Building
---------
+## Building
 
 ```
 export DRUID_VERSION=XXX # 26.0.0 for example.
@@ -22,22 +19,14 @@ docker build -t micro-druid:${DRUID_VERSION}-micro --build-arg DRUID_VERSION=${D
 docker build -t micro-druid:${DRUID_VERSION}-nano --build-arg DRUID_VERSION=${DRUID_VERSION} -f ./Containerfile --target druid-nano .
 ```
 
-Required hardware
-------------------
+## Configuration
+The `conf` folder must stay the same as the upstream one that can be found here:
+- https://github.com/apache/druid/tree/26.0.1/examples/conf
 
-Nano
-*********
-
-* 1 CPU
-* 4GB RAM
-* Launch command: `bin/start-nano-quickstart`
-* Configuration directory: `conf/druid/single-server/nano-quickstart`
-
-
-Micro
-*********
+Run this script when updating the druid version:
+```bash
+./scripts/download_upstream_conf.sh
+```
 
-* 4 CPU
-* 16GB RAM
-* Launch command: `bin/start-micro-quickstart`
-* Configuration directory: `conf/druid/single-server/micro-quickstart`
+## Required hardware
+- https://github.com/apache/druid/blob/26.0.1/docs/operations/single-server.md
diff --git a/conf/druid/.DS_Store b/conf/druid/.DS_Store
diff --git a/conf/druid/auto/_common/common.jvm.config b/conf/druid/auto/_common/common.jvm.config
@@ -0,0 +1,7 @@
+-server
+-XX:+ExitOnOutOfMemoryError
+-XX:+UseG1GC
+-Duser.timezone=UTC
+-Dfile.encoding=UTF-8
+-Djava.io.tmpdir=var/tmp
+-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager
diff --git a/conf/druid/auto/_common/common.runtime.properties b/conf/druid/auto/_common/common.runtime.properties
@@ -0,0 +1,161 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Extensions specified in the load list will be loaded by Druid
+# We are using local fs for deep storage - not recommended for production - use S3, HDFS, or NFS instead
+# We are using local derby for the metadata store - not recommended for production - use MySQL or Postgres instead
+
+# If you specify `druid.extensions.loadList=[]`, Druid won't load any extension from file system.
+# If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory.
+# More info: https://druid.apache.org/docs/latest/operations/including-extensions.html
+druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-multi-stage-query", "postgresql-metadata-storage", "druid-s3-extensions"]
+
+# If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory
+# and uncomment the line below to point to your directory.
+#druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies
+
+
+#
+# Hostname
+#
+druid.host=localhost
+
+#
+# Logging
+#
+
+# Log all runtime properties on startup. Disable to avoid logging properties on startup:
+druid.startup.logging.logProperties=true
+
+#
+# Zookeeper
+#
+
+druid.zk.service.host=localhost
+druid.zk.paths.base=/druid
+
+#
+# Metadata storage
+#
+
+# For Derby server on your Druid Coordinator (only viable in a cluster with a single Coordinator, no fail-over):
+druid.metadata.storage.type=derby
+druid.metadata.storage.connector.connectURI=jdbc:derby://localhost:1527/var/druid/metadata.db;create=true
+druid.metadata.storage.connector.host=localhost
+druid.metadata.storage.connector.port=1527
+
+# For MySQL (make sure to include the MySQL JDBC driver on the classpath):
+#druid.metadata.storage.type=mysql
+#druid.metadata.storage.connector.connectURI=jdbc:mysql://db.example.com:3306/druid
+#druid.metadata.storage.connector.user=...
+#druid.metadata.storage.connector.password=...
+
+# For PostgreSQL:
+#druid.metadata.storage.type=postgresql
+#druid.metadata.storage.connector.connectURI=jdbc:postgresql://db.example.com:5432/druid
+#druid.metadata.storage.connector.user=...
+#druid.metadata.storage.connector.password=...
+
+#
+# Deep storage
+#
+
+# For local disk (only viable in a cluster if this is a network mount):
+# druid.storage.type=local
+# druid.storage.storageDirectory=var/druid/segments
+
+# For HDFS:
+#druid.storage.type=hdfs
+#druid.storage.storageDirectory=/druid/segments
+
+# For S3:
+druid.storage.type=s3
+druid.storage.bucket=storage-bucket
+druid.storage.baseKey=druid/segments
+druid.s3.accessKey=access_key
+druid.s3.secretKey=secret_key
+
+druid.s3.endpoint.url=http://localhost:9000
+druid.s3.enablePathStyleAccess=true
+
+#
+# Indexing service logs
+#
+
+# For local disk (only viable in a cluster if this is a network mount):
+# druid.indexer.logs.type=file
+# druid.indexer.logs.directory=var/druid/indexing-logs
+
+# For HDFS:
+#druid.indexer.logs.type=hdfs
+#druid.indexer.logs.directory=/druid/indexing-logs
+
+# For S3:
+druid.indexer.logs.type=s3
+druid.indexer.logs.s3Bucket=logs-bucket
+druid.indexer.logs.s3Prefix=druid/indexing-logs
+
+#
+# Service discovery
+#
+
+druid.selectors.indexing.serviceName=druid/overlord
+druid.selectors.coordinator.serviceName=druid/coordinator
+
+#
+# Monitoring
+#
+
+druid.monitoring.monitors=["org.apache.druid.java.util.metrics.JvmMonitor"]
+druid.emitter=noop
+druid.emitter.logging.logLevel=info
+
+# Storage type of double columns
+# ommiting this will lead to index double as float at the storage layer
+
+druid.indexing.doubleStorage=double
+
+#
+# Security
+#
+druid.server.hiddenProperties=["druid.s3.accessKey","druid.s3.secretKey","druid.metadata.storage.connector.password", "password", "key", "token", "pwd"]
+
+
+#
+# SQL
+#
+druid.sql.enable=true
+
+# Planning SQL query when there is aggregate distinct in the statement
+druid.sql.planner.useGroupingSetForExactDistinct=true
+
+#
+# Lookups
+#
+druid.lookup.enableLookupSyncOnStartup=false
+
+#
+# Expression processing config
+#
+druid.expressions.useStrictBooleans=true
+
+#
+# Http client
+#
+druid.global.http.eagerInitialization=false
diff --git a/conf/druid/auto/_common/log4j2.xml b/conf/druid/auto/_common/log4j2.xml
@@ -0,0 +1,86 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements.  See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership.  The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License.  You may obtain a copy of the License at
+ ~
+ ~   http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied.  See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+-->
+
+<Configuration status="WARN">
+  <Properties>
+    <!-- to change log directory, set DRUID_LOG_DIR environment variable to your directory before launching Druid -->
+    <Property name="druid.log.path" value="log" />
+  </Properties>
+
+  <Appenders>
+    <Console name="Console" target="SYSTEM_OUT">
+      <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/>
+    </Console>
+
+    <!-- Rolling Files-->
+    <RollingRandomAccessFile name="FileAppender"
+                             fileName="${sys:druid.log.path}/${sys:druid.node.type}.log"
+                             filePattern="${sys:druid.log.path}/${sys:druid.node.type}.%d{yyyyMMdd}.log">
+      <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/>
+      <Policies>
+        <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
+      </Policies>
+      <DefaultRolloverStrategy>
+        <Delete basePath="${sys:druid.log.path}/" maxDepth="1">
+          <IfFileName glob="*.log" />
+          <IfLastModified age="7d" />
+        </Delete>
+      </DefaultRolloverStrategy>
+    </RollingRandomAccessFile>
+
+  </Appenders>
+
+  <Loggers>
+    <Root level="info">
+      <AppenderRef ref="FileAppender"/>
+    </Root>
+
+    <!-- Set level="debug" to see stack traces for query errors -->
+    <Logger name="org.apache.druid.server.QueryResource" level="info" additivity="false">
+      <Appender-ref ref="FileAppender"/>
+    </Logger>
+    <Logger name="org.apache.druid.server.QueryLifecycle" level="info" additivity="false">
+      <Appender-ref ref="FileAppender"/>
+    </Logger>
+
+    <!-- Set level="debug" or "trace" to see more Coordinator details (segment balancing, load/drop rules, etc) -->
+    <Logger name="org.apache.druid.server.coordinator" level="info" additivity="false">
+      <Appender-ref ref="FileAppender"/>
+    </Logger>
+
+    <!-- Set level="debug" to see low-level details about segments and ingestion -->
+    <Logger name="org.apache.druid.segment" level="info" additivity="false">
+      <Appender-ref ref="FileAppender"/>
+    </Logger>
+
+    <!-- Set level="debug" to see more information about extension initialization -->
+    <Logger name="org.apache.druid.initialization" level="info" additivity="false">
+      <Appender-ref ref="FileAppender"/>
+    </Logger>
+
+    <!-- Quieter logging at startup -->
+    <Logger name="org.skife.config" level="warn" additivity="false">
+      <Appender-ref ref="FileAppender"/>
+    </Logger>
+    <Logger name="com.sun.jersey.guice" level="warn" additivity="false">
+      <Appender-ref ref="FileAppender"/>
+    </Logger>
+  </Loggers>
+</Configuration>
diff --git a/conf/druid/auto/broker/runtime.properties b/conf/druid/auto/broker/runtime.properties
@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+druid.service=druid/broker
+druid.plaintextPort=8082
+
+# HTTP server settings
+# HTTP server thread pool size. Higher values increase peak load on the Broker, but
+# may be useful for high-concurrency workloads.
+# Default is max(10, (Number of processors * 17) / 16 + 2) + 30.
+# druid.server.http.numThreads=N
+
+# HTTP client settings
+# Connection pool size from the Broker to each data server. May be useful to
+# raise this for high-concurrency workloads.
+# druid.broker.http.numConnections=20
+
+# Processing threads and buffers
+# Determined automatically based on available memory. For details on how to manually set parameters:
+# https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#guidelines-for-processing-threads-and-buffers
+druid.processing.tmpDir=var/druid/processing
+
+# Query cache disabled -- push down caching and merging instead
+druid.broker.cache.useCache=false
+druid.broker.cache.populateCache=false