From e6fd2162516b05bd35272a8d662c71f3eadcd6db Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Tue, 26 Aug 2025 13:05:59 +0800 Subject: [PATCH 1/3] HADOOP-19659. Upgrade Debian 10 to 11 in build env Dockerfile --- BUILDING.txt | 2 +- dev-support/Jenkinsfile | 16 ++--- ...kerfile_debian_10 => Dockerfile_debian_11} | 40 +++++------ .../pkg-resolver/install-pkg-resolver.sh | 2 +- dev-support/docker/pkg-resolver/packages.json | 71 ++++++++++--------- .../docker/pkg-resolver/platforms.json | 2 +- 6 files changed, 67 insertions(+), 66 deletions(-) rename dev-support/docker/{Dockerfile_debian_10 => Dockerfile_debian_11} (71%) diff --git a/BUILDING.txt b/BUILDING.txt index dfbf95f77bd5a..8e7feba5857f8 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -31,7 +31,7 @@ On Linux / Mac: $ ./start-build-env.sh [OS platform] - - [OS Platform] One of [rockylinux_8, debian_10, ubuntu_20, ubuntu_24, windows_10]. + - [OS Platform] One of [rockylinux_8, debian_11, ubuntu_20, ubuntu_24, windows_10]. Default is 'ubuntu_20'. Note: Currently only default ('ubuntu_20') is supported on arm machine diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 2a2b52b20f97e..672061ea09ad9 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -91,10 +91,10 @@ pipeline { ''' } - dir("${WORKSPACE}/debian-10") { + dir("${WORKSPACE}/debian-11") { sh '''#!/usr/bin/env bash - cp -Rp ${WORKSPACE}/src ${WORKSPACE}/debian-10 + cp -Rp ${WORKSPACE}/src ${WORKSPACE}/debian-11 ''' } @@ -157,12 +157,12 @@ pipeline { // C++/C++ build/platform. // This stage serves as a means of cross platform validation, which is // really needed to ensure that any C++ related/platform change doesn't - // break the Hadoop build on Debian 10. - stage ('precommit-run Debian 10') { + // break the Hadoop build on Debian 11. + stage ('precommit-run Debian 11') { environment { - SOURCEDIR = "${WORKSPACE}/debian-10/src" - PATCHDIR = "${WORKSPACE}/debian-10/out" - DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_10" + SOURCEDIR = "${WORKSPACE}/debian-11/src" + PATCHDIR = "${WORKSPACE}/debian-11/out" + DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_11" IS_OPTIONAL = 1 } @@ -182,7 +182,7 @@ pipeline { failure { sh '''#!/usr/bin/env bash - cp -Rp "${WORKSPACE}/debian-10/out" "${WORKSPACE}" + cp -Rp "${WORKSPACE}/debian-11/out" "${WORKSPACE}" ''' archiveArtifacts "out/**" } diff --git a/dev-support/docker/Dockerfile_debian_10 b/dev-support/docker/Dockerfile_debian_11 similarity index 71% rename from dev-support/docker/Dockerfile_debian_10 rename to dev-support/docker/Dockerfile_debian_11 index 71446b27f686b..2dc47eee6ac6e 100644 --- a/dev-support/docker/Dockerfile_debian_10 +++ b/dev-support/docker/Dockerfile_debian_11 @@ -17,7 +17,7 @@ # Dockerfile for installing the necessary dependencies for building Hadoop. # See BUILDING.txt. -FROM debian:10 +FROM debian:11 WORKDIR /root @@ -29,26 +29,24 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras -ENV DEBIAN_FRONTEND noninteractive -ENV DEBCONF_TERSE true +ENV DEBIAN_FRONTEND=noninteractive +ENV DEBCONF_TERSE=true ###### # Platform package dependency resolver ###### COPY pkg-resolver pkg-resolver RUN chmod a+x pkg-resolver/install-pkg-resolver.sh -RUN pkg-resolver/install-pkg-resolver.sh debian:10 +RUN pkg-resolver/install-pkg-resolver.sh debian:11 ###### # Install packages from apt ###### # hadolint ignore=DL3008,SC2046 RUN apt-get -q update \ - && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py debian:10) \ - && echo 'deb http://deb.debian.org/debian bullseye main' >> /etc/apt/sources.list \ - && apt-get -q update \ - && apt-get -q install -y --no-install-recommends -t bullseye $(pkg-resolver/resolve.py --release=bullseye debian:10) \ + && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py debian:11) \ && apt-get clean \ + && update-java-alternatives -s java-1.11.0-openjdk-amd64 \ && rm -rf /var/lib/apt/lists/* # TODO : Set locale @@ -56,38 +54,38 @@ RUN apt-get -q update \ ###### # Set env vars required to build Hadoop ###### -ENV MAVEN_HOME /usr +ENV MAVEN_HOME=/usr # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) -ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 +ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 ####### # Set env vars for SpotBugs 4.2.2 ####### -ENV SPOTBUGS_HOME /opt/spotbugs +ENV SPOTBUGS_HOME=/opt/spotbugs ####### # Set env vars for Google Protobuf 3.21.12 ####### -ENV PROTOBUF_HOME /opt/protobuf -ENV PATH "${PATH}:/opt/protobuf/bin" +ENV PROTOBUF_HOME=/opt/protobuf +ENV PATH="${PATH}:/opt/protobuf/bin" ### # Avoid out of memory errors in builds ### -ENV MAVEN_OPTS -Xms256m -Xmx3072m +ENV MAVEN_OPTS="-Xms256m -Xmx3072m" # Skip gpg verification when downloading Yetus via yetus-wrapper -ENV HADOOP_SKIP_YETUS_VERIFICATION true +ENV HADOOP_SKIP_YETUS_VERIFICATION=true #### # Install packages #### -RUN pkg-resolver/install-cmake.sh debian:10 -RUN pkg-resolver/install-spotbugs.sh debian:10 -RUN pkg-resolver/install-boost.sh debian:10 -RUN pkg-resolver/install-protobuf.sh debian:10 -RUN pkg-resolver/install-hadolint.sh debian:10 -RUN pkg-resolver/install-intel-isa-l.sh debian:10 +RUN pkg-resolver/install-cmake.sh debian:11 +RUN pkg-resolver/install-spotbugs.sh debian:11 +RUN pkg-resolver/install-boost.sh debian:11 +RUN pkg-resolver/install-protobuf.sh debian:11 +RUN pkg-resolver/install-hadolint.sh debian:11 +RUN pkg-resolver/install-intel-isa-l.sh debian:11 ### # Everything past this point is either not needed for testing or breaks Yetus. diff --git a/dev-support/docker/pkg-resolver/install-pkg-resolver.sh b/dev-support/docker/pkg-resolver/install-pkg-resolver.sh index 70e94b3792d9c..81dd0002f40fb 100644 --- a/dev-support/docker/pkg-resolver/install-pkg-resolver.sh +++ b/dev-support/docker/pkg-resolver/install-pkg-resolver.sh @@ -24,7 +24,7 @@ fi chmod a+x pkg-resolver/*.sh pkg-resolver/*.py chmod a+r pkg-resolver/*.json -if [ "$1" == "debian:10" ]; then +if [ "$1" == "debian:11" ]; then apt-get -q update apt-get -q install -y --no-install-recommends python3 \ python3-pip \ diff --git a/dev-support/docker/pkg-resolver/packages.json b/dev-support/docker/pkg-resolver/packages.json index 28b8733e04b97..452ed015c329d 100644 --- a/dev-support/docker/pkg-resolver/packages.json +++ b/dev-support/docker/pkg-resolver/packages.json @@ -1,38 +1,38 @@ { "ant": { - "debian:10": "ant", + "debian:11": "ant", "ubuntu:focal": "ant", "ubuntu:focal::arch64": "ant", "ubuntu:noble": "ant", "rockylinux:8": "ant" }, "apt-utils": { - "debian:10": "apt-utils", + "debian:11": "apt-utils", "ubuntu:focal": "apt-utils", "ubuntu:noble": "apt-utils", "ubuntu:focal::arch64": "apt-utils" }, "automake": { - "debian:10": "automake", + "debian:11": "automake", "ubuntu:focal": "automake", "ubuntu:noble": "automake", "ubuntu:focal::arch64": "automake", "rockylinux:8": "automake" }, "bats": { - "debian:10": "bats", + "debian:11": "bats", "ubuntu:focal": "bats", "ubuntu:noble": "bats", "ubuntu:focal::arch64": "bats" }, "build-essential": { - "debian:10": "build-essential", + "debian:11": "build-essential", "ubuntu:focal": "build-essential", "ubuntu:noble": "build-essential", "ubuntu:focal::arch64": "build-essential" }, "bzip2": { - "debian:10": [ + "debian:11": [ "bzip2", "libbz2-dev" ], @@ -54,7 +54,7 @@ ] }, "clang": { - "debian:10": "clang", + "debian:11": "clang", "ubuntu:focal": "clang", "ubuntu:noble": "clang", "ubuntu:focal::arch64": "clang", @@ -66,7 +66,7 @@ "ubuntu:focal::arch64": "cmake" }, "curl": { - "debian:10": [ + "debian:11": [ "curl", "libcurl4-openssl-dev" ], @@ -88,7 +88,7 @@ ] }, "doxygen": { - "debian:10": "doxygen", + "debian:11": "doxygen", "ubuntu:focal": "doxygen", "ubuntu:noble": "doxygen", "ubuntu:focal::arch64": "doxygen" @@ -97,7 +97,7 @@ "rockylinux:8": "dnf" }, "fuse": { - "debian:10": [ + "debian:11": [ "fuse", "libfuse-dev" ], @@ -120,7 +120,7 @@ ] }, "gcc": { - "debian:10": { + "debian:11": { "bullseye": [ "gcc", "g++" @@ -140,39 +140,39 @@ ] }, "git": { - "debian:10": "git", + "debian:11": "git", "ubuntu:focal": "git", "ubuntu:noble": "git", "ubuntu:focal::arch64": "git", "rockylinux:8": "git" }, "gnupg-agent": { - "debian:10": "gnupg-agent", + "debian:11": "gnupg-agent", "ubuntu:focal": "gnupg-agent", "ubuntu:noble": "gnupg-agent", "ubuntu:focal::arch64": "gnupg-agent" }, "hugo": { - "debian:10": "hugo", + "debian:11": "hugo", "ubuntu:focal": "hugo", "ubuntu:noble": "hugo", "ubuntu:focal::arch64": "hugo" }, "libbcprov-java": { - "debian:10": "libbcprov-java", + "debian:11": "libbcprov-java", "ubuntu:focal": "libbcprov-java", "ubuntu:noble": "libbcprov-java", "ubuntu:focal::arch64": "libbcprov-java" }, "libtool": { - "debian:10": "libtool", + "debian:11": "libtool", "ubuntu:focal": "libtool", "ubuntu:noble": "libtool", "ubuntu:focal::arch64": "libtool", "rockylinux:8": "libtool" }, "openssl": { - "debian:10": "libssl-dev", + "debian:11": "libssl-dev", "ubuntu:focal": "libssl-dev", "ubuntu:noble": "libssl-dev", "ubuntu:focal::arch64": "libssl-dev", @@ -185,7 +185,7 @@ ] }, "protocol-buffers": { - "debian:10": [ + "debian:11": [ "libprotobuf-dev", "libprotoc-dev" ], @@ -203,20 +203,20 @@ ] }, "sasl": { - "debian:10": "libsasl2-dev", + "debian:11": "libsasl2-dev", "ubuntu:focal": "libsasl2-dev", "ubuntu:noble": "libsasl2-dev", "ubuntu:focal::arch64": "libsasl2-dev", "rockylinux:8": "cyrus-sasl-devel" }, "snappy": { - "debian:10": "libsnappy-dev", + "debian:11": "libsnappy-dev", "ubuntu:focal": "libsnappy-dev", "ubuntu:noble": "libsnappy-dev", "ubuntu:focal::arch64": "libsnappy-dev" }, "zlib": { - "debian:10": [ + "debian:11": [ "libzstd-dev", "zlib1g-dev" ], @@ -238,7 +238,7 @@ ] }, "locales": { - "debian:10": "locales", + "debian:11": "locales", "ubuntu:focal": "locales", "ubuntu:noble": "locales", "ubuntu:focal::arch64": "locales" @@ -248,20 +248,23 @@ "ubuntu:noble": "libtirpc-dev" }, "make": { - "debian:10": "make", + "debian:11": "make", "ubuntu:focal": "make", "ubuntu:noble": "make", "ubuntu:focal::arch64": "make", "rockylinux:8": "make" }, "maven": { - "debian:10": "maven", + "debian:11": "maven", "ubuntu:focal": "maven", "ubuntu:noble": "maven", "ubuntu:focal::arch64": "maven" }, "java": { - "debian:10": "openjdk-11-jdk", + "debian:11": [ + "openjdk-11-jdk", + "openjdk-17-jdk" + ], "ubuntu:focal": [ "temurin-24-jdk", "openjdk-8-jdk", @@ -281,21 +284,21 @@ ] }, "pinentry-curses": { - "debian:10": "pinentry-curses", + "debian:11": "pinentry-curses", "ubuntu:focal": "pinentry-curses", "ubuntu:noble": "pinentry-curses", "ubuntu:focal::arch64": "pinentry-curses", "rockylinux:8": "pinentry-curses" }, "pkg-config": { - "debian:10": "pkg-config", + "debian:11": "pkg-config", "ubuntu:focal": "pkg-config", "ubuntu:noble": "pkg-config", "ubuntu:focal::arch64": "pkg-config", "rockylinux:8": "pkg-config" }, "python": { - "debian:10": [ + "debian:11": [ "python3", "python3-pip", "python3-pkg-resources", @@ -332,14 +335,14 @@ ] }, "rsync": { - "debian:10": "rsync", + "debian:11": "rsync", "ubuntu:focal": "rsync", "ubuntu:noble": "rsync", "ubuntu:focal::arch64": "rsync", "rockylinux:8": "rsync" }, "shellcheck": { - "debian:10": "shellcheck", + "debian:11": "shellcheck", "ubuntu:focal": "shellcheck", "ubuntu:noble": "shellcheck", "ubuntu:focal::arch64": "shellcheck" @@ -348,27 +351,27 @@ "rockylinux:8": "perl-Digest-SHA" }, "software-properties-common": { - "debian:10": "software-properties-common", + "debian:11": "software-properties-common", "ubuntu:focal": "software-properties-common", "ubuntu:noble": "software-properties-common", "ubuntu:focal::arch64": "software-properties-common" }, "sudo": { - "debian:10": "sudo", + "debian:11": "sudo", "ubuntu:focal": "sudo", "ubuntu:noble": "sudo", "ubuntu:focal::arch64": "sudo", "rockylinux:8": "sudo" }, "valgrind": { - "debian:10": "valgrind", + "debian:11": "valgrind", "ubuntu:focal": "valgrind", "ubuntu:noble": "valgrind", "ubuntu:focal::arch64": "valgrind", "rockylinux:8": "valgrind" }, "yasm": { - "debian:10": "yasm", + "debian:11": "yasm", "ubuntu:focal": "yasm", "ubuntu:noble": "yasm", "ubuntu:focal::arch64": "yasm" diff --git a/dev-support/docker/pkg-resolver/platforms.json b/dev-support/docker/pkg-resolver/platforms.json index eebf7fc948f62..00f6c129b207d 100644 --- a/dev-support/docker/pkg-resolver/platforms.json +++ b/dev-support/docker/pkg-resolver/platforms.json @@ -3,5 +3,5 @@ "ubuntu:focal::arch64", "ubuntu:noble", "rockylinux:8", - "debian:10" + "debian:11" ] From 1dc355c16ac6bb71f4a9830084ea0b96ba94a1ae Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Fri, 29 Aug 2025 14:30:04 +0800 Subject: [PATCH 2/3] Rename Dockerfile_debian_11 to Dockerfile_debian_10 --- dev-support/Jenkinsfile | 2 +- .../docker/{Dockerfile_debian_11 => Dockerfile_debian_10} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename dev-support/docker/{Dockerfile_debian_11 => Dockerfile_debian_10} (100%) diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 672061ea09ad9..e28baa67f209d 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -162,7 +162,7 @@ pipeline { environment { SOURCEDIR = "${WORKSPACE}/debian-11/src" PATCHDIR = "${WORKSPACE}/debian-11/out" - DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_11" + DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_10" IS_OPTIONAL = 1 } diff --git a/dev-support/docker/Dockerfile_debian_11 b/dev-support/docker/Dockerfile_debian_10 similarity index 100% rename from dev-support/docker/Dockerfile_debian_11 rename to dev-support/docker/Dockerfile_debian_10 From d7d9893cf80efe73d802c253eb12508143e8a636 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Fri, 29 Aug 2025 14:30:37 +0800 Subject: [PATCH 3/3] touch a test --- .../src/test/java/org/apache/hadoop/cli/TestCLI.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/TestCLI.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/TestCLI.java index e7cd5eb8a9ac4..b433a47def3af 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/TestCLI.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/TestCLI.java @@ -43,7 +43,6 @@ public void tearDown() throws Exception { @Override protected CommandExecutor.Result execute(CLICommand cmd) throws Exception { return cmd.getExecutor("", conf).executeCommand(cmd.getCmd()); - } @Override