[BACKPORT 2.20.2] [DEVOPS-3147] build: Move off centos7

Summary:
Includes several changes:
  b78b8ca [2.20][DEVOPS-2956] build: Reduce centos7 testing on 2.20
  382c2ee [2.20][DEVOPS-3058] build/test: Reduce mac testing on older releases
  c7ce045 [2.20][DEVOPS-3069] build: Enable clang17 and amazonlinux2
  0049c5a [2.20][DEVOPS-3147] Move aarch64 build to amzn2
  ecc2d23 [2.20][DEVOPS-3174] Fix third-party deps for alma8-aarch64

Test Plan: Jenkins: compile only

Reviewers: jharveysmith

Reviewed By: jharveysmith

Subscribers: devops

Differential Revision: https://phorge.dev.yugabyte.com/D37853

build-support/common-build-env.sh

@@ -2603,7 +2603,7 @@ set_prebuilt_thirdparty_url() {
         # Transform "thin-lto" or "full-lto" into "thin" or "full" respectively.
         thirdparty_tool_cmd_line+=( "--lto=${YB_LINKING_TYPE%%-lto}" )
       fi
-      if [[ ! ${build_type} =~ ^(asan|tsan)$ && ${YB_COMPILER_TYPE} == clang* ]]; then
+      if [[ is_alma && ! ${build_type} =~ ^(asan|tsan)$ && ${YB_COMPILER_TYPE} == clang* ]]; then
         thirdparty_tool_cmd_line+=( "--allow-older-os" )
       fi
       "${thirdparty_tool_cmd_line[@]}"

build-support/thirdparty_archives.yml

@@ -1,163 +1,198 @@
-sha_for_local_checkout: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419
+sha_for_local_checkout: 5b9d0c65af588696e47baf8e24b6a64a177a4157
 archives:
   - os_type: almalinux8
     architecture: x86_64
-    compiler_type: clang15
+    compiler_type: clang16
     is_linuxbrew: false
-    sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
     lto_type:
-    tag: v2.20-20231114005155-44dd1e39bc-almalinux8-x86_64-clang15
+    tag: v2.20-20240509185516-5b9d0c65af-almalinux8-x86_64-clang16
   - os_type: almalinux8
     architecture: x86_64
-    compiler_type: clang15
+    compiler_type: clang16
     is_linuxbrew: true
-    sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
     lto_type:
-    tag: v2.20-20231114005150-44dd1e39bc-almalinux8-x86_64-clang15-linuxbrew
+    tag: v2.20-20240509185512-5b9d0c65af-almalinux8-x86_64-clang16-linuxbrew
   - os_type: almalinux8
     architecture: x86_64
-    compiler_type: clang15
+    compiler_type: clang16
     is_linuxbrew: true
-    sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
     lto_type: full
-    tag: v2.20-20231114005229-44dd1e39bc-almalinux8-x86_64-clang15-linuxbrew-full-lto
+    tag: v2.20-20240509185512-5b9d0c65af-almalinux8-x86_64-clang16-linuxbrew-full-lto
   - os_type: almalinux8
     architecture: x86_64
-    compiler_type: clang16
+    compiler_type: clang17
     is_linuxbrew: false
-    sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
     lto_type:
-    tag: v2.20-20231114005304-44dd1e39bc-almalinux8-x86_64-clang16
+    tag: v2.20-20240509185516-5b9d0c65af-almalinux8-x86_64-clang17
   - os_type: almalinux8
     architecture: x86_64
-    compiler_type: clang16
+    compiler_type: clang17
     is_linuxbrew: true
-    sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
     lto_type:
-    tag: v2.20-20231114005319-44dd1e39bc-almalinux8-x86_64-clang16-linuxbrew
+    tag: v2.20-20240509185520-5b9d0c65af-almalinux8-x86_64-clang17-linuxbrew
   - os_type: almalinux8
     architecture: x86_64
-    compiler_type: clang16
+    compiler_type: clang17
     is_linuxbrew: true
-    sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
     lto_type: full
-    tag: v2.20-20231114005149-44dd1e39bc-almalinux8-x86_64-clang16-linuxbrew-full-lto
+    tag: v2.20-20240509185505-5b9d0c65af-almalinux8-x86_64-clang17-linuxbrew-full-lto
   - os_type: almalinux8
     architecture: x86_64
     compiler_type: gcc11
     is_linuxbrew: false
-    sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
     lto_type:
-    tag: v2.20-20231114005150-44dd1e39bc-almalinux8-x86_64-gcc11
+    tag: v2.20-20240509185529-5b9d0c65af-almalinux8-x86_64-gcc11
   - os_type: almalinux9
     architecture: x86_64
     compiler_type: clang16
     is_linuxbrew: false
-    sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
     lto_type:
-    tag: v2.20-20231114005254-44dd1e39bc-almalinux9-x86_64-clang16
+    tag: v2.20-20240509185522-5b9d0c65af-almalinux9-x86_64-clang16
+  - os_type: almalinux9
+    architecture: x86_64
+    compiler_type: clang17
+    is_linuxbrew: false
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
+    lto_type:
+    tag: v2.20-20240509185514-5b9d0c65af-almalinux9-x86_64-clang17
   - os_type: almalinux9
     architecture: x86_64
     compiler_type: gcc12
     is_linuxbrew: false
-    sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
     lto_type:
-    tag: v2.20-20231114005251-44dd1e39bc-almalinux9-x86_64-gcc12
-  - os_type: centos7
+    tag: v2.20-20240625152657-ffb781854e-almalinux9-x86_64-gcc12
+  - os_type: amzn2
     architecture: aarch64
-    compiler_type: clang15
+    compiler_type: clang17
     is_linuxbrew: false
-    sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
     lto_type:
-    tag: v2.20-20231114005317-44dd1e39bc-centos7-aarch64-clang15
-  - os_type: centos7
+    tag: v2.20-20240625152843-ffb781854e-amazonlinux2-aarch64-clang17
+  - os_type: amzn2
     architecture: aarch64
-    compiler_type: clang15
+    compiler_type: clang17
     is_linuxbrew: false
-    sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
     lto_type: full
-    tag: v2.20-20231114005340-44dd1e39bc-centos7-aarch64-clang15-full-lto
+    tag: v2.20-20240625152849-ffb781854e-amazonlinux2-aarch64-clang17-full-lto
+  - os_type: amzn2
+    architecture: x86_64
+    compiler_type: clang17
+    is_linuxbrew: false
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
+    lto_type:
+    tag: v2.20-20240509185556-5b9d0c65af-amazonlinux2-x86_64-clang17
   - os_type: centos7
     architecture: aarch64
     compiler_type: clang16
     is_linuxbrew: false
-    sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
     lto_type:
-    tag: v2.20-20231114005341-44dd1e39bc-centos7-aarch64-clang16
+    tag: v2.20-20240509185638-5b9d0c65af-centos7-aarch64-clang16
   - os_type: centos7
     architecture: aarch64
     compiler_type: clang16
     is_linuxbrew: false
-    sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
     lto_type: full
-    tag: v2.20-20231114005319-44dd1e39bc-centos7-aarch64-clang16-full-lto
+    tag: v2.20-20240509185634-5b9d0c65af-centos7-aarch64-clang16-full-lto
   - os_type: centos7
-    architecture: x86_64
-    compiler_type: clang15
+    architecture: aarch64
+    compiler_type: clang17
     is_linuxbrew: false
-    sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
     lto_type:
-    tag: v2.20-20231114005245-44dd1e39bc-centos7-x86_64-clang15
+    tag: v2.20-20240509185634-5b9d0c65af-centos7-aarch64-clang17
   - os_type: centos7
-    architecture: x86_64
-    compiler_type: clang15
+    architecture: aarch64
+    compiler_type: clang17
     is_linuxbrew: false
-    sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
     lto_type: full
-    tag: v2.20-20231114005141-44dd1e39bc-centos7-x86_64-clang15-full-lto
+    tag: v2.20-20240509185632-5b9d0c65af-centos7-aarch64-clang17-full-lto
   - os_type: centos7
     architecture: x86_64
     compiler_type: clang16
     is_linuxbrew: false
-    sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
     lto_type:
-    tag: v2.20-20231114005157-44dd1e39bc-centos7-x86_64-clang16
+    tag: v2.20-20240509185520-5b9d0c65af-centos7-x86_64-clang16
   - os_type: centos7
     architecture: x86_64
     compiler_type: clang16
     is_linuxbrew: false
-    sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
+    lto_type: full
+    tag: v2.20-20240509185505-5b9d0c65af-centos7-x86_64-clang16-full-lto
+  - os_type: centos7
+    architecture: x86_64
+    compiler_type: clang17
+    is_linuxbrew: false
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
+    lto_type:
+    tag: v2.20-20240509185518-5b9d0c65af-centos7-x86_64-clang17
+  - os_type: centos7
+    architecture: x86_64
+    compiler_type: clang17
+    is_linuxbrew: false
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
     lto_type: full
-    tag: v2.20-20231114005147-44dd1e39bc-centos7-x86_64-clang16-full-lto
+    tag: v2.20-20240509185509-5b9d0c65af-centos7-x86_64-clang17-full-lto
   - os_type: centos7
     architecture: x86_64
     compiler_type: gcc11
     is_linuxbrew: false
-    sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
     lto_type:
-    tag: v2.20-20231114005152-44dd1e39bc-centos7-x86_64-gcc11
+    tag: v2.20-20240509185520-5b9d0c65af-centos7-x86_64-gcc11
   - os_type: macos
     architecture: arm64
     compiler_type: clang
     is_linuxbrew: false
-    sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
     lto_type:
-    tag: v2.20-20231114170325-44dd1e39bc-macos-arm64
+    tag: v2.20-20240517153912-5b9d0c65af-macos-arm64
   - os_type: macos
     architecture: x86_64
     compiler_type: clang
     is_linuxbrew: false
-    sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
     lto_type:
-    tag: v2.20-20231114023841-44dd1e39bc-macos-x86_64
+    tag: v2.20-20240509185549-5b9d0c65af-macos-x86_64
   - os_type: ubuntu20.04
     architecture: x86_64
-    compiler_type: clang15
+    compiler_type: clang16
+    is_linuxbrew: false
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
+    lto_type:
+    tag: v2.20-20240509185512-5b9d0c65af-ubuntu2004-x86_64-clang16
+  - os_type: ubuntu22.04
+    architecture: x86_64
+    compiler_type: clang16
     is_linuxbrew: false
-    sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
     lto_type:
-    tag: v2.20-20231114005138-44dd1e39bc-ubuntu2004-x86_64-clang15
+    tag: v2.20-20240509185514-5b9d0c65af-ubuntu2204-x86_64-clang16
   - os_type: ubuntu22.04
     architecture: x86_64
-    compiler_type: clang15
+    compiler_type: clang17
     is_linuxbrew: false
-    sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
     lto_type:
-    tag: v2.20-20231114005141-44dd1e39bc-ubuntu2204-x86_64-clang15
+    tag: v2.20-20240509185507-5b9d0c65af-ubuntu2204-x86_64-clang17
   - os_type: ubuntu22.04
     architecture: x86_64
     compiler_type: gcc11
     is_linuxbrew: false
-    sha: 44dd1e39bcbcd8f8fe235439d27ee12cf69af419
+    sha: 5b9d0c65af588696e47baf8e24b6a64a177a4157
     lto_type:
-    tag: v2.20-20231114005141-44dd1e39bc-ubuntu2204-x86_64-gcc11
+    tag: v2.20-20240509185513-5b9d0c65af-ubuntu2204-x86_64-gcc11

build-support/yugabyte-bash-common-sha1.txt

@@ -1 +1 @@
-06fdecffba1970934a139b3f2dfdf684789c212c
+bcceabfde9f45afc0ad24bd99aab925191707da6

jenkins_jobs.yml

@@ -6,13 +6,13 @@ jobs:
     build_type: asan
     release_artifact: false
 
-  - os: centos7
+  - os: alma8
     compiler: clang16
     build_type: tsan
     release_artifact: false
 
-  - os: centos7
-    compiler: clang16
+  - os: alma8
+    compiler: gcc11
     build_type: debug
     release_artifact: false
 
@@ -21,8 +21,8 @@ jobs:
     build_type: release
     release_artifact: true
 
-  - os: centos7
-    compiler: clang16
+  - os: amzn2
+    compiler: clang17
     build_type: release
     architecture: aarch64
     release_artifact: true
@@ -33,24 +33,22 @@ jobs:
     architecture: aarch64
     release_artifact: false
 
-  - os: alma8
-    compiler: gcc11
-    build_type: fastdebug
-    release_artifact: false
-
   - os: mac12
     compiler: clang
     build_type: release
     architecture: arm64
     release_artifact: true
+    test_opts: YB_COMPILE_ONLY=1
 
   - os: mac12
     compiler: clang
     build_type: release
     architecture: x86_64
     release_artifact: true
+    test_opts: YB_COMPILE_ONLY=1
 
   - os: ubuntu20.04
     compiler: clang16
     build_type: debug
     release_artifact: false
+    test_opts: YB_COMPILE_ONLY=1

python/yugabyte/os_versions.py

@@ -14,6 +14,7 @@
 
 
 UBUNTU_OS_TYPE_RE = re.compile(r'^(ubuntu)([0-9]{2})([0-9]{2})$')
+AMZN_OS_TYPE_RE = re.compile(r'^(amzn|amazonlinux)([0-9]+)$')
 RHEL_FAMILY_RE = re.compile(r'^(almalinux|centos|rhel)([0-9]+)$')
 
 
@@ -22,6 +23,10 @@ def adjust_os_type(os_type: str) -> str:
     if match:
         # Convert e.g. ubuntu2004 -> ubuntu20.04 for clarity.
         return f'{match.group(1)}{match.group(2)}.{match.group(3)}'
+    match = AMZN_OS_TYPE_RE.match(os_type)
+    if match:
+        # Convert e.g. amazonlinux2 -> amzn2 to match OS ID name
+        return f'amzn{match.group(2)}'
     return os_type
 
 

python/yugabyte/release_util.py

@@ -261,8 +261,10 @@ def get_release_file(self) -> str:
             # in our downsstream release code.  So here we munge the name to 'centos' to keep things
             # working while we fix downstream code.
             # TODO(jharveymsith): Remove the almalinux to centos mapping once downstream is fixed.
+            # DEVOPS-3154
             if distro.id() == "centos" and distro.major_version() == "7" \
-                    or distro.id() == "almalinux" and platform.machine().lower() == "x86_64":
+                    or distro.id() == "almalinux" and platform.machine().lower() == "x86_64" \
+                    or distro.id() == "amzn" and platform.machine().lower() == "aarch64":
                 system = "centos"
             elif distro.id == "ubuntu":
                 system = distro.id() + distro.version()

python/yugabyte/thirdparty_tool.py

@@ -258,19 +258,22 @@ def is_consistent_with_yb_version(self, yb_version: str) -> bool:
 
     def should_skip_as_too_os_specific(self) -> bool:
         """
+        Centos7 is EOL June 2024, this condition no longer applies.
+
         Certain build types of specific OSes could be skipped because we can use the CentOS 7 build
         instead. We can do that in cases we know we don't need to run ASAN/TSAN. We know that we
         don't use ASAN/TSAN on aarch64 or for LTO builds as of 11/07/2022. Also we don't skip
         Linuxbrew builds or GCC builds.
         """
-        return (
-            self.os_type != 'centos7' and
-            self.compiler_type.startswith('clang') and
-            # We handle Linuxbrew builds in a special way, e.g. they could be built on AlmaLinux 8.
-            not self.is_linuxbrew and
-            # We don't run ASAN/TSAN on aarch64 or with LTO yet.
-            (self.architecture == 'aarch64' or self.lto_type is not None)
-        )
+        # return (
+        #    self.os_type != 'centos7' and
+        #    self.compiler_type.startswith('clang') and
+        #    # We handle Linuxbrew builds in a special way, e.g. they could be built on AlmaLinux 8.
+        #    not self.is_linuxbrew and
+        #    # We don't run ASAN/TSAN on aarch64 or with LTO yet.
+        #    (self.architecture == 'aarch64' or self.lto_type is not None)
+        # )
+        return False
 
 
 @ruamel_yaml_object.register_class