From 13bb4cfd1005d8e62885c91288b53df046404778 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 3 Feb 2025 13:10:52 -0800 Subject: [PATCH 1/7] RANGER-5126: Add coverage for testing ranger upgrades --- .github/workflows/test-upgrade.yaml | 211 ++++++++++++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 .github/workflows/test-upgrade.yaml diff --git a/.github/workflows/test-upgrade.yaml b/.github/workflows/test-upgrade.yaml new file mode 100644 index 0000000000..93af49bdc9 --- /dev/null +++ b/.github/workflows/test-upgrade.yaml @@ -0,0 +1,211 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Upgrade Ranger + +on: + push: + pull_request: + branches: [ "master" ] + +env: + MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + +jobs: + install: + strategy: + fail-fast: false # Ensures all matrix jobs run to completion + matrix: + release: [ranger-2.4, ranger-2.5, ranger-2.6] # builds all these release versions + runs-on: ubuntu-22.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - name: Cache for maven dependencies + uses: actions/cache@v4 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/ranger + key: maven-repo-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-repo- + + - name: Set up .m2 directory and permissions + run: | + mkdir -p ~/.m2/repository + chmod -R 777 ~/.m2 + chmod -R 777 ./ + + - name: Set up JDK 8 + uses: actions/setup-java@v4 + with: + java-version: '8' + distribution: 'temurin' + + - name: Update .env + run: | + cd dev-support/ranger-docker + RELEASE="${{ matrix.release }}" + python3 < Date: Mon, 3 Feb 2025 15:10:03 -0800 Subject: [PATCH 2/7] Add oracle and refactor file. --- .github/workflows/test-upgrade.yaml | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test-upgrade.yaml b/.github/workflows/test-upgrade.yaml index 93af49bdc9..f9e9f2f8d3 100644 --- a/.github/workflows/test-upgrade.yaml +++ b/.github/workflows/test-upgrade.yaml @@ -13,15 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - -# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven - -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - name: Upgrade Ranger on: @@ -82,11 +73,6 @@ jobs: EOF cat .env - - name: Run download-archives.sh - run: | - cd dev-support/ranger-docker - ./download-archives.sh none - - name: Clean up Docker space run: docker system prune --all --force --volumes @@ -118,7 +104,7 @@ jobs: fail-fast: false matrix: release: [ranger-2.4, ranger-2.5] - db: [postgres, mysql] + db: [postgres, mysql, oracle] runs-on: ubuntu-22.04 timeout-minutes: 60 steps: @@ -203,7 +189,7 @@ jobs: docker compose -f docker-compose.ranger-base.yml build docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-${RANGER_DB_TYPE}.yml build docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-${RANGER_DB_TYPE}.yml up -d - sleep 60 + sleep 120 docker logs ranger - name: Remove running containers From 4e0b444d42895e2b0d39cc97a9ea938fde043157 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 3 Feb 2025 17:52:32 -0800 Subject: [PATCH 3/7] Refactor install to build and increase sleep time before docker logs. --- .github/workflows/test-upgrade.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-upgrade.yaml b/.github/workflows/test-upgrade.yaml index f9e9f2f8d3..ac937ac662 100644 --- a/.github/workflows/test-upgrade.yaml +++ b/.github/workflows/test-upgrade.yaml @@ -24,7 +24,7 @@ env: MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 jobs: - install: + build: strategy: fail-fast: false # Ensures all matrix jobs run to completion matrix: @@ -99,7 +99,7 @@ jobs: path: dev-support/ranger-docker/dist/* upgrade: - needs: install + needs: build strategy: fail-fast: false matrix: @@ -189,7 +189,7 @@ jobs: docker compose -f docker-compose.ranger-base.yml build docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-${RANGER_DB_TYPE}.yml build docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-${RANGER_DB_TYPE}.yml up -d - sleep 120 + sleep 180 docker logs ranger - name: Remove running containers From 19c7febe393da66bfbc4702d309b3de5b3e9fc16 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 4 Feb 2025 18:46:27 -0800 Subject: [PATCH 4/7] Create dot_env_util.py --- .github/scripts/dot_env_util.py | 58 +++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 .github/scripts/dot_env_util.py diff --git a/.github/scripts/dot_env_util.py b/.github/scripts/dot_env_util.py new file mode 100644 index 0000000000..f25cc6c0a6 --- /dev/null +++ b/.github/scripts/dot_env_util.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse + +branch_to_snapshot_version_map = { + 'master': '3.0.0-SNAPSHOT', + 'ranger-2.4': '2.4.1-SNAPSHOT', + 'ranger-2.5': '2.5.1-SNAPSHOT', + 'ranger-2.6': '2.6.0', +} + + +def update_env_mvn_build(data, release_branch): + data = data.replace('BUILD_HOST_SRC=true', 'BUILD_HOST_SRC=false') + data = data.replace('BRANCH=master', f'BRANCH={release_branch}') + return data + + +def update_env(current_branch, release_branch, is_mvn_build): + with open(r'.env', 'r') as file: + data = file.read() + + ranger_release_version = branch_to_snapshot_version_map[release_branch] + ranger_current_version = branch_to_snapshot_version_map[current_branch] + + if is_mvn_build: + data = update_env_mvn_build(data, release_branch) + + data = data.replace(ranger_current_version, ranger_release_version) + + with open(r'.env', 'w') as file: + file.write(data) + return + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Utility to update .env') + parser.add_argument('--release_branch', help='release branch') + parser.add_argument('--current_branch', default='master', help='current checked out ranger branch') + parser.add_argument('--maven_build', default=False, help='Maven build in Docker required?') + + args = parser.parse_args() + update_env(args.current_branch, args.release_branch, args.maven_build) + From dc28730f398158c0b5bc81a5e5e6f1df8ca5a771 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 4 Feb 2025 18:56:34 -0800 Subject: [PATCH 5/7] Update test-upgrade.yaml --- .github/workflows/test-upgrade.yaml | 38 +++-------------------------- 1 file changed, 3 insertions(+), 35 deletions(-) diff --git a/.github/workflows/test-upgrade.yaml b/.github/workflows/test-upgrade.yaml index ac937ac662..72f3be130e 100644 --- a/.github/workflows/test-upgrade.yaml +++ b/.github/workflows/test-upgrade.yaml @@ -58,19 +58,7 @@ jobs: - name: Update .env run: | cd dev-support/ranger-docker - RELEASE="${{ matrix.release }}" - python3 < Date: Tue, 4 Feb 2025 19:07:20 -0800 Subject: [PATCH 6/7] Update test-upgrade.yaml --- .github/workflows/test-upgrade.yaml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-upgrade.yaml b/.github/workflows/test-upgrade.yaml index 72f3be130e..42f246cb5b 100644 --- a/.github/workflows/test-upgrade.yaml +++ b/.github/workflows/test-upgrade.yaml @@ -16,9 +16,11 @@ name: Upgrade Ranger on: - push: - pull_request: - branches: [ "master" ] + workflow_dispatch: + inputs: + release_branch: + description: 'release branch to be used for upgrade' + required: true env: MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 @@ -28,7 +30,7 @@ jobs: strategy: fail-fast: false # Ensures all matrix jobs run to completion matrix: - release: [ranger-2.4, ranger-2.5, ranger-2.6] # builds all these release versions + release: [ranger-2.4, ranger-2.5, "${{ github.event.inputs.release_branch }}"] # builds all these release versions runs-on: ubuntu-22.04 timeout-minutes: 60 steps: @@ -111,7 +113,7 @@ jobs: - name: Download release artifacts for upgrade uses: actions/download-artifact@v4 with: - name: ranger-2.6-build-artifacts + name: ${{ github.event.inputs.release_branch }}-build-artifacts - name: Copy artifacts for docker build run: | @@ -145,10 +147,10 @@ jobs: - name: Update .env for upgrade run: | cd dev-support/ranger-docker - python3 ./../../.github/scripts/dot_env_util.py --release_branch=ranger-2.6 --current_branch=master + python3 ./../../.github/scripts/dot_env_util.py --release_branch=${{ github.event.inputs.release_branch }} --current_branch=master cat .env - - name: Upgrading to ranger-2.6 in docker + - name: Upgrading to ${{ github.event.inputs.release_branch }} in docker run: | cd dev-support/ranger-docker export DOCKER_BUILDKIT=1 From 3d7fbc244c5f07b3f4cc55df98930b4ba1425b13 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 4 Feb 2025 20:18:16 -0800 Subject: [PATCH 7/7] Fix last update to .env before upgrade --- .github/workflows/test-upgrade.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-upgrade.yaml b/.github/workflows/test-upgrade.yaml index 42f246cb5b..acd615ed83 100644 --- a/.github/workflows/test-upgrade.yaml +++ b/.github/workflows/test-upgrade.yaml @@ -147,7 +147,7 @@ jobs: - name: Update .env for upgrade run: | cd dev-support/ranger-docker - python3 ./../../.github/scripts/dot_env_util.py --release_branch=${{ github.event.inputs.release_branch }} --current_branch=master + python3 ./../../.github/scripts/dot_env_util.py --release_branch=${{ github.event.inputs.release_branch }} --current_branch="${{ matrix.release }}" cat .env - name: Upgrading to ${{ github.event.inputs.release_branch }} in docker