From 2bc6b12fdbaa1bafe7fb278802aac785a6cc1bc8 Mon Sep 17 00:00:00 2001 From: Poh Peng Date: Fri, 10 Nov 2023 15:44:07 +0800 Subject: [PATCH 01/14] Prepare v3 version of workflow --- .github/workflows/terraform.yaml | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/.github/workflows/terraform.yaml b/.github/workflows/terraform.yaml index 15638d3..0c05a69 100644 --- a/.github/workflows/terraform.yaml +++ b/.github/workflows/terraform.yaml @@ -55,10 +55,6 @@ on: description: Enforce tflint warnings for changed files by default type: boolean default: false - default_runner_override_label: - description: Change this to "self-hosted" or "ubuntu-latest" - type: string - default: "ubuntu-latest" runner_label: description: Runner label to point to self hosted runners type: string @@ -81,7 +77,6 @@ jobs: if: github.ref_name != 'main' name: Format and Validate runs-on: - - ${{ inputs.default_runner_override_label }} - ${{ inputs.runner_label }} steps: - name: Checkout @@ -94,6 +89,14 @@ jobs: with: python-version: '3.11' + - name: Setup Node only for self-hosted runners + uses: actions/setup-node@v3 + with: + node-version: 19 + + - name: Install missing binaries zip jq curl + run: sudo apt-get update && sudo apt-get install zip jq curl -y + - run: mkdir -p "${TF_PLUGIN_CACHE_DIR}" - name: Cache Terraform uses: actions/cache@v3 @@ -106,12 +109,6 @@ jobs: path: ~/.tflint.d/plugins key: ${{ runner.os }}-tflint-${{ hashFiles('**/.tflint.hcl') }} - - name: Setup Node only for self-hosted runners - uses: actions/setup-node@v3 - if: ${{ inputs.default_runner_override_label == 'self-hosted' }} - with: - node-version: 18 - - name: Setup Terraform uses: hashicorp/setup-terraform@v2 with: @@ -179,7 +176,6 @@ jobs: name: Linting if: github.ref_name != 'main' runs-on: - - ${{ inputs.default_runner_override_label }} - ${{ inputs.runner_label }} steps: - name: Checkout @@ -223,12 +219,6 @@ jobs: tflint_version: "v0.47.0" github_token: ${{ secrets.GITHUB_TOKEN }} - - name: Setup Node only for self-hosted runners - uses: actions/setup-node@v3 - if: ${{ inputs.default_runner_override_label == 'self-hosted' }} - with: - node-version: 19 - - name: Pre-init Hook run: ${{ inputs.pre_init_hook }} @@ -308,7 +298,6 @@ jobs: name: Security Checks if: github.ref_name != 'main' runs-on: - - ${{ inputs.default_runner_override_label }} - ${{ inputs.runner_label }} steps: - name: Checkout @@ -331,7 +320,7 @@ jobs: # https://github.com/aquasecurity/trivy/issues/5003 - name: Remove git from url for sarif uploading shell: bash - run: | + run: | sed -i 's#git::https:/##g' trivy-results.sarif - name: Upload Trivy scan results to GitHub Security tab From 332bb905f307c31125d882e1d94307f86f99a082 Mon Sep 17 00:00:00 2001 From: Poh Peng Date: Fri, 10 Nov 2023 15:48:44 +0800 Subject: [PATCH 02/14] Add missing git --- .github/workflows/terraform.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/terraform.yaml b/.github/workflows/terraform.yaml index 0c05a69..f7abf49 100644 --- a/.github/workflows/terraform.yaml +++ b/.github/workflows/terraform.yaml @@ -95,7 +95,7 @@ jobs: node-version: 19 - name: Install missing binaries zip jq curl - run: sudo apt-get update && sudo apt-get install zip jq curl -y + run: sudo apt-get update && sudo apt-get install zip jq curl git -y - run: mkdir -p "${TF_PLUGIN_CACHE_DIR}" - name: Cache Terraform @@ -188,6 +188,14 @@ jobs: with: python-version: '3.11' + - name: Setup Node only for self-hosted runners + uses: actions/setup-node@v3 + with: + node-version: 19 + + - name: Install missing binaries zip jq curl + run: sudo apt-get update && sudo apt-get install zip jq curl git -y + - run: mkdir -p "${TF_PLUGIN_CACHE_DIR}" - name: Cache Terraform uses: actions/cache@v3 From 1a5f5caae5d43a78d4171ae4dda73bce6c76b447 Mon Sep 17 00:00:00 2001 From: Poh Peng Date: Fri, 10 Nov 2023 16:08:01 +0800 Subject: [PATCH 03/14] Test set export GIT_DISCOVERY_ACROSS_FILESYSTEM=1 --- .github/workflows/terraform.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/terraform.yaml b/.github/workflows/terraform.yaml index f7abf49..1b85295 100644 --- a/.github/workflows/terraform.yaml +++ b/.github/workflows/terraform.yaml @@ -148,6 +148,7 @@ jobs: SKIP: ${{ steps.precommit_skips.outputs.skips }} run: | pip install pre-commit + export GIT_DISCOVERY_ACROSS_FILESYSTEM=1 git fetch origin if [ "$GITHUB_EVENT_NAME" == 'pull_request' ] then @@ -265,6 +266,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | pip install pre-commit + export GIT_DISCOVERY_ACROSS_FILESYSTEM=1 git fetch origin if [ "$GITHUB_EVENT_NAME" == 'pull_request' ] then From 29363a79108b9cddf6e8a8456204bcfda548fca7 Mon Sep 17 00:00:00 2001 From: Poh Peng Date: Fri, 10 Nov 2023 16:12:47 +0800 Subject: [PATCH 04/14] Get pwd --- .github/workflows/terraform.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/terraform.yaml b/.github/workflows/terraform.yaml index 1b85295..2d7eb91 100644 --- a/.github/workflows/terraform.yaml +++ b/.github/workflows/terraform.yaml @@ -149,6 +149,7 @@ jobs: run: | pip install pre-commit export GIT_DISCOVERY_ACROSS_FILESYSTEM=1 + pwd git fetch origin if [ "$GITHUB_EVENT_NAME" == 'pull_request' ] then @@ -267,6 +268,7 @@ jobs: run: | pip install pre-commit export GIT_DISCOVERY_ACROSS_FILESYSTEM=1 + pwd git fetch origin if [ "$GITHUB_EVENT_NAME" == 'pull_request' ] then From a87a017da0569514a9960e46c5460c4941277481 Mon Sep 17 00:00:00 2001 From: Poh Peng Date: Fri, 10 Nov 2023 16:19:26 +0800 Subject: [PATCH 05/14] Patch missing git right at the start --- .github/workflows/terraform.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/terraform.yaml b/.github/workflows/terraform.yaml index 2d7eb91..315946e 100644 --- a/.github/workflows/terraform.yaml +++ b/.github/workflows/terraform.yaml @@ -79,6 +79,11 @@ jobs: runs-on: - ${{ inputs.runner_label }} steps: + - name: Setup Node only for self-hosted runners + uses: actions/setup-node@v3 + with: + node-version: 19 + - name: Checkout uses: actions/checkout@v3 with: @@ -89,11 +94,6 @@ jobs: with: python-version: '3.11' - - name: Setup Node only for self-hosted runners - uses: actions/setup-node@v3 - with: - node-version: 19 - - name: Install missing binaries zip jq curl run: sudo apt-get update && sudo apt-get install zip jq curl git -y @@ -180,6 +180,9 @@ jobs: runs-on: - ${{ inputs.runner_label }} steps: + - name: Install missing binaries zip jq curl + run: sudo apt-get update && sudo apt-get install zip jq curl git -y + - name: Checkout uses: actions/checkout@v3 with: @@ -195,9 +198,6 @@ jobs: with: node-version: 19 - - name: Install missing binaries zip jq curl - run: sudo apt-get update && sudo apt-get install zip jq curl git -y - - run: mkdir -p "${TF_PLUGIN_CACHE_DIR}" - name: Cache Terraform uses: actions/cache@v3 From 1e7aa076bb3a53da5f89c5eff990700b210b4dc1 Mon Sep 17 00:00:00 2001 From: Poh Peng Date: Fri, 10 Nov 2023 16:41:51 +0800 Subject: [PATCH 06/14] Set the correct sequence --- .github/workflows/terraform.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/terraform.yaml b/.github/workflows/terraform.yaml index 315946e..4b5f744 100644 --- a/.github/workflows/terraform.yaml +++ b/.github/workflows/terraform.yaml @@ -79,10 +79,8 @@ jobs: runs-on: - ${{ inputs.runner_label }} steps: - - name: Setup Node only for self-hosted runners - uses: actions/setup-node@v3 - with: - node-version: 19 + - name: Install missing binaries zip jq curl + run: sudo apt-get update && sudo apt-get install zip jq curl git -y - name: Checkout uses: actions/checkout@v3 @@ -94,8 +92,10 @@ jobs: with: python-version: '3.11' - - name: Install missing binaries zip jq curl - run: sudo apt-get update && sudo apt-get install zip jq curl git -y + - name: Setup Node only for self-hosted runners + uses: actions/setup-node@v3 + with: + node-version: 19 - run: mkdir -p "${TF_PLUGIN_CACHE_DIR}" - name: Cache Terraform From 689b493b06a1a92e375f3d388b60c4d2144e6e11 Mon Sep 17 00:00:00 2001 From: Poh Peng Date: Fri, 10 Nov 2023 16:42:18 +0800 Subject: [PATCH 07/14] remove test code --- .github/workflows/terraform.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/terraform.yaml b/.github/workflows/terraform.yaml index 4b5f744..dde4c70 100644 --- a/.github/workflows/terraform.yaml +++ b/.github/workflows/terraform.yaml @@ -148,8 +148,6 @@ jobs: SKIP: ${{ steps.precommit_skips.outputs.skips }} run: | pip install pre-commit - export GIT_DISCOVERY_ACROSS_FILESYSTEM=1 - pwd git fetch origin if [ "$GITHUB_EVENT_NAME" == 'pull_request' ] then @@ -267,8 +265,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | pip install pre-commit - export GIT_DISCOVERY_ACROSS_FILESYSTEM=1 - pwd git fetch origin if [ "$GITHUB_EVENT_NAME" == 'pull_request' ] then From 43695a323e24cb8fc172da34f17d1a2f44aafaf4 Mon Sep 17 00:00:00 2001 From: Uchinda Padmaperuma <89894943+uchinda-sph@users.noreply.github.com> Date: Thu, 9 May 2024 11:25:42 +0800 Subject: [PATCH 08/14] update trivy version to 0.20.0 --- .github/workflows/terraform.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/terraform.yaml b/.github/workflows/terraform.yaml index dde4c70..2e531ff 100644 --- a/.github/workflows/terraform.yaml +++ b/.github/workflows/terraform.yaml @@ -315,7 +315,7 @@ jobs: submodules: ${{ inputs.enable_submodules }} - name: Run Trivy vulnerability scanner in IaC mode - uses: aquasecurity/trivy-action@0.12.0 + uses: aquasecurity/trivy-action@0.20.0 with: scan-type: 'config' hide-progress: false From d9a5c91d7a5c795aa6fb98e140d22f7d8dd077c7 Mon Sep 17 00:00:00 2001 From: Uchinda Padmaperuma <89894943+uchinda-sph@users.noreply.github.com> Date: Fri, 10 May 2024 00:41:24 +0800 Subject: [PATCH 09/14] add docker build --- .github/workflows/terraform.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/terraform.yaml b/.github/workflows/terraform.yaml index 67afbae..9368449 100644 --- a/.github/workflows/terraform.yaml +++ b/.github/workflows/terraform.yaml @@ -318,9 +318,14 @@ jobs: fetch-depth: 1 submodules: ${{ inputs.enable_submodules }} + - name: Build an image from Dockerfile + run: | + docker build -t docker.io/my-organization/my-app:${{ github.sha }} . --network host + - name: Run Trivy vulnerability scanner in IaC mode uses: aquasecurity/trivy-action@0.20.0 with: + image-ref: 'docker.io/my-organization/my-app:${{ github.sha }}' scan-type: 'fs' hide-progress: false format: 'sarif' From f845f4a9fcb5914ee0412db7c86c2e0ec24adcd4 Mon Sep 17 00:00:00 2001 From: Uchinda Padmaperuma <89894943+uchinda-sph@users.noreply.github.com> Date: Fri, 10 May 2024 00:46:35 +0800 Subject: [PATCH 10/14] revert the changes --- .github/workflows/terraform.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/terraform.yaml b/.github/workflows/terraform.yaml index 9368449..6c072b3 100644 --- a/.github/workflows/terraform.yaml +++ b/.github/workflows/terraform.yaml @@ -318,14 +318,14 @@ jobs: fetch-depth: 1 submodules: ${{ inputs.enable_submodules }} - - name: Build an image from Dockerfile - run: | - docker build -t docker.io/my-organization/my-app:${{ github.sha }} . --network host + # - name: Build an image from Dockerfile + # run: | + # docker build -t docker.io/my-organization/my-app:${{ github.sha }} . --network host - name: Run Trivy vulnerability scanner in IaC mode uses: aquasecurity/trivy-action@0.20.0 with: - image-ref: 'docker.io/my-organization/my-app:${{ github.sha }}' + # image-ref: 'docker.io/my-organization/my-app:${{ github.sha }}' scan-type: 'fs' hide-progress: false format: 'sarif' From 7712d1e318e3edaab778eba199c2704eb8a948cb Mon Sep 17 00:00:00 2001 From: Uchinda Padmaperuma <89894943+uchinda-sph@users.noreply.github.com> Date: Tue, 14 May 2024 12:12:46 +0800 Subject: [PATCH 11/14] fix trivy --- .github/workflows/terraform.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/terraform.yaml b/.github/workflows/terraform.yaml index 2e531ff..ffda773 100644 --- a/.github/workflows/terraform.yaml +++ b/.github/workflows/terraform.yaml @@ -317,7 +317,7 @@ jobs: - name: Run Trivy vulnerability scanner in IaC mode uses: aquasecurity/trivy-action@0.20.0 with: - scan-type: 'config' + scan-type: 'fs' hide-progress: false format: 'sarif' output: 'trivy-results.sarif' From 702c2d53e87e8afde7258381d1ab8f0bad7e27c9 Mon Sep 17 00:00:00 2001 From: Uchinda Padmaperuma <89894943+uchinda-sph@users.noreply.github.com> Date: Tue, 14 May 2024 12:53:15 +0800 Subject: [PATCH 12/14] fix name --- .github/workflows/terraform.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/terraform.yaml b/.github/workflows/terraform.yaml index 1abfc1e..6c072b3 100644 --- a/.github/workflows/terraform.yaml +++ b/.github/workflows/terraform.yaml @@ -225,7 +225,7 @@ jobs: path: /home/runner/.tflint.d/plugins key: ${{ runner.os }}-tflint-${{ hashFiles('.tflint.hcl') }} - - name: Setup TFLin + - name: Setup TFLint uses: terraform-linters/setup-tflint@v4 with: tflint_version: "v0.50.3" From e9950ef8f7b33583648dae46ca836d57578d81e6 Mon Sep 17 00:00:00 2001 From: Uchinda Padmaperuma <89894943+uchinda-sph@users.noreply.github.com> Date: Tue, 16 Jul 2024 09:59:53 +0800 Subject: [PATCH 13/14] update trivy (#97) (#98) * update trivy (#97) * [CIRDEVOPS-2553] Adds parameters to debug trivy issues (#99) * [CIRDEVOPS-2553] parameterize trivy output format * [CIRDEVOPS-2553] parameterize trivy output filename * [CIRDEVOPS-2553] parameterize trivy output filename [1] * [CIRDEVOPS-2553] Add parameter to inspect trivy output * [CIRDEVOPS-2553] When trivy inspect is on, upload result as an artifact --------- Co-authored-by: Abhishek Srivastava Co-authored-by: Paul Yeoh --- .github/workflows/terraform.yaml | 35 ++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/.github/workflows/terraform.yaml b/.github/workflows/terraform.yaml index 693b5fd..5c855b9 100644 --- a/.github/workflows/terraform.yaml +++ b/.github/workflows/terraform.yaml @@ -7,6 +7,21 @@ on: type: boolean default: true required: false + trivy_format: + description: Output format (table, json, sarif, github) + type: string + default: sarif + required: false + trivy_output: + description: Save results to a file + type: string + default: 'trivy-results.sarif' + required: false + trivy_inspect_output: + description: Print trivy output for inspection, set to 'true' for debugging purposes + type: string + default: 'false' + required: false main_branch: description: Name of the main branch type: string @@ -323,26 +338,34 @@ jobs: # docker build -t docker.io/my-organization/my-app:${{ github.sha }} . --network host - name: Run Trivy vulnerability scanner in IaC mode - uses: aquasecurity/trivy-action@0.20.0 + uses: aquasecurity/trivy-action@0.23.0 with: scan-type: 'config' hide-progress: false - format: 'sarif' - output: 'trivy-results.sarif' + format: ${{ inputs.trivy_format }} + output: ${{ inputs.trivy_output }} ignore-unfixed: true severity: 'CRITICAL,HIGH' + - name: Upload Trivy scan results to Github for inspection + if: ${{ inputs.trivy_inspect_output == 'true' }} + uses: actions/upload-artifact@v4 + with: + path: ${{ inputs.trivy_output }} + retention-days: 1 + # https://github.com/aquasecurity/trivy/issues/5003 - name: Remove git from url for sarif uploading + if: ${{ inputs.trivy_format == 'sarif' && inputs.trivy_output != '' }} shell: bash run: | - sed -i 's#git::https:/##g' trivy-results.sarif + sed -i 's#git::https:/##g' ${{ inputs.trivy_output }} - name: Upload Trivy scan results to GitHub Security tab + if: inputs.upload_sarif == true uses: github/codeql-action/upload-sarif@v3 with: - sarif_file: 'trivy-results.sarif' - # if: inputs.upload_sarif == true + sarif_file: ${{ inputs.trivy_output }} - name: Get changed files id: changed-files From 7bb0d5411e83ec0cdacf68c04a1096f70c8544c0 Mon Sep 17 00:00:00 2001 From: Uchinda Padmaperuma <89894943+uchinda-sph@users.noreply.github.com> Date: Wed, 7 Aug 2024 11:50:46 +0800 Subject: [PATCH 14/14] Sync with V2 (#102) * update trivy (#97) * [CIRDEVOPS-2553] Adds parameters to debug trivy issues (#99) * [CIRDEVOPS-2553] parameterize trivy output format * [CIRDEVOPS-2553] parameterize trivy output filename * [CIRDEVOPS-2553] parameterize trivy output filename [1] * [CIRDEVOPS-2553] Add parameter to inspect trivy output * [CIRDEVOPS-2553] When trivy inspect is on, upload result as an artifact --------- Co-authored-by: Abhishek Srivastava Co-authored-by: Paul Yeoh