DAT-19348 DevOps :: refactor databricks test-harness workflow to run …

…terraform once (#244)

* ✨ (lth.yml): restructure GitHub Actions workflow to improve clarity and organization by renaming jobs and adding dependencies
♻️ (lth.yml): separate infrastructure setup and teardown into distinct jobs for better management and readability
💡 (lth.yml): add comments to clarify the purpose of each job and step in the workflow

* ✨ (lth.yml): add output for DATABRICKS_URL to streamline database connection
♻️ (lth.yml): refactor Databricks config collection step for better clarity
🐛 (lth.yml): fix reference to DATABRICKS_URL in Liquibase test harness command
📝 (lth.yml): remove redundant stop test database logic for cleaner workflow

* ♻️ (lth.yml): remove redundant step to stop test database to streamline workflow
✨ (lth.yml): update needs syntax for stop-test-harness-infra job for clarity and consistency

* 🔧 (lth.yml): remove quotes around $GITHUB_OUTPUT to ensure proper variable expansion in the workflow configuration

* ♻️ (lth.yml): refactor environment variable assignment to streamline the process of setting DATABRICKS_URL in GitHub Actions workflow

.github/workflows/lth.yml

@@ -12,8 +12,8 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
-  liquibase-test-harness:
-    name: Liquibase Test Harness
+  start-test-harness-infra:
+    name: Start Liquibase Test Harness infra
     runs-on: ubuntu-22.04
     permissions:
       checks: write
@@ -24,14 +24,8 @@ jobs:
       TF_VAR_DBX_TOKEN: ${{ secrets.TH_DATABRICKS_WORKSPACE_TOKEN }}
       TF_VAR_TEST_CATALOG: main
       TF_VAR_TEST_SCHEMA: liquibase_harness_test_ds
-      WORKSPACE_ID: ${{ secrets.TH_DATABRICKS_WORKSPACE_ID }}
-      LIQUIBOT_TOKEN: ${{ secrets.LIQUIBOT_PAT }}
-      GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
-    strategy:
-      max-parallel: 1
-      matrix:
-        liquibase-support-level: [Foundational, Contributed, Advanced] # Define the different test levels to run
-      fail-fast: false # Set fail-fast to false to run all test levels even if some of them fail
+    outputs:
+      DATABRICKS_URL: ${{ steps.collect-databricks-config.outputs.DATABRICKS_URL }}
 
     steps:
       - name: Checkout code
@@ -50,11 +44,35 @@ jobs:
         working-directory: src/test/terraform
 
       - name: Collect Databricks Config
+        id: collect-databricks-config
         working-directory: src/test/terraform
         run: |
           CLUSTER_ID=$(terraform output -raw endpoint_url)
           DATABRICKS_HOST=${TF_VAR_DBX_HOST#https://}
-          echo "DATABRICKS_URL=jdbc:databricks://$DATABRICKS_HOST:443/default;transportMode=http;ssl=1;httpPath=/sql/1.0/warehouses/$CLUSTER_ID;AuthMech=3;ConnCatalog=$TF_VAR_TEST_CATALOG;ConnSchema=$TF_VAR_TEST_SCHEMA;EnableArrow=0" >> "$GITHUB_ENV" 
+          echo "DATABRICKS_URL=jdbc:databricks://$DATABRICKS_HOST:443/default;transportMode=http;ssl=1;httpPath=/sql/1.0/warehouses/$CLUSTER_ID;AuthMech=3;ConnCatalog=$TF_VAR_TEST_CATALOG;ConnSchema=$TF_VAR_TEST_SCHEMA;EnableArrow=0" >> $GITHUB_OUTPUT
+
+  liquibase-test-harness:
+    name: Liquibase Test Harness
+    runs-on: ubuntu-22.04
+    needs: start-test-harness-infra
+    permissions:
+      checks: write
+      pull-requests: write
+      contents: write
+    env:
+      WORKSPACE_ID: ${{ secrets.TH_DATABRICKS_WORKSPACE_ID }}
+      LIQUIBOT_TOKEN: ${{ secrets.LIQUIBOT_PAT }}
+      GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
+      TF_VAR_DBX_TOKEN: ${{ secrets.TH_DATABRICKS_WORKSPACE_TOKEN }}
+    strategy:
+      max-parallel: 1
+      matrix:
+        liquibase-support-level: [Foundational, Contributed, Advanced] # Define the different test levels to run
+      fail-fast: false # Set fail-fast to false to run all test levels even if some of them fail
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
 
       - name: Setup Temurin Java 17
         uses: actions/setup-java@v4
@@ -110,7 +128,7 @@ jobs:
 
       - name: Run ${{ matrix.liquibase-support-level }} Liquibase Test Harness # Run the Liquibase test harness at each test level
         if: always() # Run the action even if the previous steps fail
-        run: mvn -B -ntp -DdbPassword=${{env.TF_VAR_DBX_TOKEN}} -DdbUrl='${{env.DATABRICKS_URL}}' -Dtest=liquibase.ext.databricks.${{ matrix.liquibase-support-level }}ExtensionHarnessTestSuite test -Dliquibase.version=master-SNAPSHOT # Run the Liquibase test harness at each test level
+        run: mvn -B -ntp -DdbPassword=${{env.TF_VAR_DBX_TOKEN}} -DdbUrl='${{needs.start-test-harness-infra.outputs.DATABRICKS_URL}}' -Dtest=liquibase.ext.databricks.${{ matrix.liquibase-support-level }}ExtensionHarnessTestSuite test -Dliquibase.version=master-SNAPSHOT # Run the Liquibase test harness at each test level
 
       - name: Test Reporter # Generate a test report using the Test Reporter action
         uses: dorny/[email protected]
@@ -121,19 +139,28 @@ jobs:
           reporter: java-junit # Set the reporter to use
           fail-on-error: false # Set fail-on-error to false to show report even if it has failed tests
 
+  stop-test-harness-infra:
+    name: Stop Liquibase Test Harness infra
+    needs: [liquibase-test-harness]
+    if: always() # Always destroy, even if the previous steps fail
+    runs-on: ubuntu-22.04
+    permissions:
+      checks: write
+      pull-requests: write
+      contents: write
+    env:
+      TF_VAR_DBX_HOST: ${{ secrets.TH_DATABRICKS_WORKSPACE_HOST }}
+      TF_VAR_DBX_TOKEN: ${{ secrets.TH_DATABRICKS_WORKSPACE_TOKEN }}
+      TF_VAR_TEST_CATALOG: main
+      TF_VAR_TEST_SCHEMA: liquibase_harness_test_ds
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - run: terraform init
+        working-directory: src/test/terraform
+
       - name: Stop test database
-        if: always() # Always destroy, even if the previous steps fail
         working-directory: src/test/terraform
-        run: |
-          set -e
-          TERRAFORM_OUTPUT=$(terraform show -json)
-          if [ -z "$TERRAFORM_OUTPUT" ]; then
-            echo "Terraform output is empty. Skipping removal."
-          else
-            SCHEMA_EXISTS=$(echo $TERRAFORM_OUTPUT | jq -r '.values.root_module.resources[] | select(.address == "databricks_schema.test_harness") | .values.name')
-            if [ "$SCHEMA_EXISTS" == "liquibase_harness_test_ds" ]; then
-              terraform destroy -auto-approve
-            else
-              echo "Schema does not exist. Skipping removal."
-            fi
-          fi
+        run: terraform destroy -auto-approve