From b8d3e27a6c8ec5984c0b3d6b394a3356b7edd438 Mon Sep 17 00:00:00 2001 From: Christophe D Date: Mon, 23 Sep 2024 11:14:54 -0700 Subject: [PATCH 01/12] initial_objects --- data/Migration/V1.1.1__initial_objects.sql | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 data/Migration/V1.1.1__initial_objects.sql diff --git a/data/Migration/V1.1.1__initial_objects.sql b/data/Migration/V1.1.1__initial_objects.sql new file mode 100644 index 0000000..436371c --- /dev/null +++ b/data/Migration/V1.1.1__initial_objects.sql @@ -0,0 +1,6 @@ +CREATE SCHEMA DEMO; +CREATE TABLE HELLO_WORLD +( + FIRST_NAME VARCHAR + ,LAST_NAME VARCHAR +); \ No newline at end of file From fb72817b4cb9939db96b642a74b3077870047712 Mon Sep 17 00:00:00 2001 From: Christophe D Date: Mon, 23 Sep 2024 11:43:33 -0700 Subject: [PATCH 02/12] Create snowflake-devops-demo.yml --- .github/workflows/snowflake-devops-demo.yml | 43 +++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/snowflake-devops-demo.yml diff --git a/.github/workflows/snowflake-devops-demo.yml b/.github/workflows/snowflake-devops-demo.yml new file mode 100644 index 0000000..3922fc5 --- /dev/null +++ b/.github/workflows/snowflake-devops-demo.yml @@ -0,0 +1,43 @@ +name: snowflake-devops-demo + +# Controls when the action will run. +on: + push: + branches: + - main + paths: + - 'migrations/**' + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + deploy-snowflake-changes-job: + runs-on: ubuntu-latest + + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Use Python 3.8.x + uses: actions/setup-python@v2.2.1 + with: + python-version: 3.8.x + + - name: Run schemachange + env: + SF_ACCOUNT: ${{ secrets.SF_ACCOUNT }} + SF_USERNAME: ${{ secrets.SF_USERNAME }} + SF_ROLE: ${{ secrets.SF_ROLE }} + SF_WAREHOUSE: ${{ secrets.SF_WAREHOUSE }} + SF_DATABASE: ${{ secrets.SF_DATABASE }} + SNOWFLAKE_PASSWORD: ${{ secrets.SF_PASSWORD }} + run: | + echo "GITHUB_WORKSPACE: $GITHUB_WORKSPACE" + python --version + echo "Step 1: Installing schemachange" + pip install schemachange + + echo "Step 2: Running schemachange" + schemachange -f $GITHUB_WORKSPACE/migrations -a $SF_ACCOUNT -u $SF_USERNAME -r $SF_ROLE -w $SF_WAREHOUSE -d $SF_DATABASE -c $SF_DATABASE.SCHEMACHANGE.CHANGE_HISTORY --create-change-history-table From 4337118721db903d7560d882799688f596e01abc Mon Sep 17 00:00:00 2001 From: Christophe D Date: Mon, 23 Sep 2024 13:26:57 -0700 Subject: [PATCH 03/12] usedb --- data/Migration/V1.1.1__initial_objects.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/data/Migration/V1.1.1__initial_objects.sql b/data/Migration/V1.1.1__initial_objects.sql index 436371c..23ac2b5 100644 --- a/data/Migration/V1.1.1__initial_objects.sql +++ b/data/Migration/V1.1.1__initial_objects.sql @@ -1,3 +1,4 @@ +USE DATABASE demo_db; CREATE SCHEMA DEMO; CREATE TABLE HELLO_WORLD ( From acceae4df90eaba79575fe00099fc8ff4cefae36 Mon Sep 17 00:00:00 2001 From: Christophe D Date: Mon, 23 Sep 2024 14:07:51 -0700 Subject: [PATCH 04/12] Update snowflake-devops-demo.yml --- .github/workflows/snowflake-devops-demo.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/snowflake-devops-demo.yml b/.github/workflows/snowflake-devops-demo.yml index 3922fc5..4f6b85e 100644 --- a/.github/workflows/snowflake-devops-demo.yml +++ b/.github/workflows/snowflake-devops-demo.yml @@ -40,4 +40,4 @@ jobs: pip install schemachange echo "Step 2: Running schemachange" - schemachange -f $GITHUB_WORKSPACE/migrations -a $SF_ACCOUNT -u $SF_USERNAME -r $SF_ROLE -w $SF_WAREHOUSE -d $SF_DATABASE -c $SF_DATABASE.SCHEMACHANGE.CHANGE_HISTORY --create-change-history-table + # schemachange -f $GITHUB_WORKSPACE/migrations -a $SF_ACCOUNT -u $SF_USERNAME -r $SF_ROLE -w $SF_WAREHOUSE -d $SF_DATABASE -c $SF_DATABASE.SCHEMACHANGE.CHANGE_HISTORY --create-change-history-table From 4846cf80eeb938ce01df7dd035fda97d20a8f1ea Mon Sep 17 00:00:00 2001 From: Christophe D Date: Mon, 23 Sep 2024 14:12:54 -0700 Subject: [PATCH 05/12] Update snowflake-devops-demo.yml --- .github/workflows/snowflake-devops-demo.yml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/snowflake-devops-demo.yml b/.github/workflows/snowflake-devops-demo.yml index 4f6b85e..1d1a655 100644 --- a/.github/workflows/snowflake-devops-demo.yml +++ b/.github/workflows/snowflake-devops-demo.yml @@ -25,19 +25,11 @@ jobs: with: python-version: 3.8.x - - name: Run schemachange - env: - SF_ACCOUNT: ${{ secrets.SF_ACCOUNT }} - SF_USERNAME: ${{ secrets.SF_USERNAME }} - SF_ROLE: ${{ secrets.SF_ROLE }} - SF_WAREHOUSE: ${{ secrets.SF_WAREHOUSE }} - SF_DATABASE: ${{ secrets.SF_DATABASE }} - SNOWFLAKE_PASSWORD: ${{ secrets.SF_PASSWORD }} - run: | + run: | echo "GITHUB_WORKSPACE: $GITHUB_WORKSPACE" python --version echo "Step 1: Installing schemachange" pip install schemachange echo "Step 2: Running schemachange" - # schemachange -f $GITHUB_WORKSPACE/migrations -a $SF_ACCOUNT -u $SF_USERNAME -r $SF_ROLE -w $SF_WAREHOUSE -d $SF_DATABASE -c $SF_DATABASE.SCHEMACHANGE.CHANGE_HISTORY --create-change-history-table + schemachange -f $GITHUB_WORKSPACE/migrations -a $SF_ACCOUNT -u $SF_USERNAME -r $SF_ROLE -w $SF_WAREHOUSE -d $SF_DATABASE -c $SF_DATABASE.SCHEMACHANGE.CHANGE_HISTORY --create-change-history-table From 1a993beab45ead32ed9c4ab7eae95ecbbaf73044 Mon Sep 17 00:00:00 2001 From: Christophe D Date: Mon, 23 Sep 2024 14:20:34 -0700 Subject: [PATCH 06/12] Update deploy_pipeline.yml --- .github/workflows/deploy_pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy_pipeline.yml b/.github/workflows/deploy_pipeline.yml index 9cd698b..bcea091 100644 --- a/.github/workflows/deploy_pipeline.yml +++ b/.github/workflows/deploy_pipeline.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest env: - REPO_NAME: "quickstart_common.public.quickstart_repo" + REPO_NAME: "Snowflake-Devops" # Read connection secrets SNOWFLAKE_CONNECTIONS_DEFAULT_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} SNOWFLAKE_CONNECTIONS_DEFAULT_USER: ${{ secrets.SNOWFLAKE_USER }} From f31bee7a07c84947584f62965597a16224b69dbf Mon Sep 17 00:00:00 2001 From: Christophe D Date: Mon, 23 Sep 2024 14:22:32 -0700 Subject: [PATCH 07/12] Update deploy_pipeline.yml --- .github/workflows/deploy_pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy_pipeline.yml b/.github/workflows/deploy_pipeline.yml index bcea091..9cd698b 100644 --- a/.github/workflows/deploy_pipeline.yml +++ b/.github/workflows/deploy_pipeline.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest env: - REPO_NAME: "Snowflake-Devops" + REPO_NAME: "quickstart_common.public.quickstart_repo" # Read connection secrets SNOWFLAKE_CONNECTIONS_DEFAULT_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} SNOWFLAKE_CONNECTIONS_DEFAULT_USER: ${{ secrets.SNOWFLAKE_USER }} From ec1ed0e6ec38faa8f2ef461e62fb8e6ee75ced8e Mon Sep 17 00:00:00 2001 From: Christophe D Date: Mon, 23 Sep 2024 14:31:56 -0700 Subject: [PATCH 08/12] Update snowflake-devops-demo.yml --- .github/workflows/snowflake-devops-demo.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/snowflake-devops-demo.yml b/.github/workflows/snowflake-devops-demo.yml index 1d1a655..e93dccc 100644 --- a/.github/workflows/snowflake-devops-demo.yml +++ b/.github/workflows/snowflake-devops-demo.yml @@ -19,7 +19,13 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - name: Checkout repository uses: actions/checkout@v2 - +env: + SF_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SF_USERNAME: ${{ secrets.SNOWFLAKE_USERNAME }} + SF_ROLE: ${{ secrets.SNOWFLAKE_ROLE }} + SF_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }} + SF_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }} + SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - name: Use Python 3.8.x uses: actions/setup-python@v2.2.1 with: From beb76714ba5106a16663c886d99a30abbb99d4da Mon Sep 17 00:00:00 2001 From: Christophe D Date: Mon, 23 Sep 2024 14:44:40 -0700 Subject: [PATCH 09/12] TEST --- data/Migration/V1.1.1__initial_objects.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data/Migration/V1.1.1__initial_objects.sql b/data/Migration/V1.1.1__initial_objects.sql index 23ac2b5..a9cf484 100644 --- a/data/Migration/V1.1.1__initial_objects.sql +++ b/data/Migration/V1.1.1__initial_objects.sql @@ -1,7 +1,8 @@ USE DATABASE demo_db; -CREATE SCHEMA DEMO; CREATE TABLE HELLO_WORLD ( FIRST_NAME VARCHAR ,LAST_NAME VARCHAR + ,aDDRESS VARCHAR + ); \ No newline at end of file From e01ffccf214988d3d6a8686a7f0ada3a806df562 Mon Sep 17 00:00:00 2001 From: Christophe D Date: Mon, 23 Sep 2024 14:53:15 -0700 Subject: [PATCH 10/12] TET2 --- data/Migration/V1.1.1__initial_objects.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/Migration/V1.1.1__initial_objects.sql b/data/Migration/V1.1.1__initial_objects.sql index a9cf484..86a27ea 100644 --- a/data/Migration/V1.1.1__initial_objects.sql +++ b/data/Migration/V1.1.1__initial_objects.sql @@ -1,5 +1,5 @@ USE DATABASE demo_db; -CREATE TABLE HELLO_WORLD +CREATE TABLE HELLO_WORLDD ( FIRST_NAME VARCHAR ,LAST_NAME VARCHAR From 663f8262d67b75a352e797da158c1ecfecde4c10 Mon Sep 17 00:00:00 2001 From: Christophe D Date: Mon, 23 Sep 2024 15:03:38 -0700 Subject: [PATCH 11/12] AAS --- data/Migration/V1.1.1__initial_objects.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/Migration/V1.1.1__initial_objects.sql b/data/Migration/V1.1.1__initial_objects.sql index 86a27ea..74fd4ef 100644 --- a/data/Migration/V1.1.1__initial_objects.sql +++ b/data/Migration/V1.1.1__initial_objects.sql @@ -1,5 +1,5 @@ USE DATABASE demo_db; -CREATE TABLE HELLO_WORLDD +CREATE TABLE HELLO_WORLDDQ ( FIRST_NAME VARCHAR ,LAST_NAME VARCHAR From 771c19b77eef4dc8173ae21c60d0e3954b04536b Mon Sep 17 00:00:00 2001 From: Christophe D Date: Mon, 23 Sep 2024 15:10:23 -0700 Subject: [PATCH 12/12] ddd --- deploy_parametrized_pipeline.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deploy_parametrized_pipeline.sql b/deploy_parametrized_pipeline.sql index 9fe030f..f6f1328 100644 --- a/deploy_parametrized_pipeline.sql +++ b/deploy_parametrized_pipeline.sql @@ -1,3 +1,5 @@ execute immediate from 'steps/01_setup_snowflake.sql' using (environment => '{{environment}}'); execute immediate from 'steps/03_harmonize_data.sql' using (environment => '{{environment}}'); execute immediate from 'steps/04_orchestrate_jobs.sql' using (environment => '{{environment}}', retention_time => {{retention_time}}); + +execute immediate from 'migration/V1.1.1__initial_objects.sql' using (environment => '{{environment}}'); \ No newline at end of file