chore: Changes for Arch (#685)

.github/workflows/el_ssa_ip_addresses.yml

@@ -0,0 +1,31 @@
+name: Meltano - Spreadsheets Anywhere IP Addresses
+
+on:
+  schedule:
+    - cron:  '0 10 * * *'
+
+  workflow_dispatch:
+
+jobs:
+  el_ssa_ip_addresses:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: data/
+    environment: prod
+    env:
+      MELTANO_ENVIRONMENT: prod
+      MELTANO_SEND_ANONYMOUS_USAGE_STATS: 'false'
+      SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+        cache: 'pip'
+    - run: pip install -r requirements.txt
+    - run: echo "${{secrets.MELTANO_ENV_FILE }}" > .env
+    # Install Plugins
+    - run: meltano install --schedule=spreadsheets_anywhere_el
+    # Run Test
+    - run: meltano run spreadsheets_anywhere_el

.github/workflows/hub_metrics_publish.yml

@@ -0,0 +1,35 @@
+name: Meltano - Hub Metrics Publish
+
+on:
+  schedule:
+    - cron:  '0 12 * * *'
+
+  workflow_dispatch:
+
+jobs:
+  hub_metrics_publish:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: data/
+    environment: prod
+    env:
+      MELTANO_ENVIRONMENT: prod
+      MELTANO_SEND_ANONYMOUS_USAGE_STATS: 'false'
+      SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
+      TAP_SNOWFLAKE_METRICS_TABLES: '["b${{ github.SHA }}_MELTANO_HUB.FACT_VARIANT_HUB_METRICS"]'
+      TAP_SNOWFLAKE_AUDIT_TABLES: '["b${{ github.SHA }}_MELTANO_HUB.HUB_METRICS_AUDIT"]'
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+        cache: 'pip'
+    - run: pip install -r requirements.txt
+    - run: echo "${{secrets.MELTANO_ENV_FILE }}" > .env
+    # Install Plugins
+    - run: meltano install --schedule=hub_metrics_publish
+    # Run Test
+    - run: meltano run hub_metrics_publish

.github/workflows/slack_notifications.yml

@@ -0,0 +1,35 @@
+name: Meltano - Slack Publish
+
+on:
+  schedule:
+    - cron:  '0 12 * * *'
+
+  workflow_dispatch:
+
+jobs:
+  slack_notifications:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: data/
+    environment: prod
+    env:
+      MELTANO_ENVIRONMENT: prod
+      MELTANO_SEND_ANONYMOUS_USAGE_STATS: 'false'
+      SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
+      TAP_SNOWFLAKE_SINGER_ACTIVITY: '["MELTANO_HUB.SINGER_ACTIVITY_NOTIFICATIONS"]'
+      TAP_SNOWFLAKE_MELTANO_ACTIVITY: '["MELTANO_HUB.MELTANO_ACTIVITY_NOTIFICATIONS"]'
+      TARGET_APPRISE_SINGER_ACTIVITY_URIS: ${{ secrets.APPRISE_SINGER_ACTIVITY_WEBHOOK }}
+      TARGET_APPRISE_MELTANO_ACTIVITY_URIS: ${{ secrets.APPRISE_MELTANO_ACTIVITY_WEBHOOK }}
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+        cache: 'pip'
+    - run: pip install -r requirements.txt
+    - run: echo "${{secrets.MELTANO_ENV_FILE }}" > .env
+    # Install Plugins
+    - run: meltano install --schedule=slack_notifications
+    # Run Test
+    - run: meltano run slack_notifications

data/orchestrate/orchestrators.meltano.yml

@@ -97,7 +97,7 @@ jobs:
 
 - name: slack_notifications
   tasks:
-  - dbt-snowflake:run_slack_notifications
+  # - dbt-snowflake:run_slack_notifications
   - tap-snowflake-singer-activity target-apprise-singer-activity
   - tap-snowflake-meltano-activity target-apprise-meltano-activity
 
@@ -117,8 +117,8 @@ jobs:
 - name: spreadsheets_anywhere_el
   tasks:
   - tap-spreadsheets-anywhere coalesce-gcp-ips target-snowflake
-  - dbt-snowflake:test_source_spreadsheets_anywhere
-  - dbt-snowflake:run_snapshot_spreadsheets_anywhere
+  # - dbt-snowflake:test_source_spreadsheets_anywhere
+  # - dbt-snowflake:run_snapshot_spreadsheets_anywhere
 
 - name: snowplow_el
   tasks:
@@ -160,8 +160,8 @@ jobs:
 
 - name: hub_metrics_publish
   tasks:
-  - dbt-snowflake:run_hub_metrics
-  - dbt-snowflake:test_hub_metrics
+  # - dbt-snowflake:run_hub_metrics
+  # - dbt-snowflake:test_hub_metrics
   # - great_expectations:test_dbt_hub_metrics
   - tap-snowflake-metrics target-yaml-metrics awscli:s3_copy_metrics
   - tap-snowflake-audit target-yaml-audit awscli:s3_copy_audit

data/transform/dbt_project.yml

@@ -23,14 +23,14 @@ clean-targets:
 - dbt_packages
 - logs
 
-on-run-end: "{% if target.name in ('prod', 'cicd') %}{{ dbt_artifacts.upload_results(results) }}{% endif %}"
+on-run-end: "{% if target.name in ('prod', 'cicd') %} {{ dbt_artifacts.upload_results(results) }} {% endif %}"
 
 seeds:
   quote_columns: false
   +schema: SEED
 
 snapshots:
-  target_database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP') }}"
+  target_database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP', 'PREP') }}"
   target_schema: snapshots
   transient: false
 
@@ -41,7 +41,7 @@ models:
     # Staging
     # ------------------
     staging:
-      +database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP') }}"
+      +database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP', 'PREP') }}"
       cloudwatch:
         +schema: cloudwatch
       dynamodb:
@@ -77,7 +77,7 @@ models:
     # Common
     # ------------------
     common:
-      +database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP') }}"
+      +database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP', 'PREP') }}"
       +schema: workspace
       +materialized: view
     # ------------------
@@ -87,21 +87,21 @@ models:
       +materialized: table
       telemetry:
         base:
-          +database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP') }}"
+          +database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP', 'PREP') }}"
           +schema: workspace
           +materialized: view
         +schema: telemetry
       community:
         base:
-          +database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP') }}"
+          +database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP', 'PREP') }}"
           +schema: workspace
           +materialized: view
         +schema: community
   dbt_artifacts:
     +schema: dbt_artifacts
     staging:
-      +database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP') }}"
+      +database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP', 'PREP') }}"
       +schema: dbt_artifacts
     sources:
-      +database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP') }}"
+      +database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP', 'PREP') }}"
       +schema: "{{ env_var('DBT_SNOWFLAKE_SOURCE_SCHEMA_PREFIX', '') }}SOURCE_DBT_ARTIFACTS"

data/transform/models/staging/cloudwatch/sources.yml

@@ -2,7 +2,7 @@ config-version: 2
 version: 2
 sources:
   - name: tap_cloudwatch
-    database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW") }}'
+    database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW", "RAW") }}'
     schema: '{{ env_var("DBT_SNOWFLAKE_SOURCE_SCHEMA_PREFIX", "") }}TAP_CLOUDWATCH'
     tables:
       - name: log

data/transform/models/staging/dynamodb/sources.yml

@@ -2,7 +2,7 @@ config-version: 2
 version: 2
 sources:
   - name: tap_dynamodb
-    database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW") }}'
+    database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW", "RAW") }}'
     schema: '{{ env_var("DBT_SNOWFLAKE_SOURCE_SCHEMA_PREFIX", "") }}TAP_DYNAMODB'
     tables:
       - name: workload_metadata_table

data/transform/models/staging/github_meltano/sources.yml

@@ -2,7 +2,7 @@ config-version: 2
 version: 2
 sources:
   - name: tap_github_meltano
-    database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW") }}'
+    database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW", "RAW") }}'
     schema: '{{ env_var("DBT_SNOWFLAKE_SOURCE_SCHEMA_PREFIX", "") }}TAP_GITHUB_MELTANO'
     tables:
       - name: issues

data/transform/models/staging/github_search/sources.yml

@@ -2,7 +2,7 @@ config-version: 2
 version: 2
 sources:
   - name: tap_github_search
-    database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW") }}'
+    database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW", "RAW") }}'
     schema: '{{ env_var("DBT_SNOWFLAKE_SOURCE_SCHEMA_PREFIX", "") }}TAP_GITHUB_SEARCH'
     tables:
       - name: issues

data/transform/models/staging/gitlab/sources.yml

@@ -2,14 +2,14 @@ config-version: 2
 version: 2
 sources:
   - name: tap_gitlab
-    database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW") }}'
+    database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW", "RAW") }}'
     schema: '{{ env_var("DBT_SNOWFLAKE_SOURCE_SCHEMA_PREFIX", "") }}TAP_GITLAB'
     tables:
       - name: issues
       - name: merge_requests
       - name: projects
   - name: tap_gitlab_hotglue
-    database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW") }}'
+    database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW", "RAW") }}'
     schema: '{{ env_var("DBT_SNOWFLAKE_SOURCE_SCHEMA_PREFIX", "") }}TAP_GITLAB_HOTGLUE'
     tables:
       - name: issues

data/transform/models/staging/gitlab/stg_gitlab__issues.sql

@@ -1,5 +1,3 @@
-{{ config(materialized='table') }}
-
 WITH source_meltano AS (
 
     SELECT

data/transform/models/staging/gitlab/stg_gitlab__merge_requests.sql

@@ -1,5 +1,3 @@
-{{ config(materialized='table') }}
-
 WITH source_meltano AS (
 
     SELECT

data/transform/models/staging/gitlab/stg_gitlab__projects.sql

@@ -1,5 +1,3 @@
-{{ config(materialized='table') }}
-
 WITH source AS (
 
     SELECT

data/transform/models/staging/google_analytics/sources.yml

@@ -2,7 +2,7 @@ config-version: 2
 version: 2
 sources:
   - name: tap_google_analytics
-    database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW") }}'
+    database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW", "RAW") }}'
     schema: '{{ env_var("DBT_SNOWFLAKE_SOURCE_SCHEMA_PREFIX", "") }}TAP_GOOGLE_ANALYTICS'
     tables:
       - name: events

data/transform/models/staging/meltanohub/sources.yml

@@ -2,7 +2,7 @@ config-version: 2
 version: 2
 sources:
   - name: tap_meltanohub
-    database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW") }}'
+    database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW", "RAW") }}'
     schema: '{{ env_var("DBT_SNOWFLAKE_SOURCE_SCHEMA_PREFIX", "") }}TAP_MELTANOHUB'
     tables:
       - name: plugins

data/transform/models/staging/slack/sources.yml

@@ -2,14 +2,14 @@ config-version: 2
 version: 2
 sources:
   - name: tap_slack
-    database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW") }}'
+    database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW", "RAW") }}'
     schema: '{{ env_var("DBT_SNOWFLAKE_SOURCE_SCHEMA_PREFIX", "") }}TAP_SLACK'
     tables:
       - name: users
       - name: messages
       - name: channels
   - name: tap_slack_public
-    database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW") }}'
+    database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW", "RAW") }}'
     schema: '{{ env_var("DBT_SNOWFLAKE_SOURCE_SCHEMA_PREFIX", "") }}TAP_SLACK_PUBLIC'
     tables:
       - name: messages

data/transform/models/staging/snowplow/sources.yml

@@ -2,7 +2,7 @@ config-version: 2
 version: 2
 sources:
   - name: snowplow
-    database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW") }}'
+    database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW", "RAW") }}'
     schema: '{{ env_var("DBT_SNOWFLAKE_SOURCE_SCHEMA_PREFIX", "") }}SNOWPLOW'
     tables:
       - name: events

data/transform/snapshots/spreadsheets_anywhere/sources.yml

@@ -2,7 +2,7 @@ config-version: 2
 version: 2
 sources:
   - name: tap_spreadsheets_anywhere
-    database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW") }}'
+    database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW", "RAW") }}'
     schema: '{{ env_var("DBT_SNOWFLAKE_SOURCE_SCHEMA_PREFIX", "") }}TAP_SPREADSHEETS_ANYWHERE'
     tables:
       - name: aws_ips

data/transform/transformers.meltano.yml

@@ -45,3 +45,4 @@ plugins:
       account: epa06486
       database_raw: RAW
       database_prep: PREP
+      profiles_dir: $MELTANO_PROJECT_ROOT/transform/profiles/squared_snowflake