DE-1833 Elementary - Reintroduce elements from quickstart (#19)

* Added new settings * Added new settings * Edited command args * Added spacing * Added equals sign * Added new commands
potloc · Jul 7, 2023 · 9499b47 · 9499b47
1 parent 5b19067
commit 9499b47
Show file tree

Hide file tree

Showing 2 changed files with 85 additions and 39 deletions.
diff --git a/README.md b/README.md
@@ -33,47 +33,66 @@ Removing this file can potentially cause unwanted changes to the project if the
  plugins:
     utilities:
       - name: elementary
-        namespace: elementary
-        pip_url: elementary-data[bigquery] git+https://github.com/potloc/elementary-ext.git
-        executable: elementary_extension
+        variant: elementary
+        pip_url: elementary-data[bigquery] git+https://github.com/potloc/elementary-ext
+        executable: elementary_invoker
         settings:
-        - name: project_dir
-          kind: string
-          value: ${MELTANO_PROJECT_ROOT}/transform/profiles/bigquery/
-        - name: profiles_dir
-          kind: string
-          value: ${MELTANO_PROJECT_ROOT}/transform/profiles/bigquery/
-        - name: file_path
-          kind: string
-          value: ${MELTANO_PROJECT_ROOT}/utilities/elementary/report.html
-        - name: skip_pre_invoke
-          env: ELEMENTARY_EXT_SKIP_PRE_INVOKE
-          kind: boolean
-          value: true
-          description: Whether to skip pre-invoke hooks which automatically run dbt clean and deps
-        - name: slack-token
-          kind: password
-        - name: slack-channel-name
-          kind: string
-          value: elementary-notifs
+          - name: project_dir
+            kind: string
+            value: ${MELTANO_PROJECT_ROOT}/transform/
+          - name: profiles_dir
+            kind: string
+            value: ${MELTANO_PROJECT_ROOT}/transform/profiles/bigquery/
+          - name: file_path
+            kind: string
+            value: ${MELTANO_PROJECT_ROOT}/utilities/elementary/report.html
+          - name: skip_pre_invoke
+            env: ELEMENTARY_EXT_SKIP_PRE_INVOKE
+            kind: boolean
+            value: true
+            description:
+              Whether to skip pre-invoke hooks which automatically run dbt clean
+              and deps
+          - name: slack-token
+            kind: password
+          - name: slack-channel-name
+            kind: string
+            value: elementary-notifs
+          - name: google-service-account-path
+            kind: string
+          - name: gcs-bucket-name
+            kind: string
+          - name: days-back
+            kind: string
+          - name: env
+            kind: string
+        commands:
+          initialize:
+            args: initialize
+            executable: elementary_extension
+          describe:
+            args: describe
+            executable: elementary_extension
+          monitor-report:
+            args: monitor-report
+            executable: elementary_extension
+          monitor-send-report:
+            args: monitor-send-report
+            executable: elementary_extension
+          send-report-gcs:
+            send-report --google-service-account-path ${ELEMENTARY_GOOGLE_SERVICE_ACCOUNT_PATH}
+            --gcs-bucket-name ${ELEMENTARY_GCS_BUCKET_NAME} --update-bucket-website true
+            --executions-limit 5
+          send-report-slack:
+            send-report --slack-token ${ELEMENTARY_SLACK_TOKEN} --slack-channel-name ${ELEMENTARY_SLACK_CHANNEL_NAME}
+
         config:
           profiles-dir: ${MELTANO_PROJECT_ROOT}/transform/profiles/bigquery/
           file-path: ${MELTANO_PROJECT_ROOT}/utilities/elementary/report.html
-          slack-channel-name: team-data-engineering-notifications
+          slack-channel-name: my-channel-name
+          google-service-account-path: ${MELTANO_PROJECT_ROOT}/.secrets/elementary-gcs.json
+          gcs-bucket-name: my-storage-device
           skip_pre_invoke: true
-        commands:
-          monitor:
-            description: Allows your to run monitoring with elementary
-            args: monitor --profiles-dir ${ELEMENTARY_PROFILES_DIR}
-          monitor-report:
-            description: Allows your to create a local report
-            args: monitor report
-                  --profiles-dir ${ELEMENTARY_PROFILES_DIR}
-                  --file-path ${ELEMENTARY_FILE_PATH}
-          monitor-send-report:
-            description: Allows your to send a report through slack
-            args: monitor send-report
-                  --profiles-dir ${ELEMENTARY_PROFILES_DIR}
-                  --slack-token ${ELEMENTARY_SLACK_TOKEN}
-                  --slack-channel-name ${ELEMENTARY_SLACK_CHANNEL_NAME}
+          env: prod
+          days-back: 3
 ```
diff --git a/elementary_ext/extension.py b/elementary_ext/extension.py
@@ -38,6 +38,14 @@ def __init__(self) -> None:
 
         self.slack_channel_name = os.getenv("ELEMENTARY_SLACK_CHANNEL_NAME", "")
         self.slack_token = os.getenv("ELEMENTARY_SLACK_TOKEN", "")
+        self.days_back= os.getenv("ELEMENTARY_DAYS_BACK", None)
+        self.timezone= os.getenv("ELEMENTARY_TIMEZONE", None)
+        self.dbt_quoting= os.getenv("ELEMENTARY_DBT_QUOTING", None)
+        self.disable_samples= os.getenv("ELEMENTARY_DISABLE_SAMPLES", None)
+        self.environment= os.getenv("ELEMENTARY_ENV", None)
+        self.full_refresh_dbt_models= os.getenv("ELEMENTARY_FULL_REFRESH_DBT_MODELS", None)
+
+
 
         self.dbt_profiles_dir = Path(
             os.getenv("ELEMENTARY_PROFILES_DIR", self.dbt_project_dir / "profiles")
@@ -102,7 +110,26 @@ def invoke(self, command_name: str | None, *command_args: Any) -> None:
                     log.info(f"Using config at `{self.config_dir_path}`...")
                 elif self.dbt_profiles_dir != "":
                     command_args = command_args + ("--profiles-dir=" + str(self.dbt_profiles_dir),)
-                    log.info(f"Using profile.yml at `{self.dbt_profiles_dir}`...")
+                    log.info(f"Using profile.yml at `{self.dbt_profiles_dir}`...")          
+                if self.days_back is not None:
+                    command_args = command_args + ("--days-back=" + str(self.days_back),)
+                    log.info(f"Using days_back `{self.days_back}`...")
+                if self.timezone is not None:
+                    command_args = command_args + ("--timezone=" + str(self.timezone),)
+                    log.info(f"Using timezone `{self.timezone}`...")
+                if self.dbt_quoting is not None:
+                    command_args = command_args + ("--dbt-quoting=" + str(self.dbt_quoting),)
+                    log.info(f"Using dbt_quoting `{self.dbt_quoting}`...")
+                if self.disable_samples is not None:
+                    command_args = command_args + ("--disable-samples=" + str(self.disable_samples),)
+                    log.info(f"Using disable_samples `{self.disable_samples}`...")
+                if self.environment is not None:
+                    command_args = command_args + ("--env=" + str(self.environment),)
+                    log.info(f"Using environment `{self.environment}`...")
+                if self.full_refresh_dbt_models is not None:
+                    command_args = command_args + ("--full-refresh-dbt-package" + str(self.full_refresh_dbt_models),)
+                    log.info(f"Using full_refresh_dbt_models `{self.environment}`...")
+
             self.elementary_invoker.run_and_log(command_name, *command_args)
         except subprocess.CalledProcessError as err:
             log_subprocess_error(