diff --git a/README.md b/README.md index 6c00716..62e5fbd 100644 --- a/README.md +++ b/README.md @@ -33,47 +33,66 @@ Removing this file can potentially cause unwanted changes to the project if the plugins: utilities: - name: elementary - namespace: elementary - pip_url: elementary-data[bigquery] git+https://github.com/potloc/elementary-ext.git - executable: elementary_extension + variant: elementary + pip_url: elementary-data[bigquery] git+https://github.com/potloc/elementary-ext + executable: elementary_invoker settings: - - name: project_dir - kind: string - value: ${MELTANO_PROJECT_ROOT}/transform/profiles/bigquery/ - - name: profiles_dir - kind: string - value: ${MELTANO_PROJECT_ROOT}/transform/profiles/bigquery/ - - name: file_path - kind: string - value: ${MELTANO_PROJECT_ROOT}/utilities/elementary/report.html - - name: skip_pre_invoke - env: ELEMENTARY_EXT_SKIP_PRE_INVOKE - kind: boolean - value: true - description: Whether to skip pre-invoke hooks which automatically run dbt clean and deps - - name: slack-token - kind: password - - name: slack-channel-name - kind: string - value: elementary-notifs + - name: project_dir + kind: string + value: ${MELTANO_PROJECT_ROOT}/transform/ + - name: profiles_dir + kind: string + value: ${MELTANO_PROJECT_ROOT}/transform/profiles/bigquery/ + - name: file_path + kind: string + value: ${MELTANO_PROJECT_ROOT}/utilities/elementary/report.html + - name: skip_pre_invoke + env: ELEMENTARY_EXT_SKIP_PRE_INVOKE + kind: boolean + value: true + description: + Whether to skip pre-invoke hooks which automatically run dbt clean + and deps + - name: slack-token + kind: password + - name: slack-channel-name + kind: string + value: elementary-notifs + - name: google-service-account-path + kind: string + - name: gcs-bucket-name + kind: string + - name: days-back + kind: string + - name: env + kind: string + commands: + initialize: + args: initialize + executable: elementary_extension + describe: + args: describe + executable: elementary_extension + monitor-report: + args: monitor-report + executable: elementary_extension + monitor-send-report: + args: monitor-send-report + executable: elementary_extension + send-report-gcs: + send-report --google-service-account-path ${ELEMENTARY_GOOGLE_SERVICE_ACCOUNT_PATH} + --gcs-bucket-name ${ELEMENTARY_GCS_BUCKET_NAME} --update-bucket-website true + --executions-limit 5 + send-report-slack: + send-report --slack-token ${ELEMENTARY_SLACK_TOKEN} --slack-channel-name ${ELEMENTARY_SLACK_CHANNEL_NAME} + config: profiles-dir: ${MELTANO_PROJECT_ROOT}/transform/profiles/bigquery/ file-path: ${MELTANO_PROJECT_ROOT}/utilities/elementary/report.html - slack-channel-name: team-data-engineering-notifications + slack-channel-name: my-channel-name + google-service-account-path: ${MELTANO_PROJECT_ROOT}/.secrets/elementary-gcs.json + gcs-bucket-name: my-storage-device skip_pre_invoke: true - commands: - monitor: - description: Allows your to run monitoring with elementary - args: monitor --profiles-dir ${ELEMENTARY_PROFILES_DIR} - monitor-report: - description: Allows your to create a local report - args: monitor report - --profiles-dir ${ELEMENTARY_PROFILES_DIR} - --file-path ${ELEMENTARY_FILE_PATH} - monitor-send-report: - description: Allows your to send a report through slack - args: monitor send-report - --profiles-dir ${ELEMENTARY_PROFILES_DIR} - --slack-token ${ELEMENTARY_SLACK_TOKEN} - --slack-channel-name ${ELEMENTARY_SLACK_CHANNEL_NAME} + env: prod + days-back: 3 ``` diff --git a/elementary_ext/extension.py b/elementary_ext/extension.py index eb0abed..de9a045 100644 --- a/elementary_ext/extension.py +++ b/elementary_ext/extension.py @@ -38,6 +38,14 @@ def __init__(self) -> None: self.slack_channel_name = os.getenv("ELEMENTARY_SLACK_CHANNEL_NAME", "") self.slack_token = os.getenv("ELEMENTARY_SLACK_TOKEN", "") + self.days_back= os.getenv("ELEMENTARY_DAYS_BACK", None) + self.timezone= os.getenv("ELEMENTARY_TIMEZONE", None) + self.dbt_quoting= os.getenv("ELEMENTARY_DBT_QUOTING", None) + self.disable_samples= os.getenv("ELEMENTARY_DISABLE_SAMPLES", None) + self.environment= os.getenv("ELEMENTARY_ENV", None) + self.full_refresh_dbt_models= os.getenv("ELEMENTARY_FULL_REFRESH_DBT_MODELS", None) + + self.dbt_profiles_dir = Path( os.getenv("ELEMENTARY_PROFILES_DIR", self.dbt_project_dir / "profiles") @@ -102,7 +110,26 @@ def invoke(self, command_name: str | None, *command_args: Any) -> None: log.info(f"Using config at `{self.config_dir_path}`...") elif self.dbt_profiles_dir != "": command_args = command_args + ("--profiles-dir=" + str(self.dbt_profiles_dir),) - log.info(f"Using profile.yml at `{self.dbt_profiles_dir}`...") + log.info(f"Using profile.yml at `{self.dbt_profiles_dir}`...") + if self.days_back is not None: + command_args = command_args + ("--days-back=" + str(self.days_back),) + log.info(f"Using days_back `{self.days_back}`...") + if self.timezone is not None: + command_args = command_args + ("--timezone=" + str(self.timezone),) + log.info(f"Using timezone `{self.timezone}`...") + if self.dbt_quoting is not None: + command_args = command_args + ("--dbt-quoting=" + str(self.dbt_quoting),) + log.info(f"Using dbt_quoting `{self.dbt_quoting}`...") + if self.disable_samples is not None: + command_args = command_args + ("--disable-samples=" + str(self.disable_samples),) + log.info(f"Using disable_samples `{self.disable_samples}`...") + if self.environment is not None: + command_args = command_args + ("--env=" + str(self.environment),) + log.info(f"Using environment `{self.environment}`...") + if self.full_refresh_dbt_models is not None: + command_args = command_args + ("--full-refresh-dbt-package" + str(self.full_refresh_dbt_models),) + log.info(f"Using full_refresh_dbt_models `{self.environment}`...") + self.elementary_invoker.run_and_log(command_name, *command_args) except subprocess.CalledProcessError as err: log_subprocess_error(