From da670e450948c93175eb5dd74251ca274fd3b7bd Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Tue, 19 Apr 2022 14:35:42 +0100 Subject: [PATCH] Skip tests if not configured --- tests/end_to_end/helpers/env.py | 49 +++++++++++-------- tests/end_to_end/target_snowflake/__init__.py | 3 ++ .../target_snowflake/tap_mariadb/__init__.py | 4 ++ .../target_snowflake/tap_mongodb/__init__.py | 4 ++ .../target_snowflake/tap_postgres/__init__.py | 4 ++ .../target_snowflake/tap_s3/__init__.py | 4 ++ 6 files changed, 47 insertions(+), 21 deletions(-) diff --git a/tests/end_to_end/helpers/env.py b/tests/end_to_end/helpers/env.py index 74cbaebc3..1cf9ef4bc 100644 --- a/tests/end_to_end/helpers/env.py +++ b/tests/end_to_end/helpers/env.py @@ -15,6 +15,7 @@ DIR = os.path.dirname(os.path.realpath(__file__)) +# pylint: disable=too-many-public-methods class E2EEnv: """Utilities class to run End to End tests @@ -22,14 +23,15 @@ class E2EEnv: to run SQL queries on the supported databases and to run common assertions on the supported databases""" - def __init__(self, project_dir): - self.sf_schema_postfix = f'_{str(uuid.uuid4())[:8]}' - self._load_env() + env = {} + sf_schema_postfix = f'_{str(uuid.uuid4())[:8]}' + def __init__(self, project_dir): # Generate test project YAMLs from templates self._init_test_project_dir(project_dir) - def _load_env(self): + @classmethod + def load_env(cls): """Connector properties vars: Load environment variables in priority order: @@ -49,7 +51,7 @@ def _load_env(self): load_dotenv( dotenv_path=os.path.join(DIR, '..', '..', '..', 'dev-project', '.env') ) - self.env = { + cls.env = { # ------------------------------------------------------------------ # Tap Postgres is a REQUIRED test connector and test database with test data available # in the docker environment @@ -205,7 +207,7 @@ def _load_env(self): 'optional': True, }, 'SCHEMA_POSTFIX': { - 'value': os.environ.get('TARGET_SNOWFLAKE_SCHEMA_POSTFIX', self.sf_schema_postfix), + 'value': os.environ.get('TARGET_SNOWFLAKE_SCHEMA_POSTFIX', cls.sf_schema_postfix), 'optional': True, } }, @@ -269,30 +271,30 @@ def _load_env(self): # Add is_configured keys for every connector # Useful to skip certain test cases dynamically when specific tap # or target database is not configured - self.env['TAP_POSTGRES']['is_configured'] = self._is_env_connector_configured( + cls.env['TAP_POSTGRES']['is_configured'] = cls._is_env_connector_configured( 'TAP_POSTGRES' ) - self.env['TAP_MYSQL']['is_configured'] = self._is_env_connector_configured( + cls.env['TAP_MYSQL']['is_configured'] = cls._is_env_connector_configured( 'TAP_MYSQL' ) - self.env['TAP_S3_CSV']['is_configured'] = self._is_env_connector_configured( + cls.env['TAP_S3_CSV']['is_configured'] = cls._is_env_connector_configured( 'TAP_S3_CSV' ) - self.env['TAP_MONGODB']['is_configured'] = self._is_env_connector_configured( + cls.env['TAP_MONGODB']['is_configured'] = cls._is_env_connector_configured( 'TAP_MONGODB' ) - self.env['TARGET_POSTGRES'][ + cls.env['TARGET_POSTGRES'][ 'is_configured' - ] = self._is_env_connector_configured('TARGET_POSTGRES') - self.env['TARGET_REDSHIFT'][ + ] = cls._is_env_connector_configured('TARGET_POSTGRES') + cls.env['TARGET_REDSHIFT'][ 'is_configured' - ] = self._is_env_connector_configured('TARGET_REDSHIFT') - self.env['TARGET_SNOWFLAKE'][ + ] = cls._is_env_connector_configured('TARGET_REDSHIFT') + cls.env['TARGET_SNOWFLAKE'][ 'is_configured' - ] = self._is_env_connector_configured('TARGET_SNOWFLAKE') - self.env['TARGET_BIGQUERY'][ + ] = cls._is_env_connector_configured('TARGET_SNOWFLAKE') + cls.env['TARGET_BIGQUERY'][ 'is_configured' - ] = self._is_env_connector_configured('TARGET_BIGQUERY') + ] = cls._is_env_connector_configured('TARGET_BIGQUERY') def _get_conn_env_var(self, connector, key): """Get the value of a specific variable in the self.env dict""" @@ -315,7 +317,8 @@ def get_aws_session(self): aws_secret_access_key=aws_secret_access_key, ) - def _is_env_connector_configured(self, env_connector): + @classmethod + def _is_env_connector_configured(cls, env_connector): """Detect if certain component(s) of env vars group is configured properly""" env_conns = [] if isinstance(env_connector, str): @@ -326,11 +329,11 @@ def _is_env_connector_configured(self, env_connector): raise Exception('env_connector must be string or list') for env_conn in env_conns: - for key, value in self.env[env_conn]['vars'].items(): + for key, value in cls.env[env_conn]['vars'].items(): # If value not defined and is not optional if not value['value'] and not value.get('optional'): # Value not defined but the entirely component is optional - if self.env[env_conn].get('optional'): + if cls.env[env_conn].get('optional'): return False # Value not defined but it's a required property raise Exception( @@ -662,3 +665,7 @@ def remove_all_state_files(): """Clean up state files to ensure tests behave the same every time""" for state_file in Path(CONFIG_DIR).glob('**/state.json'): state_file.unlink() + + +# Setup the class +E2EEnv.load_env() diff --git a/tests/end_to_end/target_snowflake/__init__.py b/tests/end_to_end/target_snowflake/__init__.py index 629cf452e..af7475db4 100644 --- a/tests/end_to_end/target_snowflake/__init__.py +++ b/tests/end_to_end/target_snowflake/__init__.py @@ -3,6 +3,8 @@ import unittest from pathlib import Path +import pytest + from tests.end_to_end.helpers import assertions, tasks from tests.end_to_end.helpers.env import E2EEnv @@ -11,6 +13,7 @@ CONFIG_DIR = os.path.join(USER_HOME, '.pipelinewise') +@pytest.mark.skipif(not E2EEnv.env['TARGET_SNOWFLAKE']['is_configured'], reason='Snowflake not configured.') class TargetSnowflake(unittest.TestCase): """ Base class for E2E tests for target snowflake diff --git a/tests/end_to_end/target_snowflake/tap_mariadb/__init__.py b/tests/end_to_end/target_snowflake/tap_mariadb/__init__.py index f9e42d9e2..964a8de5f 100644 --- a/tests/end_to_end/target_snowflake/tap_mariadb/__init__.py +++ b/tests/end_to_end/target_snowflake/tap_mariadb/__init__.py @@ -1,6 +1,10 @@ +import pytest + +from tests.end_to_end.helpers.env import E2EEnv from tests.end_to_end.target_snowflake import TargetSnowflake +@pytest.mark.skipif(not E2EEnv.env['TAP_MYSQL']['is_configured'], reason='MySql not configured.') class TapMariaDB(TargetSnowflake): """ Base class for E2E tests for tap mysql -> target snowflake diff --git a/tests/end_to_end/target_snowflake/tap_mongodb/__init__.py b/tests/end_to_end/target_snowflake/tap_mongodb/__init__.py index 460b50dbc..19472ffb9 100644 --- a/tests/end_to_end/target_snowflake/tap_mongodb/__init__.py +++ b/tests/end_to_end/target_snowflake/tap_mongodb/__init__.py @@ -1,6 +1,10 @@ +import pytest + +from tests.end_to_end.helpers.env import E2EEnv from tests.end_to_end.target_snowflake import TargetSnowflake +@pytest.mark.skipif(not E2EEnv.env['TAP_MYSQL']['is_configured'], reason='MySql not configured.') class TapMongoDB(TargetSnowflake): """ Base class for E2E tests for tap mongodb -> target snowflake diff --git a/tests/end_to_end/target_snowflake/tap_postgres/__init__.py b/tests/end_to_end/target_snowflake/tap_postgres/__init__.py index 3b20f80be..bca35771b 100644 --- a/tests/end_to_end/target_snowflake/tap_postgres/__init__.py +++ b/tests/end_to_end/target_snowflake/tap_postgres/__init__.py @@ -1,6 +1,10 @@ +import pytest + +from tests.end_to_end.helpers.env import E2EEnv from tests.end_to_end.target_snowflake import TargetSnowflake +@pytest.mark.skipif(not E2EEnv.env['TAP_POSTGRES']['is_configured'], reason='Postgres not configured.') class TapPostgres(TargetSnowflake): """ Base class for E2E tests for tap postgres -> target snowflake diff --git a/tests/end_to_end/target_snowflake/tap_s3/__init__.py b/tests/end_to_end/target_snowflake/tap_s3/__init__.py index ae4a54456..82b3588bd 100644 --- a/tests/end_to_end/target_snowflake/tap_s3/__init__.py +++ b/tests/end_to_end/target_snowflake/tap_s3/__init__.py @@ -1,6 +1,10 @@ +import pytest + +from tests.end_to_end.helpers.env import E2EEnv from tests.end_to_end.target_snowflake import TargetSnowflake +@pytest.mark.skipif(not E2EEnv.env['TAP_S3_CSV']['is_configured'], reason='S3 not configured.') class TapS3(TargetSnowflake): """ Base class for E2E tests for tap S3 -> target snowflake