diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 51b7f02..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,23 +0,0 @@ -version: 2 -jobs: - integration_tests: - docker: - - image: circleci/python:3.6.2 - steps: - - checkout - - - run: - name: install dependencies - command: make venv - - - run: - name: 'Integration Tests' - command: | - export LOGGING_CONF_FILE=$(pwd)/sample_logging.conf - make integration_test - -workflows: - version: 2 - build: - jobs: - - integration_tests diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 33a2244..9353294 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,34 @@ jobs: - name: Pylinting run: make pylint -# No unit tests :( -# - name: Unit Tests -# run: make unit_test + integration_test: + name: Integration Testing + runs-on: ubuntu-latest + environment: ci_tests + strategy: + matrix: + python-version: [ 3.8 ] + concurrency: + group: integration_tests-${{ github.head_ref }} + cancel-in-progress: true + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup virtual environment + run: make venv + + - name: Integration tests + run: make integration_test + env: + TAP_SNOWFLAKE_ACCOUNT: ${{ secrets.TAP_SNOWFLAKE_ACCOUNT }} + TAP_SNOWFLAKE_DBNAME: ${{ secrets.TAP_SNOWFLAKE_DBNAME }} + TAP_SNOWFLAKE_USER: ${{ secrets.TAP_SNOWFLAKE_USER }} + TAP_SNOWFLAKE_PASSWORD: ${{ secrets.TAP_SNOWFLAKE_PASSWORD }} + TAP_SNOWFLAKE_WAREHOUSE: ${{ secrets.TAP_SNOWFLAKE_WAREHOUSE }} diff --git a/Makefile b/Makefile index 7ab8888..aa77d42 100644 --- a/Makefile +++ b/Makefile @@ -18,4 +18,4 @@ unit_test: integration_test: . ./venv/bin/activate ;\ - pytest tests/integration/ -vv --cov tap_snowflake + pytest tests/integration/ -vvx --cov tap_snowflake diff --git a/README.md b/README.md index 76dd3a7..bb4197a 100644 --- a/README.md +++ b/README.md @@ -28,20 +28,9 @@ make venv ### Configuration -1. Create a `config.json` file with connection details to snowflake. - - ```json - { - "account": "rtxxxxx.eu-central-1", - "dbname": "database_name", - "user": "my_user", - "password": "password", - "warehouse": "my_virtual_warehouse", - "tables": "db.schema.table1,db.schema.table2" - } - ``` - -**Note**: `tables` is a mandatory parameter as well to avoid long running catalog discovery process. +1. Create a `config.json` file with connection details to snowflake, here is a [sample config file](./config_sample.json). + +**Note**: `tables` is a mandatory parameter as well to avoid a long-running catalog discovery process. Please specify fully qualified table and view names and only that ones that you need to extract otherwise you can end up with very long running discovery mode of this tap. Discovery mode is analysing table structures but Snowflake doesn't like selecting lot of rows from `INFORMATION_SCHEMA` or running `SHOW` commands that returns lot of @@ -105,7 +94,9 @@ specified in the table's metadata as well. export TAP_SNOWFLAKE_ACCOUNT= export TAP_SNOWFLAKE_DBNAME= export TAP_SNOWFLAKE_USER= - export TAP_SNOWFLAKE_PASSWORD= + export TAP_SNOWFLAKE_PASSWORD= + export TAP_SNOWFLAKE_PRIVATE_KEY_PATH= + export TAP_SNOWFLAKE_PRIVATE_KEY_PASSPHRASE= export TAP_SNOWFLAKE_WAREHOUSE= ``` diff --git a/config_sample.json b/config_sample.json new file mode 100644 index 0000000..62905bc --- /dev/null +++ b/config_sample.json @@ -0,0 +1,10 @@ +{ + "account": "xxxxx.eu-central-1", + "dbname": "database_name", + "user": "my_user", + "password": "password", + "private_key_path": "/path/pk", + "private_key_passphrase": "strong passphrase", + "warehouse": "my_virtual_warehouse", + "tables": "db.schema.table1,db.schema.table2" +} \ No newline at end of file diff --git a/tests/integration/test_tap_snowflake.py b/tests/integration/test_tap_snowflake.py index 3560cda..e3dc12c 100644 --- a/tests/integration/test_tap_snowflake.py +++ b/tests/integration/test_tap_snowflake.py @@ -32,6 +32,8 @@ class TestTypeMapping(unittest.TestCase): @classmethod def setUpClass(cls): + + cls.config = test_utils.get_db_config() cls.snowflake_conn = test_utils.get_test_connection() with cls.snowflake_conn.open_connection() as open_conn: @@ -110,9 +112,9 @@ def test_discover_catalog_with_multiple_table(self): # Three tables should be discovered tap_stream_ids = [s.tap_stream_id for s in catalog.streams] self.assertCountEqual(tap_stream_ids, - ['ANALYTICS_DB_TEST-TAP_SNOWFLAKE_TEST-EMPTY_TABLE_1', - 'ANALYTICS_DB_TEST-TAP_SNOWFLAKE_TEST-EMPTY_TABLE_2', - 'ANALYTICS_DB_TEST-TAP_SNOWFLAKE_TEST-TEST_TYPE_MAPPING']) + [f'{self.config["dbname"]}-TAP_SNOWFLAKE_TEST-EMPTY_TABLE_1', + f'{self.config["dbname"]}-TAP_SNOWFLAKE_TEST-EMPTY_TABLE_2', + f'{self.config["dbname"]}-TAP_SNOWFLAKE_TEST-TEST_TYPE_MAPPING']) def test_discover_catalog_with_single_table(self): """Validate if discovering catalog with filter_tables option working as expected""" @@ -123,7 +125,7 @@ def test_discover_catalog_with_single_table(self): # Only one table should be discovered tap_stream_ids = [s.tap_stream_id for s in catalog.streams] self.assertCountEqual(tap_stream_ids, - ['ANALYTICS_DB_TEST-TAP_SNOWFLAKE_TEST-EMPTY_TABLE_2']) + [f'{self.config["dbname"]}-TAP_SNOWFLAKE_TEST-EMPTY_TABLE_2']) def test_discover_catalog_with_not_existing_table(self): """Validate if discovering catalog raises as exception when table not exist""" @@ -141,7 +143,7 @@ def test_discover_catalog_with_view(self): # Only one view should be discovered tap_stream_ids = [s.tap_stream_id for s in catalog.streams] self.assertCountEqual(tap_stream_ids, - ['ANALYTICS_DB_TEST-TAP_SNOWFLAKE_TEST-EMPTY_VIEW_1']) + [f'{self.config["dbname"]}-TAP_SNOWFLAKE_TEST-EMPTY_VIEW_1']) def test_decimal(self): self.assertEqual(self.dt_schema.properties['C_DECIMAL'],