AP-1067 move integration tests to GHA (#72)

transferwise · Nov 5, 2021 · bc8e8da · bc8e8da
1 parent 75204ae
commit bc8e8da
Show file tree

Hide file tree

Showing 6 changed files with 55 additions and 47 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -32,6 +32,34 @@ jobs:
       - name: Pylinting
         run: make pylint
 
-# No unit tests :(
-#      - name: Unit Tests
-#        run: make unit_test
+  integration_test:
+    name: Integration Testing
+    runs-on: ubuntu-latest
+    environment: ci_tests
+    strategy:
+      matrix:
+        python-version: [ 3.8 ]
+    concurrency:
+      group: integration_tests-${{ github.head_ref }}
+      cancel-in-progress: true
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Setup virtual environment
+        run: make venv
+
+      - name: Integration tests
+        run: make integration_test
+        env:
+          TAP_SNOWFLAKE_ACCOUNT: ${{ secrets.TAP_SNOWFLAKE_ACCOUNT }}
+          TAP_SNOWFLAKE_DBNAME: ${{ secrets.TAP_SNOWFLAKE_DBNAME }}
+          TAP_SNOWFLAKE_USER: ${{ secrets.TAP_SNOWFLAKE_USER }}
+          TAP_SNOWFLAKE_PASSWORD: ${{ secrets.TAP_SNOWFLAKE_PASSWORD }}
+          TAP_SNOWFLAKE_WAREHOUSE: ${{ secrets.TAP_SNOWFLAKE_WAREHOUSE }}
diff --git a/Makefile b/Makefile
@@ -18,4 +18,4 @@ unit_test:
 
 integration_test:
 	. ./venv/bin/activate ;\
-	pytest tests/integration/ -vv --cov tap_snowflake
+	pytest tests/integration/ -vvx --cov tap_snowflake
diff --git a/README.md b/README.md
@@ -28,20 +28,9 @@ make venv
 
 ### Configuration
 
-1. Create a `config.json` file with connection details to snowflake.
-
-   ```json
-   {
-     "account": "rtxxxxx.eu-central-1",
-     "dbname": "database_name",
-     "user": "my_user",
-     "password": "password",
-     "warehouse": "my_virtual_warehouse",
-     "tables": "db.schema.table1,db.schema.table2"
-   }
-   ```
-
-**Note**: `tables` is a mandatory parameter as well to avoid long running catalog discovery process.
+1. Create a `config.json` file with connection details to snowflake, here is a [sample config file](./config_sample.json).
+
+**Note**: `tables` is a mandatory parameter as well to avoid a long-running catalog discovery process.
 Please specify fully qualified table and view names and only that ones that you need to extract otherwise you can
 end up with very long running discovery mode of this tap. Discovery mode is analysing table structures but
 Snowflake doesn't like selecting lot of rows from `INFORMATION_SCHEMA` or running `SHOW` commands that returns lot of
@@ -105,7 +94,9 @@ specified in the table's metadata as well.
   export TAP_SNOWFLAKE_ACCOUNT=<snowflake-account-name>
   export TAP_SNOWFLAKE_DBNAME=<snowflake-database-name>
   export TAP_SNOWFLAKE_USER=<snowflake-user>
-  export TAP_SNOWFLAKE_PASSWORD=<snowfale-password>
+  export TAP_SNOWFLAKE_PASSWORD=<snowflake-password>
+  export TAP_SNOWFLAKE_PRIVATE_KEY_PATH=<snowflake-pk-path>
+  export TAP_SNOWFLAKE_PRIVATE_KEY_PASSPHRASE=<snowflake-passphrase>
   export TAP_SNOWFLAKE_WAREHOUSE=<snowflake-warehouse>
 ```
 

diff --git a/config_sample.json b/config_sample.json
@@ -0,0 +1,10 @@
+{
+  "account": "xxxxx.eu-central-1",
+  "dbname": "database_name",
+  "user": "my_user",
+  "password": "password",
+  "private_key_path": "/path/pk",
+  "private_key_passphrase": "strong passphrase",
+  "warehouse": "my_virtual_warehouse",
+  "tables": "db.schema.table1,db.schema.table2"
+}
diff --git a/tests/integration/test_tap_snowflake.py b/tests/integration/test_tap_snowflake.py
@@ -32,6 +32,8 @@ class TestTypeMapping(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
+
+        cls.config = test_utils.get_db_config()
         cls.snowflake_conn = test_utils.get_test_connection()
 
         with cls.snowflake_conn.open_connection() as open_conn:
@@ -110,9 +112,9 @@ def test_discover_catalog_with_multiple_table(self):
         # Three tables should be discovered
         tap_stream_ids = [s.tap_stream_id for s in catalog.streams]
         self.assertCountEqual(tap_stream_ids,
-                              ['ANALYTICS_DB_TEST-TAP_SNOWFLAKE_TEST-EMPTY_TABLE_1',
-                               'ANALYTICS_DB_TEST-TAP_SNOWFLAKE_TEST-EMPTY_TABLE_2',
-                               'ANALYTICS_DB_TEST-TAP_SNOWFLAKE_TEST-TEST_TYPE_MAPPING'])
+                              [f'{self.config["dbname"]}-TAP_SNOWFLAKE_TEST-EMPTY_TABLE_1',
+                               f'{self.config["dbname"]}-TAP_SNOWFLAKE_TEST-EMPTY_TABLE_2',
+                               f'{self.config["dbname"]}-TAP_SNOWFLAKE_TEST-TEST_TYPE_MAPPING'])
 
     def test_discover_catalog_with_single_table(self):
         """Validate if discovering catalog with filter_tables option working as expected"""
@@ -123,7 +125,7 @@ def test_discover_catalog_with_single_table(self):
         # Only one table should be discovered
         tap_stream_ids = [s.tap_stream_id for s in catalog.streams]
         self.assertCountEqual(tap_stream_ids,
-                              ['ANALYTICS_DB_TEST-TAP_SNOWFLAKE_TEST-EMPTY_TABLE_2'])
+                              [f'{self.config["dbname"]}-TAP_SNOWFLAKE_TEST-EMPTY_TABLE_2'])
 
     def test_discover_catalog_with_not_existing_table(self):
         """Validate if discovering catalog raises as exception when table not exist"""
@@ -141,7 +143,7 @@ def test_discover_catalog_with_view(self):
         # Only one view should be discovered
         tap_stream_ids = [s.tap_stream_id for s in catalog.streams]
         self.assertCountEqual(tap_stream_ids,
-                              ['ANALYTICS_DB_TEST-TAP_SNOWFLAKE_TEST-EMPTY_VIEW_1'])
+                              [f'{self.config["dbname"]}-TAP_SNOWFLAKE_TEST-EMPTY_VIEW_1'])
 
     def test_decimal(self):
         self.assertEqual(self.dt_schema.properties['C_DECIMAL'],