AP-1039 revamp CI (#11)

.github/workflows/ci.yml

@@ -0,0 +1,30 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+
+jobs:
+  lint_and_test:
+    name: Linting and Testing
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ 3.6, 3.7, 3.8 ]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Setup virtual environment
+        run: make venv
+
+      - name: Linting
+        run: make pylint

Makefile

@@ -0,0 +1,9 @@
+venv:
+	python3 -m venv venv ;\
+	. ./venv/bin/activate ;\
+	pip install --upgrade pip setuptools wheel ;\
+	pip install -e .[test]
+
+pylint:
+	. ./venv/bin/activate ;\
+	pylint tap_salesforce -d missing-docstring,invalid-name,line-too-long,too-many-locals,too-few-public-methods,fixme,stop-iteration-return,no-else-return,chained-comparison,broad-except

README.md

@@ -24,17 +24,14 @@ installation instructions for [Mac](http://docs.python-guide.org/en/latest/start
 It's recommended to use a virtualenv:
 
 ```bash
-  python3 -m venv venv
-  pip install pipelinewise-tap-salesforce
+      python3 -m venv venv
+      pip install pipelinewise-tap-salesforce
 ```
 
 or
 
 ```bash
-  python3 -m venv venv
-  . venv/bin/activate
-  pip install --upgrade pip
-  pip install .
+  make venv
 ```
 
 ### Create a Config file
@@ -52,26 +49,32 @@ or
 
 The `client_id` and `client_secret` keys are your OAuth Salesforce App secrets. The `refresh_token` is a secret created during the OAuth flow. For more info on the Salesforce OAuth flow, visit the [Salesforce documentation](https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/intro_understanding_web_server_oauth_flow.htm).
 
-The `start_date` is used by the tap as a bound on SOQL queries when searching for records.  This should be an [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) formatted date-time, like "2018-01-08T00:00:00Z". For more details, see the [Singer best practices for dates](https://github.com/singer-io/getting-started/blob/master/BEST_PRACTICES.md#dates).
+The `start_date` is used by the tap as a bound on SOQL queries when searching for records.  This should be an [RFC3339](https://www.ietf.org/rfc/rfc3339.txt) formatted date-time, like "2018-01-08T00:00:00Z". For more details, see the [Singer best practices for dates](https://github.com/singer-io/getting-started/blob/master/docs/BEST_PRACTICES.md#dates).
 
 The `api_type` is used to switch the behavior of the tap between using Salesforce's "REST" and "BULK" APIs. When new fields are discovered in Salesforce objects, the `select_fields_by_default` key describes whether or not the tap will select those fields by default.
 
 ## Run Discovery
 
 To run discovery mode, execute the tap with the config file.
 
-```
-> tap-salesforce --config config.json --discover > properties.json
+```bash
+ tap-salesforce --config config.json --discover > properties.json
 ```
 
 ## Sync Data
 
 To sync data, select fields in the `properties.json` output and run the tap.
 
+```bash
+ tap-salesforce --config config.json --properties properties.json [--state state.json]
 ```
-> tap-salesforce --config config.json --properties properties.json [--state state.json]
+
+## Linting
+
+```bash
+    make venv pylint
 ```
 
----
+## Licence
 
-Based on Stitch documentation
+GNU AFFERO GENERAL PUBLIC [LICENSE](./LICENSE)

setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup
 
 with open('README.md') as f:
-      long_description = f.read()
+    long_description = f.read()
 
 setup(name='pipelinewise-tap-salesforce',
       version='1.1.0',
@@ -22,15 +22,21 @@
           'pipelinewise-singer-python==1.*',
           'xmltodict==0.11.0'
       ],
+      extras_require={
+          'test': [
+              'pylint==2.9.*',
+          ]
+      },
+      python_requires='>=3.6',
       entry_points='''
           [console_scripts]
           tap-salesforce=tap_salesforce:main
       ''',
       packages=['tap_salesforce', 'tap_salesforce.salesforce'],
-      package_data = {
+      package_data={
           'tap_salesforce/schemas': [
               # add schema.json filenames here
           ]
       },
       include_package_data=True,
-)
+      )

tap_salesforce/__init__.py

@@ -151,7 +151,6 @@ def do_discover(sf):
         # Loop over the object's fields
         for f in fields:
             field_name = f['name']
-            field_type = f['type']
 
             if field_name == "Id":
                 found_id_field = True

tap_salesforce/salesforce/__init__.py

@@ -129,7 +129,7 @@ def log_backoff_attempt(details):
     LOGGER.info("ConnectionError detected, triggering backoff: %d try", details.get("tries"))
 
 
-def field_to_property_schema(field, mdata):
+def field_to_property_schema(field, mdata): # pylint: disable=too-many-branches
     property_schema = {}
 
     field_name = field['name']
@@ -253,7 +253,8 @@ def check_rest_quota_usage(self, headers):
                                                            percent_used_from_total,
                                                            self.quota_percent_total)
             raise TapSalesforceQuotaExceededException(total_message)
-        elif self.rest_requests_attempted > max_requests_for_run:
+
+        if self.rest_requests_attempted > max_requests_for_run:
             partial_message = ("This replication job has made {} REST requests ({:3.2f}% of " +
                                "total quota). Terminating replication due to allotted " +
                                "quota of {}% per replication.").format(self.rest_requests_attempted,

tap_salesforce/salesforce/bulk.py

@@ -77,7 +77,8 @@ def check_bulk_quota_usage(self):
                                                            percent_used,
                                                            self.sf.quota_percent_total)
             raise TapSalesforceQuotaExceededException(total_message)
-        elif self.sf.jobs_completed > max_requests_for_run:
+
+        if self.sf.jobs_completed > max_requests_for_run:
             partial_message = ("This replication job has completed {} Bulk API jobs ({:3.2f}% of " +
                                "total quota). Terminating replication due to allotted " +
                                "quota of {}% per replication.").format(self.sf.jobs_completed,

tap_salesforce/salesforce/rest.py

@@ -101,5 +101,5 @@ def _sync_records(self, url, headers, params):
 
             if next_records_url is None:
                 break
-            else:
-                url = "{}{}".format(self.sf.instance_url, next_records_url)
+
+            url = "{}{}".format(self.sf.instance_url, next_records_url)

tap_salesforce/sync.py

@@ -98,7 +98,7 @@ def sync_stream(sf, catalog_entry, state):
             singer.write_state(state)
         except RequestException as ex:
             raise Exception("Error syncing {}: {} Response: {}".format(
-                stream, ex, ex.response.text))
+                stream, ex, ex.response.text)) from ex
         except Exception as ex:
             raise Exception("Error syncing {}: {}".format(
                 stream, ex)) from ex