Merge pull request #2335 from dlt-hub/devel

master merge for 1.7.0 release

.github/workflows/test_destination_sqlalchemy.yml

@@ -87,11 +87,11 @@ jobs:
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-local-destinations
 
       - name: Install dependencies
-        run: poetry install --no-interaction -E parquet -E filesystem -E sqlalchemy -E cli --with sentry-sdk --with pipeline && poetry run pip install ibis-framework[duckdb,postgres,bigquery,snowflake,mssql,clickhouse] && poetry run pip install mysqlclient && poetry run pip install "sqlalchemy==${{ matrix.sqlalchemy }}"
+        run: poetry install --no-interaction -E parquet -E filesystem -E sqlalchemy -E cli --with sentry-sdk --with pipeline && poetry run pip install ibis-framework[duckdb,postgres,bigquery,snowflake,mssql,clickhouse] && poetry run pip install pymysql && poetry run pip install "sqlalchemy==${{ matrix.sqlalchemy }}"
 
       - name: create secrets.toml
         run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
 
       # always run full suite, also on branches
-      - run: poetry run pytest tests/load -x --ignore tests/load/sources
+      - run: poetry run pytest tests/load --ignore tests/load/sources
         name: Run tests Linux

.github/workflows/test_doc_snippets.yml

@@ -87,6 +87,9 @@ jobs:
           path: .venv
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
 
+      - name: Install dlt-plus nightly devel build without cache
+        run: poetry run pip install --upgrade --force-reinstall --no-cache-dir https://dlt-packages.fra1.digitaloceanspaces.com/dlt-plus/dlt_plus-0.0.0+nightly-py3-none-any.whl
+
       - name: run docs preprocessor
         run: make preprocess-docs
 
@@ -100,11 +103,10 @@ jobs:
       - name: create secrets.toml for snippets
         run: pwd && echo "$DLT_SECRETS_TOML" > docs/website/docs/.dlt/secrets.toml
 
-      - name: Run linter and tests on examples
-        run: make lint-and-test-examples
-
       - name: Run linter and tests on snippets
         run: make lint-and-test-snippets
 
+      - name: Run linter and tests on examples
+        run: make lint-and-test-examples
 
 

.github/workflows/test_plus.yml

@@ -0,0 +1,110 @@
+name: plus | plus
+
+#
+# dlt-plus smoke tests against the nightly build.
+#
+
+on:
+  pull_request:
+    branches:
+      - master
+      - devel
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  RUNTIME__LOG_LEVEL: ERROR
+  RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
+
+jobs:
+  get_docs_changes:
+    name: docs changes
+    uses: ./.github/workflows/get_docs_changes.yml
+
+  run_common:
+    name: test
+    needs: get_docs_changes
+    if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
+        python-version: ["3.10.x", "3.11.x", "3.12.x"]
+        plus_dep: ["dlt-plus", "https://dlt-packages.fra1.digitaloceanspaces.com/dlt-plus/dlt_plus-0.0.0+nightly-py3-none-any.whl"]
+        # Test all python versions on ubuntu only
+        exclude:
+        - os: "macos-latest"
+          python-version: "3.10.x"
+        - os: "macos-latest"
+          python-version: "3.12.x"
+        - os: "windows-latest"
+          python-version: "3.10.x"
+        - os: "windows-latest"
+          python-version: "3.12.x"
+
+    defaults:
+      run:
+        shell: bash
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Check out
+        uses: actions/checkout@master
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Poetry
+        # https://github.com/snok/install-poetry#running-on-windows
+        uses: snok/[email protected]
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+          version: 1.8.5
+
+      # NOTE: needed for mssql source tests in plus
+      - name: Install ODBC driver for SQL Server
+        run: |
+          sudo ACCEPT_EULA=Y apt-get install --yes msodbcsql18
+        if: matrix.os == 'ubuntu-latest'
+
+      # NOTE: do not cache. we want to have a clean state each run and we upgrade depdendencies later
+      # - name: Load cached venv
+      #   id: cached-poetry-dependencies
+      #   uses: actions/cache@v3
+      #   with:
+      #     # path: ${{ steps.pip-cache.outputs.dir }}
+      #     path: .venv
+      #     key: venv-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install all dependencies
+        run: make dev
+
+
+      - name: Install dlt-plus nightly devel build without cache
+        run: poetry run pip install --upgrade --force-reinstall --no-cache-dir ${{ matrix.plus_dep }}
+
+      - name: Run tests
+        run: poetry run pytest tests/plus
+        if: matrix.os == 'ubuntu-latest'
+
+      - name: Run tests on mac on win without mssql driver
+        run: poetry run pytest tests/plus -m "not mssql"
+        if: matrix.os == 'macos-latest' || matrix.os == 'windows-latest'
+
+  matrix_job_required_check:
+    name: common | common tests
+    needs: run_common
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - name: Check matrix job results
+        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
+        run: |
+          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1

Makefile

@@ -47,14 +47,13 @@ dev: has-poetry
 	poetry install --all-extras --with docs,providers,pipeline,sources,sentry-sdk
 
 lint:
-	./tools/check-package.sh
 	poetry run python ./tools/check-lockfile.py
 	poetry run mypy --config-file mypy.ini dlt tests
 	poetry run flake8 --max-line-length=200 dlt
-	poetry run flake8 --max-line-length=200 tests --exclude tests/reflection/module_cases
+	poetry run flake8 --max-line-length=200 tests --exclude tests/reflection/module_cases,tests/common/reflection/cases/modules/
 	poetry run black dlt docs tests --check --diff --color --extend-exclude=".*syntax_error.py"
 	# poetry run isort ./ --diff
-	# $(MAKE) lint-security
+	$(MAKE) lint-security
 
 format:
 	poetry run black dlt docs tests --extend-exclude='.*syntax_error.py|_storage/.*'
@@ -78,7 +77,8 @@ test-examples:
 	cd docs/examples && poetry run pytest
 
 lint-security:
-	poetry run bandit -r dlt/ -n 3 -l
+	# go for ll by cleaning up eval and SQL warnings.
+	poetry run bandit -r dlt/ -n 3 -lll
 
 test:
 	poetry run pytest tests

dlt/cli/deploy_command_helpers.py

@@ -9,8 +9,15 @@
 from typing import List, Optional, Sequence, Tuple, Any, Dict
 
 # optional dependencies
-import pipdeptree
-import cron_descriptor
+try:
+    import pipdeptree
+except ImportError:
+    pipdeptree = None
+
+try:
+    import cron_descriptor
+except ImportError:
+    cron_descriptor = None
 
 import dlt
 

dlt/cli/init_command.py

@@ -306,7 +306,7 @@ def init_command(
     templates_storage = _get_templates_storage()
 
     # get current run context
-    run_ctx = run_context.current()
+    run_ctx = run_context.active()
 
     # discover type of source
     source_type: files_ops.TSourceType = "template"

dlt/cli/plugins.py

@@ -26,15 +26,18 @@
     DLT_DEPLOY_DOCS_URL,
 )
 
+from dlt.cli.deploy_command import (
+    DeploymentMethods,
+    COMMAND_DEPLOY_REPO_LOCATION,
+    SecretFormats,
+)
+
 try:
-    from dlt.cli.deploy_command import (
-        DeploymentMethods,
-        COMMAND_DEPLOY_REPO_LOCATION,
-        SecretFormats,
-    )
+    import pipdeptree
+    import cron_descriptor
 
     deploy_command_available = True
-except ModuleNotFoundError:
+except ImportError:
     deploy_command_available = False
 
 
@@ -481,9 +484,6 @@ def configure_parser(self, parser: argparse.ArgumentParser) -> None:
             "pipeline_script_path", metavar="pipeline-script-path", help="Path to a pipeline script"
         )
 
-        if not deploy_command_available:
-            return
-
         deploy_comm.add_argument(
             "--location",
             default=COMMAND_DEPLOY_REPO_LOCATION,

dlt/cli/telemetry_command.py

@@ -31,7 +31,7 @@ def change_telemetry_status_command(enabled: bool) -> None:
     ]
     # write local config
     # TODO: use designated (main) config provider (for non secret values) ie. taken from run context
-    run_ctx = run_context.current()
+    run_ctx = run_context.active()
     config = ConfigTomlProvider(run_ctx.settings_dir)
     if not config.is_empty:
         write_values(config._config_toml, telemetry_value, overwrite_existing=True)

dlt/cli/utils.py

@@ -66,7 +66,7 @@ def get_telemetry_status() -> bool:
 
 def make_dlt_settings_path(path: str = None) -> str:
     """Returns path to file in dlt settings folder. Returns settings folder if path not specified."""
-    ctx = run_context.current()
+    ctx = run_context.active()
     if not path:
         return ctx.settings_dir
     return ctx.get_setting(path)