Merge pull request #75 from gdubya/add_abfs_support #1
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Azurite (local) functional tests | |
| on: [push, pull_request,repository_dispatch] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| azurite-tests-linux: | |
| name: Azurite tests (Linux) | |
| runs-on: ubuntu-latest | |
| container: 'quay.io/pypa/manylinux2014_x86_64' | |
| env: | |
| VCPKG_TARGET_TRIPLET: 'x64-linux' | |
| GEN: Ninja | |
| VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake | |
| AZURE_STORAGE_CONNECTION_STRING: 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;' | |
| AZURE_STORAGE_ACCOUNT: devstoreaccount1 | |
| HTTP_PROXY_RUNNING: '1' | |
| ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| submodules: 'true' | |
| - name: install Azure test service | |
| run: | | |
| yum install -y nodejs npm squid | |
| npm install -g azurite | |
| echo -e "[azure-cli]\nname=Azure CLI\nbaseurl=https://packages.microsoft.com/yumrepos/azure-cli\nenabled=1\ngpgcheck=1\ngpgkey=https://packages.microsoft.com/keys/microsoft.asc" | tee /etc/yum.repos.d/azure-cli.repo | |
| yum install -y azure-cli | |
| - name: Launch & populate Azure test service | |
| run: | | |
| azurite > azurite_log.txt 2>&1 & | |
| ./scripts/run_squid.sh --port 3128 --log_dir squid_logs & | |
| ./scripts/run_squid.sh --port 3129 --log_dir squid_auth_logs --auth & | |
| sleep 10 | |
| ./scripts/upload_test_files_to_azurite.sh | |
| - name: Setup ManyLinux2014 | |
| run: | | |
| ./duckdb/scripts/setup_manylinux2014.sh general aws-cli ccache ssh openssl python_alias | |
| - name: Setup vcpkg | |
| uses: lukka/[email protected] | |
| with: | |
| vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6 | |
| # Build extension | |
| - name: Build extension | |
| env: | |
| GEN: ninja | |
| run: | | |
| make release | |
| - name: Test extension | |
| run: | | |
| make test | |
| - name: Run test data integrity check | |
| run: | | |
| ./build/release/duckdb -c "CREATE PERSISTENT SECRET s1 (TYPE AZURE, CONNECTION_STRING '$AZURE_STORAGE_CONNECTION_STRING')" | |
| DUCKDB_AZURE_PERSISTENT_SECRET_AVAILABLE=1 ./build/release/test/unittest "*test/sql/test_data_integrity.test" | |
| - name: Azure test server log | |
| if: always() | |
| shell: bash | |
| run: | | |
| echo "## azurite" | |
| cat azurite_log.txt | |
| echo "## squid" | |
| cat squid_logs/* | |
| echo "## squid auth" | |
| cat squid_auth_logs/* | |
| azurite-tests-macos: | |
| name: Azurite tests (MacOS) | |
| runs-on: macos-latest | |
| env: | |
| VCPKG_TARGET_TRIPLET: 'x64-osx' | |
| OSX_BUILD_ARCH: 'x86_64' | |
| GEN: Ninja | |
| VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake | |
| AZURE_STORAGE_CONNECTION_STRING: 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;' | |
| AZURE_STORAGE_ACCOUNT: devstoreaccount1 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| submodules: 'true' | |
| - name: Install Ninja | |
| run: brew install ninja | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb_azure' }} | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.11' | |
| - name: Setup vcpkg | |
| uses: lukka/run-vcpkg@v11 | |
| with: | |
| vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6 | |
| - name: Build extension | |
| shell: bash | |
| run: | | |
| make release | |
| - uses: actions/setup-node@v3 | |
| - name: Launch & populate Azure test service | |
| run: | | |
| npm install -g azurite | |
| azurite > azurite_log.txt 2>&1 & | |
| sleep 10 | |
| ./scripts/upload_test_files_to_azurite.sh | |
| - name: Test Extension | |
| shell: bash | |
| run: | | |
| make test | |
| - name: Run test data integrity check | |
| run: | | |
| ./build/release/duckdb -c "CREATE PERSISTENT SECRET s1 (TYPE AZURE, CONNECTION_STRING '$AZURE_STORAGE_CONNECTION_STRING')" | |
| DUCKDB_AZURE_PERSISTENT_SECRET_AVAILABLE=1 ./build/release/test/unittest "*test/sql/test_data_integrity.test" | |
| - name: Azure test server log | |
| if: always() | |
| shell: bash | |
| run: | | |
| echo "## azurite" | |
| cat azurite_log.txt | |
| azurite-tests-windows: | |
| name: Azurite tests (Windows) | |
| runs-on: windows-latest | |
| env: | |
| VCPKG_TARGET_TRIPLET: 'x64-windows' | |
| GEN: Ninja | |
| VCPKG_ROOT: ${{ github.workspace }}\vcpkg | |
| VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}\vcpkg\scripts\buildsystems\vcpkg.cmake | |
| AZURE_STORAGE_CONNECTION_STRING: 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;' | |
| AZURE_STORAGE_ACCOUNT: devstoreaccount1 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| submodules: 'true' | |
| - name: Setup vcpkg | |
| uses: lukka/run-vcpkg@v11 | |
| with: | |
| vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6 | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.11' | |
| - name: Build extension | |
| run: | | |
| make release | |
| - uses: actions/setup-node@v3 | |
| - name: Launch & populate Azure test service | |
| run: | | |
| npm install -g azurite | |
| azurite > azurite_log.txt 2>&1 & | |
| sleep 10 | |
| ./scripts/upload_test_files_to_azurite.sh | |
| - name: Test Extension | |
| shell: bash | |
| run: | | |
| build/release/test/Release/unittest.exe --test-dir . "[sql]" | |
| - name: Run test data integrity check | |
| run: | | |
| ls build | |
| ls build/release | |
| ./build/release/Release/duckdb.exe -c "CREATE PERSISTENT SECRET s1 (TYPE AZURE, CONNECTION_STRING '$AZURE_STORAGE_CONNECTION_STRING')" | |
| DUCKDB_AZURE_PERSISTENT_SECRET_AVAILABLE=1 build/release/test/Release/unittest.exe "*test/sql/test_data_integrity.test" | |
| - name: Azure test server log | |
| if: always() | |
| shell: bash | |
| run: | | |
| cat azurite_log.txt |