Dbt teradata team 1.6 #346
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI Integration Tests | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| push: | |
| branches: | |
| - 'main' | |
| jobs: | |
| test: | |
| strategy: | |
| matrix: | |
| python: ['3.8', '3.9','3.10', '3.11'] | |
| runs-on: macos-12 | |
| name: Functional test | |
| steps: | |
| - name: Setup Python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - name: Check out repository code | |
| uses: actions/checkout@v2 | |
| - name: Download Vantage Express | |
| shell: bash | |
| run: | | |
| brew install hudochenkov/sshpass/sshpass | |
| echo "my IP address is: " $(ipconfig getifaddr en0) | |
| mkdir /tmp/downloads | |
| cd /tmp/downloads | |
| curl -L "https://d289lrf5tw1zls.cloudfront.net/database/teradata-express/VantageExpress17.20_Sles12_20220819081111.7z?Expires=1704067200&Key-Pair-Id=APKAJ3SWQUPWKYVMO2WQ&Signature=$VE_URL_SIGNATURE" \ | |
| --compressed -o ve.7z | |
| env: | |
| VE_URL_SIGNATURE: ${{ secrets.VE_URL_SIGNATURE }} | |
| - name: Unzip Vantage Express | |
| shell: bash | |
| run: | | |
| cd /tmp/downloads | |
| 7z x ve.7z | |
| - name: Install Vantage Express | |
| shell: bash | |
| run: | | |
| export VM_IMAGE_DIR="/tmp/downloads/VantageExpress17.20_Sles12" | |
| DEFAULT_VM_NAME="vantage-express" | |
| VM_NAME="${VM_NAME:-$DEFAULT_VM_NAME}" | |
| vboxmanage createvm --name "$VM_NAME" --register --ostype openSUSE_64 | |
| vboxmanage modifyvm "$VM_NAME" --ioapic on --memory 6000 --vram 128 --nic1 nat --cpus 3 | |
| vboxmanage storagectl "$VM_NAME" --name "SATA Controller" --add sata --controller IntelAhci | |
| vboxmanage storageattach "$VM_NAME" --storagectl "SATA Controller" --port 0 --device 0 --type hdd --medium "$(find $VM_IMAGE_DIR -name '*disk1*')" | |
| vboxmanage storageattach "$VM_NAME" --storagectl "SATA Controller" --port 1 --device 0 --type hdd --medium "$(find $VM_IMAGE_DIR -name '*disk2*')" | |
| vboxmanage storageattach "$VM_NAME" --storagectl "SATA Controller" --port 2 --device 0 --type hdd --medium "$(find $VM_IMAGE_DIR -name '*disk3*')" | |
| vboxmanage modifyvm "$VM_NAME" --natpf1 "tdssh,tcp,,4422,,22" | |
| vboxmanage modifyvm "$VM_NAME" --natpf1 "tddb,tcp,,1025,,1025" | |
| vboxmanage startvm "$VM_NAME" --type headless | |
| - name: Install TTU | |
| shell: bash | |
| run: | | |
| curl -L "https://d289lrf5tw1zls.cloudfront.net/tools/ttu/TeradataToolsAndUtilitiesBase__macosx_x86_64.17.20.08.00.tar.gz?Expires=1704067200&Key-Pair-Id=APKAJ3SWQUPWKYVMO2WQ&Signature=$TTU_URL_SIGNATURE" \ | |
| --compressed -o ttu.tar.gz | |
| tar -xzf ttu.tar.gz | |
| installer -pkg ./TeradataToolsAndUtilitiesBase/*.pkg -target CurrentUserHomeDirectory | |
| env: | |
| TTU_URL_SIGNATURE: ${{ secrets.TTU_URL_SIGNATURE }} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements_dev.txt | |
| # install timeout command | |
| export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=yes | |
| export HOMEBREW_NO_INSTALL_CLEANUP=yes | |
| export HOMEBREW_NO_INSTALL_UPGRADE=yes | |
| brew install coreutils jq | |
| - name: Verify Vantage Express is running | |
| shell: bash | |
| run: ./.github/workflows/scripts/verifyVantageIsRunning.sh | |
| - name: Prepare database | |
| shell: bash | |
| run: | | |
| export PATH=$PATH:"/Users/runner/Library/Application Support/teradata/client/17.20/bin" | |
| cd md5_20080530/md5/src | |
| # prepare bteq script | |
| cat << EOF > /tmp/install_md5.bteq | |
| .SET EXITONDELAY ON MAXREQTIME 20 | |
| .logon 127.0.0.1/dbc,dbc | |
| CREATE DATABASE GLOBAL_FUNCTIONS AS PERMANENT = 60e6, SPOOL = 120e6; | |
| GRANT CREATE FUNCTION ON GLOBAL_FUNCTIONS TO dbc; | |
| DATABASE GLOBAL_FUNCTIONS; | |
| .run file = hash_md5.btq | |
| GRANT EXECUTE FUNCTION ON GLOBAL_FUNCTIONS TO PUBLIC WITH GRANT OPTION; | |
| .logoff | |
| EOF | |
| bteq < /tmp/install_md5.bteq | |
| - name: Run pytest tests | |
| run: | | |
| export DBT_TEST_USER_1=test_grants_user1 | |
| export DBT_TEST_USER_2=test_grants_user2 | |
| export DBT_TEST_USER_3=test_grants_user3 | |
| pytest --cov=dbt tests/functional | |
| coverage html -d coverage_html | |
| - name: Run pytest tests with existing database | |
| run: | | |
| export DBT_TEST_USER_1=test_grants_user1 | |
| export DBT_TEST_USER_2=test_grants_user2 | |
| export DBT_TEST_USER_3=test_grants_user3 | |
| cat << EOF > /tmp/pytestdatabase.bteq | |
| .SET EXITONDELAY ON MAXREQTIME 20 | |
| .logon 127.0.0.1/dbc,dbc | |
| CREATE DATABASE DBT_TEST | |
| AS PERMANENT = 60e6, SPOOL = 120e6; | |
| .logoff | |
| EOF | |
| # strip the random suffix from the pytest config | |
| sed -i'.original' -e "s/_{prefix}//" tests/conftest.py | |
| pytest tests/functional | |
| - name: Run performance tests for fastload | |
| run: | | |
| mkdir ~/.dbt | |
| cat << EOF > ~/.dbt/profiles.yml | |
| teradata: | |
| outputs: | |
| dbt_perf_test: | |
| type: teradata | |
| host: localhost | |
| user: dbc | |
| password: dbc | |
| logmech: TD2 | |
| schema: dbt_perf_test | |
| tmode: ANSI | |
| threads: 1 | |
| timeout_seconds: 300 | |
| priority: interactive | |
| retries: 1 | |
| dbt_catalog_test: | |
| type: teradata | |
| host: localhost | |
| user: dbc | |
| password: dbc | |
| logmech: TD2 | |
| schema: dbt_catalog_test | |
| tmode: ANSI | |
| threads: 1 | |
| timeout_seconds: 300 | |
| priority: interactive | |
| retries: 1 | |
| EOF | |
| cd $GITHUB_WORKSPACE/test/performance | |
| ./run.sh | |
| - name: Run catalog tests | |
| run: | | |
| # set DisableQVCI to false - this will give us data type info for views in DBC.ColumnsJQV | |
| sshpass -p root ssh -o StrictHostKeyChecking=no -p 4422 root@localhost "dbscontrol << EOF | |
| M internal 551=false | |
| W | |
| EOF | |
| tpareset -y Enable QVCI | |
| " | |
| $GITHUB_WORKSPACE/.github/workflows/scripts/verifyVantageIsRunning.sh | |
| cd $GITHUB_WORKSPACE/test/catalog/with_qvci | |
| ./run.sh | |
| cd $GITHUB_WORKSPACE/test/catalog/without_qvci | |
| ./run.sh | |
| - name: Run nopi tests | |
| run: | | |
| sshpass -p root ssh -o StrictHostKeyChecking=no -p 4422 root@localhost "dbscontrol << EOF | |
| M general 53=N | |
| W | |
| EOF | |
| tpareset -y Disable PrimaryIndexDefault | |
| " | |
| $GITHUB_WORKSPACE/.github/workflows/scripts/verifyVantageIsRunning.sh | |
| pytest tests/functional/adapter/test_nopi.py | |
| - uses: actions/upload-artifact@v2 | |
| if: ${{ failure() || cancelled() }} | |
| with: | |
| name: virtualbox-logs | |
| path: /Users/runner/VirtualBox* |