Fix restart #1554
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: slow_tests | |
| on: | |
| # Schedule the workflow to run at 00:30 UTC+8 every day | |
| # https://docs.github.com/zh/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#schedule | |
| schedule: | |
| - cron: '30 10,18,2 * * *' # utc-8: 18:30, 2:30, 10:30 | |
| # The "create tags" trigger is specifically focused on the creation of new tags, while the "push tags" trigger is activated when tags are pushed, including both new tag creations and updates to existing tags. | |
| create: | |
| tags: | |
| - "v*.*.*" # normal release | |
| - "nightly" # mutable tag | |
| - "slow-test" # mutable tag | |
| pull_request: | |
| types: [ opened, synchronize, reopened, labeled ] | |
| paths-ignore: | |
| - 'docs/**' | |
| - '*.md' | |
| # https://docs.github.com/en/actions/using-jobs/using-concurrency | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| slow_tests: | |
| name: run slow test | |
| if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'slow-test') }} | |
| runs-on: ["self-hosted", "slow-test" ] | |
| steps: | |
| - name: Choose test type | |
| id: choose_test_type | |
| run: | | |
| echo "RUN_TSAN_TEST=true" >> $GITHUB_OUTPUT | |
| echo "RUN_RESTART_TEST=true" >> $GITHUB_OUTPUT | |
| echo "RUN_PYTEST_PARALLEL=true" >> $GITHUB_OUTPUT | |
| - name: Show PR labels | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "Workflow triggered by ${{ github.event_name }}" | |
| if [[ ${{ github.event_name }} == 'pull_request' ]]; then | |
| echo "PR labels: ${{ join(github.event.pull_request.labels.*.name, ', ') }}" | |
| fi | |
| - name: Ensure workspace ownership | |
| if: ${{ !cancelled() && !failure() }} | |
| run: echo "chown -R $USER $GITHUB_WORKSPACE" && sudo chown -R $USER $GITHUB_WORKSPACE | |
| - name: Check out code | |
| if: ${{ !cancelled() && !failure() }} | |
| uses: actions/checkout@v4 | |
| - name: Start builder container | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| BUILDER_CONTAINER=infinity_build_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
| CPUS=${CPUS:-$(nproc)} | |
| echo "BUILDER_CONTAINER=${BUILDER_CONTAINER}" >> $GITHUB_ENV | |
| echo "CPUS=${CPUS}" >> $GITHUB_ENV | |
| TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} | |
| sudo docker rm -f -v ${BUILDER_CONTAINER} && sudo docker run --cap-add=NET_ADMIN -d --name ${BUILDER_CONTAINER} -e TZ=$TZ -e CMAKE_BUILD_PARALLEL_LEVEL=${CPUS} -v $PWD:/infinity -v /boot:/boot --cpus ${CPUS} infiniflow/infinity_builder:centos7_clang18 | |
| - name: Create Test image & Start tester container | |
| if: ${{ !cancelled() && !failure() && steps.choose_test_type.outputs.RUN_TSAN_TEST == 'true' }} | |
| run: | | |
| TESTER_IMAGE_NAME=infiniflow/infinity_tester:centos7_clang18 | |
| sudo python3 scripts/build_tester_image.py --image_name=${TESTER_IMAGE_NAME} --bin_path=${RUNNER_WORKSPACE_PREFIX}/build_tester_image | |
| TESTER_CONTAINER=infinity_tester_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
| CPUS=${CPUS:-$(nproc)} | |
| echo "TESTER_CONTAINER=${TESTER_CONTAINER}" >> $GITHUB_ENV | |
| echo "CPUS=${CPUS}" >> $GITHUB_ENV | |
| TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} | |
| INF_DIRECTORY=$PWD | |
| echo "${INF_DIRECTORY}" | sudo tee .tester_env | |
| sudo docker rm -f -v ${TESTER_CONTAINER} && sudo docker run --net=host --privileged --cap-add=NET_ADMIN -d --name ${TESTER_CONTAINER} -e TZ=$TZ -e CMAKE_BUILD_PARALLEL_LEVEL=${CPUS} -v $PWD:/infinity -v /boot:/boot -v /var/run/docker.sock:/var/run/docker.sock --cpus ${CPUS} ${TESTER_IMAGE_NAME} | |
| sudo docker exec ${TESTER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && export INF_DIRECTORY=`cat .tester_env` && echo INF_DIRECTORY=${INF_DIRECTORY}" | |
| - name: Build release version | |
| if: ${{ !cancelled() && !failure() }} | |
| run: sudo docker exec ${BUILDER_CONTAINER} bash -c "git config --global safe.directory \"*\" && cd /infinity && rm -fr cmake-build-release && mkdir -p cmake-build-release && cmake -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_JOB_POOLS:STRING=link=8 -S /infinity -B /infinity/cmake-build-release && cmake --build /infinity/cmake-build-release --target infinity" | |
| - name: Install pysdk for Python 3.10 | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| sudo docker exec ${BUILDER_CONTAINER} bash -c "rm -rf /root/.config/pip/pip.conf && cd /infinity/ && pip3 uninstall -y infinity-sdk && cd python/infinity_sdk/ && pip3 install . -v --config-settings=cmake.build-type='RelWithDebInfo' --config-settings=build-dir='cmake-build-release' -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn && cd ../.." | |
| sudo docker exec ${BUILDER_CONTAINER} bash -c "rm -rf /root/.config/pip/pip.conf && cd /infinity/ && pip3 uninstall -y infinity-embedded-sdk && pip3 install . -v --config-settings=cmake.build-type='RelWithDebInfo' --config-settings=build-dir='cmake-build-release' -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn" | |
| - name: Build debug version with thread sanitizer | |
| if: ${{ !cancelled() && !failure() && steps.choose_test_type.outputs.RUN_TSAN_TEST == 'true' }} | |
| run: sudo docker exec ${BUILDER_CONTAINER} bash -c "git config --global safe.directory \"*\" && cd /infinity && rm -fr cmake-build-debug && mkdir -p cmake-build-debug && cmake -G Ninja -DENABLE_SANITIZER_THREAD=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_JOB_POOLS:STRING=link=8 -S /infinity -B /infinity/cmake-build-debug && cmake --build /infinity/cmake-build-debug --target infinity" | |
| - name: Prepare restart test data | |
| if: ${{ !cancelled() && !failure() && steps.choose_test_type.outputs.RUN_RESTART_TEST == 'true' }} | |
| run: | | |
| RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-$HOME} | |
| echo "RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX}" >> $GITHUB_ENV | |
| sudo python3 scripts/prepare_restart_test_data.py --from_dir=${RUNNER_WORKSPACE_PREFIX} --op=add | |
| - name: Run restart test with thread sanitizer | |
| if: ${{ !cancelled() && !failure() && steps.choose_test_type.outputs.RUN_TSAN_TEST == 'true' }} | |
| id: run_restart_test_sanitizer | |
| run : | | |
| sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pip3 install -r python/restart_test/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn" | |
| # Fix sanitizer: https://github.com/ClickHouse/ClickHouse/issues/64086 | |
| old_value=$(sudo sysctl -n vm.mmap_rnd_bits) | |
| sudo sysctl -w vm.mmap_rnd_bits=28 | |
| sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && LD_PRELOAD=/usr/local/lib/clang/18/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.so TSAN_OPTIONS=log_path=restart_tsan.log ASAN_OPTIONS=detect_leaks=0 python3 tools/run_restart_test.py --infinity_path=cmake-build-debug/src/infinity" | |
| sudo sysctl -w vm.mmap_rnd_bits=$old_value | |
| - name: Collect restart test output with thread sanitizer | |
| if: ${{ !cancelled() && steps.choose_test_type.outputs.RUN_TSAN_TEST == 'true' }} # always run this step even if previous steps failed | |
| # remove symbolic link | |
| # find all log file like [debug.log.*] in directory, and cat to stdout | |
| run: | | |
| failure="${{ steps.run_restart_test_sanitizer.outcome == 'failure'}}" | |
| sudo python3 scripts/collect_restart_log.py --executable_path=cmake-build-debug/src/infinity --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure} --tsan_log=restart_tsan.log | |
| sudo rm restart_tsan.log* | |
| - name: Run cluster test | |
| if: ${{ !cancelled() && !failure() && steps.choose_test_type.outputs.RUN_TSAN_TEST == 'true' }} | |
| id: run_cluster_test | |
| run: | | |
| sudo docker exec ${TESTER_CONTAINER} bash -c "rm -rf /root/.config/pip/pip.conf && cd /infinity/ && pip3 uninstall -y infinity-sdk infinity-embedded-sdk && cd python/infinity_sdk/ && pip3 install . -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn && cd ../.." | |
| # Fix sanitizer: https://github.com/ClickHouse/ClickHouse/issues/64086 | |
| old_value=$(sudo sysctl -n vm.mmap_rnd_bits) | |
| sudo sysctl -w vm.mmap_rnd_bits=28 | |
| sudo docker exec ${TESTER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && export INF_DIRECTORY=`cat .tester_env` && echo INF_DIRECTORY=${INF_DIRECTORY} && python3 tools/run_cluster_test.py --infinity_path=cmake-build-debug/src/infinity --infinity_dir=${INF_DIRECTORY} --minio_port=9005 --minio_console_port=9006" | |
| sudo sysctl -w vm.mmap_rnd_bits=$old_value | |
| - name: Collect thread sanitizer output in cluster test | |
| if: ${{ !cancelled() && steps.choose_test_type.outputs.RUN_TSAN_TEST == 'true' }} | |
| run: | | |
| failure="${{ steps.run_cluster_test.outcome == 'failure'}}" | |
| sudo python3 scripts/collect_cluster_log.py --executable_path=cmake-build-debug/src/infinity --log_dir=/var/infinity/ --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure} --tsan_log=cluster_tsan.log | |
| sudo rm -f cluster_tsan.log* | |
| - name: Start infinity debug version with thread sanitizer | |
| if: ${{ !cancelled() && !failure() && steps.choose_test_type.outputs.RUN_TSAN_TEST == 'true' }} | |
| id: start_tests_debug | |
| run: | | |
| # Fix sanitizer: https://github.com/ClickHouse/ClickHouse/issues/64086 | |
| old_value=$(sudo sysctl -n vm.mmap_rnd_bits) | |
| echo "old_mmmap_rnd_bits=${old_value}" >> "$GITHUB_OUTPUT" | |
| sudo sysctl -w vm.mmap_rnd_bits=28 | |
| sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && TSAN_OPTIONS=log_path=tsan.log cmake-build-debug/src/infinity --config=conf/pytest_parallel_infinity_conf.toml > debug.log 2> debug_error.log &" && sleep 1s | |
| if ! sudo docker exec ${BUILDER_CONTAINER} pgrep -f cmake-build-debug/src/infinity > /dev/null; then | |
| echo "Failed to start infinity debug version" | |
| cat debug_error.log | |
| exit 1 | |
| fi | |
| - name: Run pytest parallel with thread sanitizer | |
| if: ${{ !cancelled() && !failure() && steps.choose_test_type.outputs.RUN_TSAN_TEST == 'true' }} | |
| id: run_tests_debug | |
| run: | | |
| sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && LD_PRELOAD=/usr/local/lib/clang/18/lib/x86_64-unknown-linux-gnu/libclang_rt.asan.so ASAN_OPTIONS=detect_leaks=0 python3 tools/run_pytest_parallel.py" && sleep 1s | |
| - name: Stop infinity pysdk & http_api & sqllogictest debug | |
| if: ${{ !cancelled() && steps.choose_test_type.outputs.RUN_TSAN_TEST == 'true' }} | |
| id: stop_tests_debug | |
| env: | |
| old_value: ${{ steps.start_tests_debug.outputs.old_mmmap_rnd_bits }} | |
| run: | | |
| pids=$(sudo docker exec ${BUILDER_CONTAINER} pgrep -f cmake-build-debug/src/infinity | xargs echo) | |
| sudo chmod +x scripts/timeout_kill.sh | |
| sudo docker exec ${BUILDER_CONTAINER} bash -c "/infinity/scripts/timeout_kill.sh 10 ${pids}" | |
| if [ $? -ne 0 ]; then | |
| echo "Failed to kill infinity debug version" | |
| exit 1 | |
| fi | |
| sudo sysctl -w vm.mmap_rnd_bits=$old_value | |
| - name: Collect thread sanitizer output in pytest parallel | |
| if: ${{ !cancelled() && steps.choose_test_type.outputs.RUN_TSAN_TEST == 'true' }} | |
| # GitHub Actions interprets output lines starting with "Error" as error messages, and it automatically sets the step status to failed when such lines are detected. | |
| run: | | |
| failure="${{ steps.run_tests_debug.outcome == 'failure' || steps.stop_tests_debug.outcome == 'failure' }}" | |
| sudo python3 scripts/collect_log.py --log_path=/var/infinity/log/infinity.log --stdout_path=debug.log --stderror_path=debug_error.log --executable_path=cmake-build-debug/src/infinity --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure} --tsan_log=tsan.log | |
| sudo rm -f tsan.log* | |
| - name: Run restart test | |
| if: ${{ !cancelled() && !failure() && steps.choose_test_type.outputs.RUN_RESTART_TEST == 'true' }} | |
| id: run_restart_test | |
| run : | | |
| sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pip3 install -r python/restart_test/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn" | |
| sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && python3 tools/run_restart_test_continuously.py --infinity_path=cmake-build-release/src/infinity" | |
| # sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pytest python/restart_test/test_insert.py -k "test_data[infinity_runner0-columns5-gen-1000000-test/data/config/restart_test/test_insert/1.toml]" -s --infinity_path=cmake-build-release/src/infinity" | |
| # sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && pytest python/restart_test/test_insert.py -k "test_index[infinity_runner0-columns2-indexes2-gen-1000000-test/data/config/restart_test/test_insert/1.toml]" -s --infinity_path=cmake-build-release/src/infinity" | |
| - name: Collect restart test output | |
| if: ${{ !cancelled() && steps.choose_test_type.outputs.RUN_RESTART_TEST == 'true' }} # always run this step even if previous steps failed | |
| # remove symbolic link | |
| # find all log file like [debug.log.*] in directory, and cat to stdout | |
| run: | | |
| sudo python3 scripts/prepare_restart_test_data.py --op=remove | |
| failure="${{ steps.run_restart_test.outcome == 'failure'}}" | |
| sudo python3 scripts/collect_restart_log.py --executable_path=cmake-build-release/src/infinity --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure} | |
| - name: Start infinity release version | |
| if: ${{ !cancelled() && !failure() && steps.choose_test_type.outputs.RUN_PYTEST_PARALLEL == 'true' }} | |
| # && !contains(github.event.pull_request.labels.*.name, 'invalid') | |
| run: | | |
| # Run a command in the background | |
| sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && cmake-build-release/src/infinity --config=conf/pytest_parallel_continuous_conf.toml > release.log 2> release_error.log" & | |
| - name: Run pysdk remote infinity & parallel & http_api & sqllogic test release version continously | |
| if: ${{ !cancelled() && !failure() && steps.choose_test_type.outputs.RUN_PYTEST_PARALLEL == 'true' }} | |
| # && !contains(github.event.pull_request.labels.*.name, 'invalid') | |
| id: run_py_tests | |
| run: sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && python3 tools/run_pytest_parallel_continuous.py" && sleep 1s | |
| - name: Stop infinity release | |
| if: ${{ !cancelled() && !failure() && steps.choose_test_type.outputs.RUN_PYTEST_PARALLEL == 'true' }} | |
| # && !contains(github.event.pull_request.labels.*.name, 'invalid') | |
| id: stop_py_tests | |
| run: | | |
| pids=$(sudo docker exec ${BUILDER_CONTAINER} pgrep -f cmake-build-release/src/infinity | xargs echo) | |
| sudo chmod +x scripts/timeout_kill.sh | |
| sudo docker exec ${BUILDER_CONTAINER} bash -c "/infinity/scripts/timeout_kill.sh 10 ${pids}" | |
| if [ $? -ne 0 ]; then | |
| echo "Failed to kill infinity release version" | |
| exit 1 | |
| fi | |
| - name: Collect infinity release output | |
| if: ${{ !cancelled() && steps.choose_test_type.outputs.RUN_PYTEST_PARALLEL == 'true' }} # always run this step even if previous steps failed | |
| run: | | |
| failure="${{ steps.run_py_tests.outcome == 'failure' || steps.stop_py_tests.outcome == 'failure' }}" | |
| sudo python3 scripts/collect_log.py --log_path=/var/infinity/log/infinity.log --stdout_path=release.log --stderror_path=release_error.log --executable_path=cmake-build-release/src/infinity --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure} | |
| - name: Destroy builder container | |
| if: always() # always run this step even if previous steps failed | |
| run: | | |
| if [ -n "${BUILDER_CONTAINER}" ]; then | |
| sudo docker rm -f -v ${BUILDER_CONTAINER} | |
| fi | |