Feat[Storagetool]: List queue appid numrecords #2582
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tests | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| get_dependencies: | |
| name: "Dependencies" | |
| uses: ./.github/workflows/dependencies.yaml | |
| build_ubuntu: | |
| name: Build [ubuntu] | |
| runs-on: ubuntu-latest | |
| needs: get_dependencies | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/cache/restore@v4 | |
| with: | |
| path: deps | |
| key: ${{ needs.get_dependencies.outputs.cache_key }} | |
| - name: Set up dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -qy build-essential \ | |
| gdb \ | |
| curl \ | |
| python3.10 \ | |
| python3-pip \ | |
| cmake \ | |
| ninja-build \ | |
| pkg-config \ | |
| bison \ | |
| libfl-dev \ | |
| libbenchmark-dev \ | |
| libgmock-dev \ | |
| libz-dev | |
| - name: Install cached non packaged dependencies | |
| working-directory: deps | |
| run: ../docker/build_deps.sh | |
| - name: Build BlazingMQ | |
| env: | |
| PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig | |
| run: | | |
| cmake -S . -B build/blazingmq -G Ninja \ | |
| -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \ | |
| -DCMAKE_BUILD_TYPE=Debug \ | |
| -DBDE_BUILD_TARGET_SAFE=ON \ | |
| -DBDE_BUILD_TARGET_64=ON \ | |
| -DBDE_BUILD_TARGET_CPP17=ON \ | |
| -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \ | |
| -DCMAKE_INSTALL_LIBDIR=lib64 | |
| cmake --build build/blazingmq --parallel 8 --target all | |
| - name: Clean-up build directories before caching | |
| run: | | |
| find . -name "*.o" -type f -delete | |
| find . -name "*.a" -type f -delete | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| build/blazingmq | |
| deps | |
| /opt/bb/include | |
| key: cache-${{ github.sha }} | |
| unit_tests_cxx: | |
| name: UT [c++] | |
| runs-on: ubuntu-latest | |
| needs: get_dependencies | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/cache/restore@v4 | |
| with: | |
| path: deps | |
| key: ${{ needs.get_dependencies.outputs.cache_key }} | |
| - name: Set up dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -qy build-essential \ | |
| gdb \ | |
| curl \ | |
| python3.10 \ | |
| python3-pip \ | |
| cmake \ | |
| ninja-build \ | |
| pkg-config \ | |
| bison \ | |
| libfl-dev \ | |
| libbenchmark-dev \ | |
| libgmock-dev \ | |
| libz-dev | |
| - name: Install cached non packaged dependencies | |
| working-directory: deps | |
| run: ../docker/build_deps.sh | |
| - name: Build BlazingMQ UTs | |
| env: | |
| PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig | |
| run: | | |
| cmake -S . -B build/blazingmq -G Ninja \ | |
| -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \ | |
| -DCMAKE_BUILD_TYPE=Debug \ | |
| -DBDE_BUILD_TARGET_SAFE=ON \ | |
| -DBDE_BUILD_TARGET_64=ON \ | |
| -DBDE_BUILD_TARGET_CPP17=ON \ | |
| -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \ | |
| -DCMAKE_INSTALL_LIBDIR=lib64 | |
| cmake --build build/blazingmq --parallel 8 --target all.t | |
| - name: Run C++ Unit Tests | |
| run: | | |
| cd ${{ github.workspace }}/build/blazingmq | |
| ctest --output-on-failure | |
| unit_tests_python: | |
| name: UT [python] | |
| runs-on: ubuntu-latest | |
| needs: build_ubuntu | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/cache/@v4 | |
| with: | |
| path: | | |
| build/blazingmq | |
| deps | |
| /opt/bb/include | |
| key: cache-${{ github.sha }} | |
| - name: Run Python Unit Tests | |
| env: | |
| PYTHONPATH: ${{ github.workspace }}/src/python | |
| run: | | |
| pip install -r ${{ github.workspace }}/src/python/requirements.txt | |
| pip install "xsdata[cli]" | |
| cd ${{ github.workspace }} | |
| src/python/bin/schemagen | |
| src/python/bin/tweakgen | |
| pytest src/python | |
| integration_tests_ubuntu: | |
| name: IT [${{ matrix.cluster }}/${{ matrix.mode }}] | |
| strategy: | |
| matrix: | |
| mode: ["legacy_mode", "fsm_mode"] | |
| cluster: ["single", "multi"] | |
| fail-fast: false | |
| runs-on: ubuntu-latest | |
| needs: build_ubuntu | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/cache/@v4 | |
| with: | |
| path: | | |
| build/blazingmq | |
| deps | |
| /opt/bb/include | |
| key: cache-${{ github.sha }} | |
| - name: Run Integration Tests | |
| run: | | |
| pip install -r ${{ github.workspace }}/src/python/requirements.txt | |
| ${{ github.workspace }}/src/integration-tests/run-tests "${{ matrix.mode }} and ${{ matrix.cluster }}" \ | |
| --log-level ERROR \ | |
| --log-file-level=info \ | |
| --bmq-tolerate-dirty-shutdown \ | |
| --bmq-log-dir=failure-logs \ | |
| --bmq-log-level=INFO \ | |
| --junitxml=integration-tests.xml \ | |
| --tb long \ | |
| --reruns=3 \ | |
| -n 4 -v | |
| - name: Upload failure-logs as artifacts | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: failure_logs_${{ matrix.mode }}_${{ matrix.cluster }} | |
| path: ${{ github.workspace }}/src/integration-tests/failure-logs | |
| retention-days: 5 | |
| fuzz_tests_ubuntu: | |
| name: Fuzz test [${{ matrix.request }}] | |
| strategy: | |
| matrix: | |
| request: ["identity", "open_queue", "configure_queue_stream", "put", "confirm", "close_queue", "disconnect"] | |
| fail-fast: false | |
| runs-on: ubuntu-latest | |
| needs: build_ubuntu | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/cache/@v4 | |
| with: | |
| path: | | |
| build/blazingmq | |
| deps | |
| /opt/bb/include | |
| key: cache-${{ github.sha }} | |
| - name: Run Fuzz Test | |
| run: | | |
| pip install -r ${{ github.workspace }}/src/python/requirements.txt | |
| cd src/python | |
| python3 -m blazingmq.dev.fuzztest --broker-dir ${{ github.workspace }}/build/blazingmq/src/applications/bmqbrkr --request ${{ matrix.request }} | |
| build_macosx: | |
| name: Build [macosx_${{ matrix.arch }}] | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| include: | |
| - os: macos-14 | |
| arch: arm64 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up dependencies | |
| run: | | |
| brew install \ | |
| curl \ | |
| [email protected] \ | |
| ninja \ | |
| bison \ | |
| flex \ | |
| google-benchmark | |
| - name: Build BlazingMQ | |
| run: bin/build-darwin.sh | |
| build_and_test_bmqprometheus_plugin: | |
| name: "Build Prometheus plugin [ubuntu]" | |
| runs-on: ubuntu-latest | |
| needs: build_ubuntu | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/cache/@v4 | |
| with: | |
| path: | | |
| build/blazingmq | |
| deps | |
| /opt/bb/include | |
| key: cache-${{ github.sha }} | |
| - name: Set up plugins dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -qy build-essential \ | |
| gdb \ | |
| curl \ | |
| python3.10 \ | |
| cmake \ | |
| ninja-build \ | |
| pkg-config \ | |
| bison \ | |
| libfl-dev \ | |
| libbenchmark-dev \ | |
| libgmock-dev \ | |
| libz-dev \ | |
| autoconf \ | |
| libtool | |
| - name: Create dependency fetcher working directory | |
| run: mkdir -p deps | |
| - name: Fetch & Build non packaged plugins dependencies | |
| working-directory: deps | |
| run: ${{ github.workspace }}/src/plugins/bmqprometheus/build_prometheus_deps.sh | |
| - name: Build plugins | |
| env: | |
| PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig | |
| run: | | |
| cmake -S . -B build/blazingmq -G Ninja \ | |
| -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \ | |
| -DCMAKE_BUILD_TYPE=Debug \ | |
| -DBDE_BUILD_TARGET_SAFE=ON \ | |
| -DBDE_BUILD_TARGET_64=ON \ | |
| -DBDE_BUILD_TARGET_CPP17=ON \ | |
| -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \ | |
| -DCMAKE_INSTALL_LIBDIR=lib64 \ | |
| -DINSTALL_TARGETS=prometheus | |
| cmake --build build/blazingmq --parallel 8 --target all | |
| - name: Create prometheus dir | |
| run: mkdir -p prometheus_dir | |
| - name: Download Prometheus | |
| run: curl -SL "https://github.com/prometheus/prometheus/releases/download/v2.45.1/prometheus-2.45.1.linux-amd64.tar.gz" | tar -xzC prometheus_dir/ | |
| - name: Run Prometheus | |
| run: ./prometheus_dir/prometheus-2.45.1.linux-amd64/prometheus --config.file=${{ github.workspace }}/src/plugins/bmqprometheus/tests/prometheus_localhost.yaml --web.enable-lifecycle --storage.tsdb.path=${{ github.workspace }}/prometheus_dir/data & | |
| - name: Run Pushgateway | |
| run: | | |
| docker pull prom/pushgateway | |
| docker run -d -p 9091:9091 prom/pushgateway | |
| - name: Run BMQPrometheus plugin integration test in "pull" mode | |
| run: ${{ github.workspace }}/src/plugins/bmqprometheus/tests/bmqprometheus_prometheusstatconsumer_test.py -p ${{ github.workspace }}/build/blazingmq -m pull --no-docker | |
| - name: Clear Prometheus database and restart | |
| run: | | |
| curl -X POST "http://localhost:9090/-/quit" | |
| rm -rf prometheus_dir/data | |
| ./prometheus_dir/prometheus-2.45.1.linux-amd64/prometheus --config.file=${{ github.workspace }}/src/plugins/bmqprometheus/tests/prometheus_localhost.yaml --web.enable-lifecycle --storage.tsdb.path=${{ github.workspace }}/prometheus_dir/data & | |
| - name: Run Prometheus plugin integration test in "push" mode | |
| run: ${{ github.workspace }}/src/plugins/bmqprometheus/tests/bmqprometheus_prometheusstatconsumer_test.py -p ${{ github.workspace }}/build/blazingmq -m push --no-docker | |
| documentation: | |
| name: "Documentation" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -qy doxygen | |
| - name: Set up output directory | |
| run: | | |
| mkdir -p docs/docs/apidocs | |
| - name: Build docs | |
| run: | | |
| doxygen Doxyfile | |
| shellcheck: | |
| name: Shellcheck | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Run ShellCheck | |
| uses: ludeeus/action-shellcheck@master | |
| docker_build_ubuntu: | |
| name: "Docker [ubuntu]" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Docker Single-Node Workflow | |
| run: docker compose -f docker/single-node/docker-compose.yaml up --build -d | |
| - name: Docker Cluster Workflow | |
| run: docker compose -f docker/cluster/docker-compose.yaml up --build -d |