From ead088f20ef52c77f517a9540b876ec8abc963ca Mon Sep 17 00:00:00 2001 From: Mitya <8863314@gmail.com> Date: Tue, 22 Oct 2024 12:24:56 +0300 Subject: [PATCH 01/69] Compiled cuBool and run tests with CUDA 12.6 --- .gitmodules | 3 --- CMakeLists.txt | 12 ++++++------ cubool/sources/cuda/cuda_vector.cu | 2 ++ cubool/sources/cuda/kernels/spewiseadd.cuh | 2 ++ cubool/sources/cuda/kernels/spewisemult.cuh | 2 ++ cubool/sources/cuda/kernels/spgemv.cuh | 2 ++ cubool/sources/cuda/kernels/spgemv_t.cuh | 2 ++ cubool/sources/cuda/kernels/spreduce.cuh | 2 ++ cubool/sources/cuda/kernels/sptranspose2.cuh | 2 ++ cubool/sources/utils/data_utils.hpp | 1 + deps/cub | 1 - deps/nsparse-um/CMakeLists.txt | 3 ++- deps/nsparse-um/include/nsparse/unified_allocator.h | 2 +- deps/nsparse-um/test/CMakeLists.txt | 3 ++- deps/nsparse/test/CMakeLists.txt | 3 ++- 15 files changed, 28 insertions(+), 14 deletions(-) delete mode 160000 deps/cub diff --git a/.gitmodules b/.gitmodules index 3adb3ec..f98d82d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ [submodule "deps/gtest"] path = deps/gtest url = https://github.com/google/googletest.git -[submodule "deps/cub"] - path = deps/cub - url = https://github.com/NVIDIA/cub.git diff --git a/CMakeLists.txt b/CMakeLists.txt index fb9212a..e3ab3eb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,12 +33,12 @@ endif() # Configure cuda dependencies if (CUBOOL_WITH_CUDA) message(STATUS "Add cub as cuda utility") - set(CUB_ENABLE_HEADER_TESTING OFF CACHE BOOL "" FORCE) - set(CUB_ENABLE_TESTING OFF CACHE BOOL "" FORCE) - set(CUB_ENABLE_EXAMPLES OFF CACHE BOOL "" FORCE) - add_subdirectory(deps/cub) - add_library(cub INTERFACE IMPORTED) - target_link_libraries(cub INTERFACE CUB::CUB) + # set(CUB_ENABLE_HEADER_TESTING OFF CACHE BOOL "" FORCE) + # set(CUB_ENABLE_TESTING OFF CACHE BOOL "" FORCE) + # set(CUB_ENABLE_EXAMPLES OFF CACHE BOOL "" FORCE) + # add_subdirectory(deps/cub) + # add_library(cub INTERFACE IMPORTED) + # target_link_libraries(cub INTERFACE CUB::CUB) message(STATUS "Add nsparse library as crs matrix multiplication backend") add_subdirectory(deps/nsparse-um) diff --git a/cubool/sources/cuda/cuda_vector.cu b/cubool/sources/cuda/cuda_vector.cu index ced56a0..f2fa9b0 100644 --- a/cubool/sources/cuda/cuda_vector.cu +++ b/cubool/sources/cuda/cuda_vector.cu @@ -29,6 +29,8 @@ #include #include +#include + namespace cubool { CudaVector::CudaVector(size_t nrows, CudaInstance &instance) diff --git a/cubool/sources/cuda/kernels/spewiseadd.cuh b/cubool/sources/cuda/kernels/spewiseadd.cuh index f6ad892..d1e1e87 100644 --- a/cubool/sources/cuda/kernels/spewiseadd.cuh +++ b/cubool/sources/cuda/kernels/spewiseadd.cuh @@ -27,6 +27,8 @@ #include +#include + namespace cubool { namespace kernels { diff --git a/cubool/sources/cuda/kernels/spewisemult.cuh b/cubool/sources/cuda/kernels/spewisemult.cuh index a4705de..6d17a46 100644 --- a/cubool/sources/cuda/kernels/spewisemult.cuh +++ b/cubool/sources/cuda/kernels/spewisemult.cuh @@ -30,6 +30,8 @@ #include #include +#include + namespace cubool { namespace kernels { diff --git a/cubool/sources/cuda/kernels/spgemv.cuh b/cubool/sources/cuda/kernels/spgemv.cuh index 7340174..818ff62 100644 --- a/cubool/sources/cuda/kernels/spgemv.cuh +++ b/cubool/sources/cuda/kernels/spgemv.cuh @@ -32,6 +32,8 @@ #include #include +#include + namespace cubool { namespace kernels { diff --git a/cubool/sources/cuda/kernels/spgemv_t.cuh b/cubool/sources/cuda/kernels/spgemv_t.cuh index 6225d71..2a5db83 100644 --- a/cubool/sources/cuda/kernels/spgemv_t.cuh +++ b/cubool/sources/cuda/kernels/spgemv_t.cuh @@ -32,6 +32,8 @@ #include #include +#include + namespace cubool { namespace kernels { diff --git a/cubool/sources/cuda/kernels/spreduce.cuh b/cubool/sources/cuda/kernels/spreduce.cuh index 3a15bb1..cfcf398 100644 --- a/cubool/sources/cuda/kernels/spreduce.cuh +++ b/cubool/sources/cuda/kernels/spreduce.cuh @@ -30,6 +30,8 @@ #include #include +#include + namespace cubool { namespace kernels { diff --git a/cubool/sources/cuda/kernels/sptranspose2.cuh b/cubool/sources/cuda/kernels/sptranspose2.cuh index 8c50e45..c686818 100644 --- a/cubool/sources/cuda/kernels/sptranspose2.cuh +++ b/cubool/sources/cuda/kernels/sptranspose2.cuh @@ -29,6 +29,8 @@ #include #include +#include + namespace cubool { namespace kernels { diff --git a/cubool/sources/utils/data_utils.hpp b/cubool/sources/utils/data_utils.hpp index 2356382..b7863b3 100644 --- a/cubool/sources/utils/data_utils.hpp +++ b/cubool/sources/utils/data_utils.hpp @@ -27,6 +27,7 @@ #include #include +#include namespace cubool { diff --git a/deps/cub b/deps/cub deleted file mode 160000 index b229817..0000000 --- a/deps/cub +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b229817e3963fc942c7cc2c61715a6b2b2c49bed diff --git a/deps/nsparse-um/CMakeLists.txt b/deps/nsparse-um/CMakeLists.txt index ee5ac0d..cdc4463 100644 --- a/deps/nsparse-um/CMakeLists.txt +++ b/deps/nsparse-um/CMakeLists.txt @@ -3,7 +3,8 @@ project(nsparse_um LANGUAGES CXX CUDA) add_library(nsparse_um INTERFACE) target_include_directories(nsparse_um INTERFACE include/) -target_link_libraries(nsparse_um INTERFACE cub) +# target_link_libraries(nsparse_um INTERFACE cub) +target_link_libraries(nsparse_um INTERFACE) target_compile_options(nsparse_um INTERFACE $<$: --expt-relaxed-constexpr --expt-extended-lambda>) if (CUBOOL_BUILD_NSPARSE_TESTS) diff --git a/deps/nsparse-um/include/nsparse/unified_allocator.h b/deps/nsparse-um/include/nsparse/unified_allocator.h index ea6a98b..c15f19e 100644 --- a/deps/nsparse-um/include/nsparse/unified_allocator.h +++ b/deps/nsparse-um/include/nsparse/unified_allocator.h @@ -3,7 +3,7 @@ #include #include #include -#include +// #include #include #include diff --git a/deps/nsparse-um/test/CMakeLists.txt b/deps/nsparse-um/test/CMakeLists.txt index 3bece29..6a29690 100644 --- a/deps/nsparse-um/test/CMakeLists.txt +++ b/deps/nsparse-um/test/CMakeLists.txt @@ -3,7 +3,8 @@ project(nsparse_um_test CXX CUDA) add_executable(${PROJECT_NAME} src/nsparse_test.cu src/utils.cpp) -target_link_libraries(${PROJECT_NAME} PRIVATE gtest_main nsparse_um cub) +# target_link_libraries(${PROJECT_NAME} PRIVATE gtest_main nsparse_um cub) +target_link_libraries(${PROJECT_NAME} PRIVATE gtest_main nsparse_um) target_compile_features(${PROJECT_NAME} PRIVATE cxx_std_14) set_target_properties(${PROJECT_NAME} PROPERTIES CXX_STANDARD 17) diff --git a/deps/nsparse/test/CMakeLists.txt b/deps/nsparse/test/CMakeLists.txt index 9e71d90..9927725 100644 --- a/deps/nsparse/test/CMakeLists.txt +++ b/deps/nsparse/test/CMakeLists.txt @@ -2,7 +2,8 @@ cmake_minimum_required(VERSION 3.11) project(nsparse_test CXX CUDA) add_executable(${PROJECT_NAME} src/nsparse_test.cu src/utils.cpp) -target_link_libraries(${PROJECT_NAME} PRIVATE gtest_main nsparse cub) +# target_link_libraries(${PROJECT_NAME} PRIVATE gtest_main nsparse cub) +target_link_libraries(${PROJECT_NAME} PRIVATE gtest_main nsparse) target_compile_features(${PROJECT_NAME} PRIVATE cxx_std_14) set_target_properties(${PROJECT_NAME} PROPERTIES CXX_STANDARD 17) From 443d78deae3a4ef525e722afba2a5db3f1c08948 Mon Sep 17 00:00:00 2001 From: Mitya <8863314@gmail.com> Date: Tue, 22 Oct 2024 12:47:31 +0300 Subject: [PATCH 02/69] Temporary removed python package support for compiling rpq algorithm --- CMakeLists.txt | 4 ++-- cubool/CMakeLists.txt | 22 +++++++++++----------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e3ab3eb..da9e6d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -58,7 +58,7 @@ endif() add_subdirectory(cubool) # Copy scripts into binary directory -file(COPY scripts DESTINATION ${CMAKE_BINARY_DIR}/) +# file(COPY scripts DESTINATION ${CMAKE_BINARY_DIR}/) # Copy python related stuff -file(COPY python DESTINATION ${CMAKE_BINARY_DIR}/) \ No newline at end of file +# file(COPY python DESTINATION ${CMAKE_BINARY_DIR}/) diff --git a/cubool/CMakeLists.txt b/cubool/CMakeLists.txt index 092504b..88591de 100644 --- a/cubool/CMakeLists.txt +++ b/cubool/CMakeLists.txt @@ -246,14 +246,14 @@ if (CUBOOL_BUILD_TESTS) endif() # Copy cubool library after build if allowed -if (CUBOOL_COPY_TO_PY_PACKAGE) - set(LIBRARY_FILE_NAME ${TARGET_FILE_NAME}) - - add_custom_command( - TARGET cubool POST_BUILD - COMMAND "${CMAKE_COMMAND}" -E - copy - "${CMAKE_BINARY_DIR}/cubool/${LIBRARY_FILE_NAME}" - "${CMAKE_BINARY_DIR}/python/pycubool" - COMMENT "Copy ${LIBRARY_FILE_NAME} compiled lib into python folder") -endif() \ No newline at end of file +# if (CUBOOL_COPY_TO_PY_PACKAGE) +# set(LIBRARY_FILE_NAME ${TARGET_FILE_NAME}) +# +# add_custom_command( +# TARGET cubool POST_BUILD +# COMMAND "${CMAKE_COMMAND}" -E +# copy +# "${CMAKE_BINARY_DIR}/cubool/${LIBRARY_FILE_NAME}" +# "${CMAKE_BINARY_DIR}/python/pycubool" +# COMMENT "Copy ${LIBRARY_FILE_NAME} compiled lib into python folder") +# endif() From f429420f50c6297171f8cd7c485e1a6379fef2d5 Mon Sep 17 00:00:00 2001 From: Mitya <8863314@gmail.com> Date: Tue, 5 Nov 2024 20:57:12 +0300 Subject: [PATCH 03/69] Fixed cubool tests --- cubool/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cubool/CMakeLists.txt b/cubool/CMakeLists.txt index 88591de..406510c 100644 --- a/cubool/CMakeLists.txt +++ b/cubool/CMakeLists.txt @@ -219,7 +219,7 @@ endforeach() if (CUBOOL_WITH_CUDA) set_target_properties(cubool PROPERTIES CUDA_STANDARD 14) set_target_properties(cubool PROPERTIES CUDA_STANDARD_REQUIRED ON) - set_target_properties(cubool PROPERTIES CUDA_SEPARABLE_COMPILATION ON) + # set_target_properties(cubool PROPERTIES CUDA_SEPARABLE_COMPILATION ON) # Settings: https://arnon.dk/matching-sm-architectures-arch-and-gencode-for-various-nvidia-cards/ From e434bb145970a835f9470fc4870f1295b669e80b Mon Sep 17 00:00:00 2001 From: Mitya <8863314@gmail.com> Date: Tue, 5 Nov 2024 22:41:40 +0300 Subject: [PATCH 04/69] Implemented applying inverted mask --- cubool/CMakeLists.txt | 5 + cubool/include/cubool/cubool.h | 26 ++++ cubool/sources/backend/matrix_base.hpp | 3 +- cubool/sources/core/matrix.cpp | 44 +++++- cubool/sources/core/matrix.hpp | 3 +- .../cuBool_Matrix_EWiseMultInverted.cpp | 44 ++++++ cubool/sources/cuda/cuda_matrix.hpp | 3 +- .../cuda/cuda_matrix_ewisemult_inverted.cu | 63 ++++++++ .../cuda/kernels/spewisemultinverted.cuh | 106 ++++++++++++++ .../sequential/sq_ewisemultinverted.cpp | 105 ++++++++++++++ .../sequential/sq_ewisemultinverted.hpp | 42 ++++++ cubool/sources/sequential/sq_matrix.cpp | 26 ++++ cubool/sources/sequential/sq_matrix.hpp | 1 + cubool/tests/CMakeLists.txt | 3 + .../tests/test_matrix_ewisemult_inverted.cpp | 134 ++++++++++++++++++ 15 files changed, 604 insertions(+), 4 deletions(-) create mode 100644 cubool/sources/cuBool_Matrix_EWiseMultInverted.cpp create mode 100644 cubool/sources/cuda/cuda_matrix_ewisemult_inverted.cu create mode 100644 cubool/sources/cuda/kernels/spewisemultinverted.cuh create mode 100644 cubool/sources/sequential/sq_ewisemultinverted.cpp create mode 100644 cubool/sources/sequential/sq_ewisemultinverted.hpp create mode 100644 cubool/tests/test_matrix_ewisemult_inverted.cpp diff --git a/cubool/CMakeLists.txt b/cubool/CMakeLists.txt index 406510c..cd80032 100644 --- a/cubool/CMakeLists.txt +++ b/cubool/CMakeLists.txt @@ -87,6 +87,7 @@ set(CUBOOL_C_API_SOURCES sources/cuBool_Matrix_Reduce2.cpp sources/cuBool_Matrix_EWiseAdd.cpp sources/cuBool_Matrix_EWiseMult.cpp + sources/cuBool_Matrix_EWiseMultInverted.cpp sources/cuBool_Vector_New.cpp sources/cuBool_Vector_Build.cpp sources/cuBool_Vector_SetElement.cpp @@ -125,6 +126,7 @@ if (CUBOOL_WITH_CUDA) sources/cuda/cuda_matrix.cu sources/cuda/cuda_matrix_ewiseadd.cu sources/cuda/cuda_matrix_ewisemult.cu + sources/cuda/cuda_matrix_ewisemult_inverted.cu sources/cuda/cuda_matrix_kronecker.cu sources/cuda/cuda_matrix_multiply.cu sources/cuda/cuda_matrix_transpose.cu @@ -147,6 +149,7 @@ if (CUBOOL_WITH_CUDA) sources/cuda/kernels/spgemv_t.cuh sources/cuda/kernels/spewiseadd.cuh sources/cuda/kernels/spewisemult.cuh + sources/cuda/kernels/spewisemultinverted.cuh sources/cuda/kernels/sptranspose.cuh sources/cuda/kernels/sptranspose2.cuh sources/cuda/kernels/spkron.cuh @@ -173,6 +176,8 @@ if (CUBOOL_WITH_SEQUENTIAL) sources/sequential/sq_ewiseadd.hpp sources/sequential/sq_ewisemult.cpp sources/sequential/sq_ewisemult.hpp + sources/sequential/sq_ewisemultinverted.cpp + sources/sequential/sq_ewisemultinverted.hpp sources/sequential/sq_spgemm.cpp sources/sequential/sq_spgemm.hpp sources/sequential/sq_spgemv.cpp diff --git a/cubool/include/cubool/cubool.h b/cubool/include/cubool/cubool.h index cd16bb3..ef7d9b5 100644 --- a/cubool/include/cubool/cubool.h +++ b/cubool/include/cubool/cubool.h @@ -925,4 +925,30 @@ CUBOOL_EXPORT CUBOOL_API cuBool_Status cuBool_Kronecker( cuBool_Hints hints ); +/** + * Performs result = left + ~right, where + * '+' is boolean semiring 'or' operation + * '~' is operation for invert matrix (0 swaps to 1 and 1 to 0) + * + * @note Matrices must be compatible + * dim(result) = M x N + * dim(left) = M x N + * dim(right) = M x N + * + * @note Pass `CUBOOL_HINT_TIME_CHECK` hint to measure operation time + * + * @param result[out] Destination matrix to store result + * @param left Source matrix to be added + * @param right Source matrix to be inverted and added + * @param hints Hints for the operation + * + * @return Error code on this operation + */ +CUBOOL_EXPORT CUBOOL_API cuBool_Status cuBool_Matrix_EWiseMulInverted( + cuBool_Matrix result, + cuBool_Matrix matrix, + cuBool_Matrix mask, + cuBool_Hints hints +); + #endif //CUBOOL_CUBOOL_H diff --git a/cubool/sources/backend/matrix_base.hpp b/cubool/sources/backend/matrix_base.hpp index 5332031..8842eb9 100644 --- a/cubool/sources/backend/matrix_base.hpp +++ b/cubool/sources/backend/matrix_base.hpp @@ -49,6 +49,7 @@ namespace cubool { virtual void kronecker(const MatrixBase &aBase, const MatrixBase &bBase, bool checkTime) = 0; virtual void eWiseAdd(const MatrixBase &aBase, const MatrixBase &bBase, bool checkTime) = 0; virtual void eWiseMult(const MatrixBase &aBase, const MatrixBase &bBase, bool checkTime) = 0; + virtual void eWiseMultInverted(const MatrixBase &matrix, const MatrixBase &mask, bool checkTime) = 0; virtual index getNrows() const = 0; virtual index getNcols() const = 0; @@ -59,4 +60,4 @@ namespace cubool { } -#endif //CUBOOL_MATRIX_BASE_HPP \ No newline at end of file +#endif //CUBOOL_MATRIX_BASE_HPP diff --git a/cubool/sources/core/matrix.cpp b/cubool/sources/core/matrix.cpp index 571b010..730fe63 100644 --- a/cubool/sources/core/matrix.cpp +++ b/cubool/sources/core/matrix.cpp @@ -349,6 +349,48 @@ namespace cubool { mHnd->eWiseMult(*a->mHnd, *b->mHnd, false); } + void Matrix::eWiseMultInverted(const MatrixBase &matrix, const MatrixBase &mask, bool checkTime) { + + const auto* mat = dynamic_cast(&matrix); + const auto* msk = dynamic_cast(&mask); + + CHECK_RAISE_ERROR(mat != nullptr, InvalidArgument, "Passed matrix does not belong to core matrix class"); + CHECK_RAISE_ERROR(msk != nullptr, InvalidArgument, "Passed matrix does not belong to core matrix class"); + + index M = mat->getNrows(); + index N = msk->getNcols(); + + CHECK_RAISE_ERROR(M == msk->getNrows(), InvalidArgument, "Passed matrices have incompatible size"); + CHECK_RAISE_ERROR(N == msk->getNcols(), InvalidArgument, "Passed matrices have incompatible size"); + + CHECK_RAISE_ERROR(M == this->getNrows(), InvalidArgument, "Matrix has incompatible size for operation result"); + CHECK_RAISE_ERROR(N == this->getNcols(), InvalidArgument, "Matrix has incompatible size for operation result"); + + mat->commitCache(); + msk->commitCache(); + this->releaseCache(); + + /* + if (checkTime) { + TIMER_ACTION(timer, mHnd->eWiseMult(*a->mHnd, *b->mHnd, false)); + + LogStream stream(*Library::getLogger()); + stream << Logger::Level::Info + << "Time: " << timer.getElapsedTimeMs() << " ms " + << "Matrix::eWiseMult: " + << this->getDebugMarker() << " = " + << a->getDebugMarker() << " + " + << b->getDebugMarker() << LogStream::cmt; + + return; + } + */ + + mHnd->eWiseMultInverted(*mat->mHnd, *msk->mHnd, false); + } + + + index Matrix::getNrows() const { return mHnd->getNrows(); } @@ -396,4 +438,4 @@ namespace cubool { // Clear arrays releaseCache(); } -} \ No newline at end of file +} diff --git a/cubool/sources/core/matrix.hpp b/cubool/sources/core/matrix.hpp index 20cb890..6c4d38b 100644 --- a/cubool/sources/core/matrix.hpp +++ b/cubool/sources/core/matrix.hpp @@ -56,6 +56,7 @@ namespace cubool { void kronecker(const MatrixBase &aBase, const MatrixBase &bBase, bool checkTime) override; void eWiseAdd(const MatrixBase &aBase, const MatrixBase &bBase, bool checkTime) override; void eWiseMult(const MatrixBase &a, const MatrixBase &b, bool checkTime) override; + void eWiseMultInverted(const MatrixBase &matrix, const MatrixBase &mask, bool checkTime) override; index getNrows() const override; index getNcols() const override; @@ -77,4 +78,4 @@ namespace cubool { } -#endif //CUBOOL_MATRIX_HPP \ No newline at end of file +#endif //CUBOOL_MATRIX_HPP diff --git a/cubool/sources/cuBool_Matrix_EWiseMultInverted.cpp b/cubool/sources/cuBool_Matrix_EWiseMultInverted.cpp new file mode 100644 index 0000000..37fe607 --- /dev/null +++ b/cubool/sources/cuBool_Matrix_EWiseMultInverted.cpp @@ -0,0 +1,44 @@ +/**********************************************************************************/ +/* MIT License */ +/* */ +/* Copyright (c) 2020, 2021 JetBrains-Research */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining a copy */ +/* of this software and associated documentation files (the "Software"), to deal */ +/* in the Software without restriction, including without limitation the rights */ +/* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell */ +/* copies of the Software, and to permit persons to whom the Software is */ +/* furnished to do so, subject to the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be included in all */ +/* copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR */ +/* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, */ +/* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE */ +/* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER */ +/* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, */ +/* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE */ +/* SOFTWARE. */ +/**********************************************************************************/ + +#include + +CUBOOL_EXPORT CUBOOL_API cuBool_Status cuBool_Matrix_EWiseMulInverted( + cuBool_Matrix result, + cuBool_Matrix matrix, + cuBool_Matrix mask, + cuBool_Hints hints +) { + CUBOOL_BEGIN_BODY + CUBOOL_VALIDATE_LIBRARY + CUBOOL_ARG_NOT_NULL(result) + CUBOOL_ARG_NOT_NULL(matrix) + CUBOOL_ARG_NOT_NULL(mask) + auto resultM = (cubool::Matrix *) result; + auto matrixM = (cubool::Matrix *) matrix; + auto maskM = (cubool::Matrix *) mask; + + resultM->eWiseMultInverted(*matrixM, *maskM, hints & CUBOOL_HINT_TIME_CHECK); + CUBOOL_END_BODY +} diff --git a/cubool/sources/cuda/cuda_matrix.hpp b/cubool/sources/cuda/cuda_matrix.hpp index e19fec1..9f1b7a2 100644 --- a/cubool/sources/cuda/cuda_matrix.hpp +++ b/cubool/sources/cuda/cuda_matrix.hpp @@ -56,6 +56,7 @@ namespace cubool { void kronecker(const MatrixBase &a, const MatrixBase &b, bool checkTime) override; void eWiseAdd(const MatrixBase &a, const MatrixBase &b, bool checkTime) override; void eWiseMult(const MatrixBase &a, const MatrixBase &b, bool checkTime) override; + void eWiseMultInverted(const MatrixBase &a, const MatrixBase &b, bool checkTime) override; index getNrows() const override; index getNcols() const override; @@ -79,4 +80,4 @@ namespace cubool { }; }; -#endif //CUBOOL_CUDA_MATRIX_HPP \ No newline at end of file +#endif //CUBOOL_CUDA_MATRIX_HPP diff --git a/cubool/sources/cuda/cuda_matrix_ewisemult_inverted.cu b/cubool/sources/cuda/cuda_matrix_ewisemult_inverted.cu new file mode 100644 index 0000000..2e0fa87 --- /dev/null +++ b/cubool/sources/cuda/cuda_matrix_ewisemult_inverted.cu @@ -0,0 +1,63 @@ +/**********************************************************************************/ +/* MIT License */ +/* */ +/* Copyright (c) 2020, 2021 JetBrains-Research */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining a copy */ +/* of this software and associated documentation files (the "Software"), to deal */ +/* in the Software without restriction, including without limitation the rights */ +/* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell */ +/* copies of the Software, and to permit persons to whom the Software is */ +/* furnished to do so, subject to the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be included in all */ +/* copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR */ +/* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, */ +/* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE */ +/* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER */ +/* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, */ +/* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE */ +/* SOFTWARE. */ +/**********************************************************************************/ + +#include +#include + +namespace cubool { + + void CudaMatrix::eWiseMultInverted(const MatrixBase &aBase, const MatrixBase &bBase, bool checkTime) { + auto a = dynamic_cast(&aBase); + auto b = dynamic_cast(&bBase); + + CHECK_RAISE_ERROR(a != nullptr, InvalidArgument, "Passed matrix does not belong to csr matrix class"); + CHECK_RAISE_ERROR(b != nullptr, InvalidArgument, "Passed matrix does not belong to csr matrix class"); + + index M = this->getNrows(); + index N = this->getNcols(); + + + assert(a->getNrows() == M); + assert(a->getNcols() == N); + + assert(b->getNrows() == M); + assert(b->getNcols() == N); + + if (a->isMatrixEmpty() || b->isMatrixEmpty()) { + this->clearAndResizeStorageToDim(); + return; + } + + // Ensure csr proper csr format even if empty + a->resizeStorageToDim(); + b->resizeStorageToDim(); + + kernels::SpVectorEWiseMultInverted> spFunctor; + auto result = spFunctor(a->mMatrixImpl, b->mMatrixImpl); + + // Assign the actual impl result to this storage + this->mMatrixImpl = std::move(result); + } + +} diff --git a/cubool/sources/cuda/kernels/spewisemultinverted.cuh b/cubool/sources/cuda/kernels/spewisemultinverted.cuh new file mode 100644 index 0000000..353bc97 --- /dev/null +++ b/cubool/sources/cuda/kernels/spewisemultinverted.cuh @@ -0,0 +1,106 @@ +/**********************************************************************************/ +/* MIT License */ +/* */ +/* Copyright (c) 2020, 2021 JetBrains-Research */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining a copy */ +/* of this software and associated documentation files (the "Software"), to deal */ +/* in the Software without restriction, including without limitation the rights */ +/* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell */ +/* copies of the Software, and to permit persons to whom the Software is */ +/* furnished to do so, subject to the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be included in all */ +/* copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR */ +/* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, */ +/* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE */ +/* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER */ +/* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, */ +/* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE */ +/* SOFTWARE. */ +/**********************************************************************************/ + +#ifndef CUBOOL_SPEWISEMULT_CUH +#define CUBOOL_SPEWISEMULT_CUH + +#include +#include +#include +#include + +#include + +namespace cubool { + namespace kernels { + + template + struct SpVectorEWiseMultInverted { + template + using ContainerType = thrust::device_vector::other>; + using MatrixType = nsparse::matrix; + using LargeIndexType = unsigned long; + + static_assert(sizeof(LargeIndexType) > sizeof(IndexType), "Values intersection index must be larger"); + + static void fillIndices(const MatrixType& m, ContainerType& out) { + thrust::for_each(thrust::counting_iterator(0), thrust::counting_iterator(m.m_vals), + [rowOffset=m.m_row_index.data(), colIndex=m.m_col_index.data(), + outIndices=out.data(), nrows=m.m_rows, ncols=m.m_cols] __device__ (IndexType valueId) { + LargeIndexType row = findNearestRowIdx(valueId, nrows, rowOffset); + LargeIndexType col = colIndex[valueId]; + LargeIndexType index = row * ncols + col; + outIndices[valueId] = index; + }); + } + + MatrixType operator()(const MatrixType& a, const MatrixType& b) { + auto aNvals = a.m_vals; + auto bNvals = b.m_vals; + auto worst = aNvals; + + // Allocate memory for the worst case scenario + ContainerType inputA(aNvals); + ContainerType inputB(bNvals); + + fillIndices(a, inputA); + fillIndices(b, inputB); + + ContainerType intersected(worst); + + auto out = thrust::set_difference(inputA.begin(), inputA.end(), + inputB.begin(), inputB.end(), + intersected.begin()); + + // Count result nvals count + auto nvals = thrust::distance(intersected.begin(), out); + + ContainerType rowOffsetTmp(a.m_rows + 1); + ContainerType colIndex(nvals); + + thrust::fill(rowOffsetTmp.begin(), rowOffsetTmp.end(), 0); + + thrust::for_each(thrust::counting_iterator(0), thrust::counting_iterator(nvals), + [rowOffset=rowOffsetTmp.data(), colIndex=colIndex.data(), intersected=intersected.data(), + nrows=a.m_rows, ncols=a.m_cols] __device__ (IndexType valueId) { + LargeIndexType i = intersected[valueId]; + LargeIndexType row = i / ncols; + LargeIndexType col = i % ncols; + atomicAdd((rowOffset + row).get(), 1); + colIndex[valueId] = (IndexType) col; + }); + + ContainerType rowOffset(a.m_rows + 1); + thrust::exclusive_scan(rowOffsetTmp.begin(), rowOffsetTmp.end(), rowOffset.begin(), 0, thrust::plus()); + + assert(nvals == rowOffset.back()); + + return MatrixType(std::move(colIndex), std::move(rowOffset), a.m_rows, a.m_cols, nvals); + } + }; + + } +} + +#endif //CUBOOL_SPEWISEMULT_CUH diff --git a/cubool/sources/sequential/sq_ewisemultinverted.cpp b/cubool/sources/sequential/sq_ewisemultinverted.cpp new file mode 100644 index 0000000..ee810c9 --- /dev/null +++ b/cubool/sources/sequential/sq_ewisemultinverted.cpp @@ -0,0 +1,105 @@ +/**********************************************************************************/ +/* MIT License */ +/* */ +/* Copyright (c) 2020, 2021 JetBrains-Research */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining a copy */ +/* of this software and associated documentation files (the "Software"), to deal */ +/* in the Software without restriction, including without limitation the rights */ +/* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell */ +/* copies of the Software, and to permit persons to whom the Software is */ +/* furnished to do so, subject to the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be included in all */ +/* copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR */ +/* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, */ +/* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE */ +/* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER */ +/* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, */ +/* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE */ +/* SOFTWARE. */ +/**********************************************************************************/ + +#include +#include + +namespace cubool { + + void sq_ewisemultinverted(const CsrData& a, const CsrData& b, CsrData& out) { + out.rowOffsets.resize(a.nrows + 1, 0); + + size_t nvals = 0; + + // Count nnz of the result matrix to allocate memory + for (index i = 0; i < a.nrows; i++) { + index ak = a.rowOffsets[i]; + index bk = b.rowOffsets[i]; + index asize = a.rowOffsets[i + 1] - ak; + index bsize = b.rowOffsets[i + 1] - bk; + + const index* ar = &a.colIndices[ak]; + const index* br = &b.colIndices[bk]; + const index* arend = ar + asize; + const index* brend = br + bsize; + + index nvalsInRow = 0; + + while (ar != arend && br != brend) { + if (*ar == *br) { + ar++; + br++; + } + else if (*ar < *br) { + nvalsInRow++; + ar++; + } + else { + br++; + } + } + nvalsInRow += arend - ar; + + nvals += nvalsInRow; + out.rowOffsets[i] = nvalsInRow; + } + + // Eval row offsets + exclusive_scan(out.rowOffsets.begin(), out.rowOffsets.end(), 0); + + // Allocate memory for values + out.nvals = nvals; + out.colIndices.resize(nvals); + + // Fill sorted column indices + size_t k = 0; + for (index i = 0; i < a.nrows; i++) { + const index* ar = &a.colIndices[a.rowOffsets[i]]; + const index* br = &b.colIndices[b.rowOffsets[i]]; + const index* arend = &a.colIndices[a.rowOffsets[i + 1]]; + const index* brend = &b.colIndices[b.rowOffsets[i + 1]]; + + while (ar != arend && br != brend) { + if (*ar == *br) { + ar++; + br++; + } + else if (*ar < *br) { + out.colIndices[k] = *ar; + k++; + ar++; + } + else { + br++; + } + } + + while (ar != arend) { + out.colIndices[k] = *ar; + k++; + ar++; + } + } + } +} diff --git a/cubool/sources/sequential/sq_ewisemultinverted.hpp b/cubool/sources/sequential/sq_ewisemultinverted.hpp new file mode 100644 index 0000000..99b2615 --- /dev/null +++ b/cubool/sources/sequential/sq_ewisemultinverted.hpp @@ -0,0 +1,42 @@ +/**********************************************************************************/ +/* MIT License */ +/* */ +/* Copyright (c) 2020, 2021 JetBrains-Research */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining a copy */ +/* of this software and associated documentation files (the "Software"), to deal */ +/* in the Software without restriction, including without limitation the rights */ +/* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell */ +/* copies of the Software, and to permit persons to whom the Software is */ +/* furnished to do so, subject to the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be included in all */ +/* copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR */ +/* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, */ +/* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE */ +/* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER */ +/* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, */ +/* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE */ +/* SOFTWARE. */ +/**********************************************************************************/ + +#ifndef CUBOOL_SQ_EWISEMULTINVERTED_HPP +#define CUBOOL_SQ_EWISEMULTINVERTED_HPP + +#include + +namespace cubool { + + /** + * Element-wise multiplication of the matrices `a` and inverted `b`. + * + * @param a Input matrix + * @param b Input matrix, which will be inverted before multiplication + * @param[out] out Where to store the result + */ + void sq_ewisemultinverted(const CsrData& a, const CsrData& b, CsrData& out); +} + +#endif //CUBOOL_SQ_EWISEMULTINVERTED_HPP diff --git a/cubool/sources/sequential/sq_matrix.cpp b/cubool/sources/sequential/sq_matrix.cpp index df96e4e..00473f1 100644 --- a/cubool/sources/sequential/sq_matrix.cpp +++ b/cubool/sources/sequential/sq_matrix.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -235,6 +236,31 @@ namespace cubool { this->mData = std::move(out); } + void SqMatrix::eWiseMultInverted(const MatrixBase &aBase, const MatrixBase &bBase, bool checkTime) { + auto a = dynamic_cast(&aBase); + auto b = dynamic_cast(&bBase); + + CHECK_RAISE_ERROR(a != nullptr, InvalidArgument, "Provided matrix does not belongs to sequential matrix class"); + CHECK_RAISE_ERROR(b != nullptr, InvalidArgument, "Provided matrix does not belongs to sequential matrix class"); + + assert(a->getNrows() == this->getNrows()); + assert(a->getNcols() == this->getNcols()); + assert(a->getNrows() == b->getNrows()); + assert(a->getNcols() == b->getNcols()); + + CsrData out; + out.nrows = this->getNrows(); + out.ncols = this->getNcols(); + + a->allocateStorage(); + b->allocateStorage(); + sq_ewisemultinverted(a->mData, b->mData, out); + + this->mData = std::move(out); + } + + + index SqMatrix::getNrows() const { return mData.nrows; } diff --git a/cubool/sources/sequential/sq_matrix.hpp b/cubool/sources/sequential/sq_matrix.hpp index 1a3f539..d348232 100644 --- a/cubool/sources/sequential/sq_matrix.hpp +++ b/cubool/sources/sequential/sq_matrix.hpp @@ -51,6 +51,7 @@ namespace cubool { void kronecker(const MatrixBase &aBase, const MatrixBase &bBase, bool checkTime) override; void eWiseAdd(const MatrixBase &aBase, const MatrixBase &bBase, bool checkTime) override; void eWiseMult(const MatrixBase &a, const MatrixBase &b, bool checkTime) override; + void eWiseMultInverted(const MatrixBase &a, const MatrixBase &b, bool checkTime) override; index getNrows() const override; index getNcols() const override; diff --git a/cubool/tests/CMakeLists.txt b/cubool/tests/CMakeLists.txt index 0e0f620..84ddc96 100644 --- a/cubool/tests/CMakeLists.txt +++ b/cubool/tests/CMakeLists.txt @@ -34,6 +34,9 @@ target_link_libraries(test_matrix_ewiseadd PUBLIC testing) add_executable(test_matrix_ewisemult test_matrix_ewisemult.cpp) target_link_libraries(test_matrix_ewisemult PUBLIC testing) +add_executable(test_matrix_ewisemult_inverted test_matrix_ewisemult_inverted.cpp) +target_link_libraries(test_matrix_ewisemult_inverted PUBLIC testing) + add_executable(test_vector_misc test_vector_misc.cpp) target_link_libraries(test_vector_misc PUBLIC testing) diff --git a/cubool/tests/test_matrix_ewisemult_inverted.cpp b/cubool/tests/test_matrix_ewisemult_inverted.cpp new file mode 100644 index 0000000..2e32520 --- /dev/null +++ b/cubool/tests/test_matrix_ewisemult_inverted.cpp @@ -0,0 +1,134 @@ +#include "cubool/cubool.h" +#include +#include + +using DataMatrix = std::vector>; + +static void printTestingMatrix(const testing::Matrix &matrix, std::string name = "") { + if (name != "") { + std::cout << name << std::endl; + } + + for (int i = 0; i < matrix.nvals; i++) { + printf("(%d, %d)\n", matrix.rowsIndex[i], matrix.colsIndex[i]); + } +} + +static void printCuboolMatrix(cuBool_Matrix matrix, std::string name = "") { + if (name != "") { + std::cout << name << std::endl; + } + + cuBool_Index nvals; + cuBool_Matrix_Nvals(matrix, &nvals); + std::vector rows(nvals), cols(nvals); + cuBool_Matrix_ExtractPairs(matrix, rows.data(), cols.data(), &nvals); + + for (int i = 0; i < nvals; i++) { + printf("(%d, %d)\n", rows[i], cols[i]); + } +} + +void testApplyNotMask(const DataMatrix &matrix_data, const DataMatrix &mask_data) { + cuBool_Index nrows, ncols; + nrows = matrix_data.size(); + ncols = matrix_data[0].size(); + testing::Matrix test_matrix = testing::Matrix::generatet(nrows, ncols, + [&matrix_data](cuBool_Index i, cuBool_Index j) { return matrix_data[i][j]; }); + nrows = mask_data.size(); + ncols = mask_data[0].size(); + testing::Matrix test_mask = testing::Matrix::generatet(nrows, ncols, + [&mask_data](cuBool_Index i, cuBool_Index j) { return mask_data[i][j]; }); + + cuBool_Matrix matrix, mask, result; + ASSERT_EQ(cuBool_Matrix_New(&matrix, test_matrix.nrows, test_matrix.ncols), CUBOOL_STATUS_SUCCESS); + ASSERT_EQ(cuBool_Matrix_New(&mask, test_mask.nrows, test_mask.ncols), CUBOOL_STATUS_SUCCESS); + ASSERT_EQ(cuBool_Matrix_New(&result, test_matrix.nrows, test_matrix.ncols), CUBOOL_STATUS_SUCCESS); + + ASSERT_EQ(cuBool_Matrix_Build(matrix, test_matrix.rowsIndex.data(), test_matrix.colsIndex.data(), test_matrix.nvals, + CUBOOL_HINT_VALUES_SORTED & CUBOOL_HINT_NO_DUPLICATES), CUBOOL_STATUS_SUCCESS); + ASSERT_EQ(cuBool_Matrix_Build(mask, test_mask.rowsIndex.data(), test_mask.colsIndex.data(), test_mask.nvals, + CUBOOL_HINT_VALUES_SORTED & CUBOOL_HINT_NO_DUPLICATES), CUBOOL_STATUS_SUCCESS); + + cuBool_Matrix_EWiseMulInverted(result, matrix, mask, CUBOOL_HINT_NO); + + // validate value of algorithm + cuBool_Index nvals; + cuBool_Matrix_Nvals(result, &nvals); + std::vector rows(nvals), cols(nvals); + cuBool_Matrix_ExtractPairs(result, rows.data(), cols.data(), &nvals); + + cuBool_Matrix_Free(matrix); + cuBool_Matrix_Free(mask); + cuBool_Matrix_Free(result); + + auto mask_data_inverted = mask_data; + for (auto &row : mask_data_inverted) { + for (int &value : row) { + value = !value; + } + } + + std::vector result_data(matrix_data.size(), std::vector(matrix_data[0].size(), 0)); + for (int i = 0; i < nvals; i++) { + result_data[rows[i]][cols[i]] = 1; + } + + for (int i = 0; i < matrix_data.size(); i++) { + for (int j = 0; j < matrix_data[0].size(); j++) { + ASSERT_EQ(matrix_data[i][j] * mask_data_inverted[i][j], result_data[i][j]); + } + } + +} + +TEST(cuBool_Matrix, ApplyMatrix) { + ASSERT_EQ(cuBool_Initialize(CUBOOL_HINT_NO), CUBOOL_STATUS_SUCCESS); + + DataMatrix matrix { + {1, 0, 0}, + {0, 0, 0}, + {0, 1, 0}, + }; + + DataMatrix mask { + {0, 1, 1}, + {1, 0, 1}, + {0, 1, 1}, + }; + // iverted is + // 1 0 0 + // 0 1 0 + // 1 0 0 + // matrix & ~mask must have (0, 0) + + testApplyNotMask(matrix, mask); + + ASSERT_EQ(cuBool_Finalize(), CUBOOL_STATUS_SUCCESS); +} + +TEST(cuBool_Matrix, ApplyMatrixRandom) { + ASSERT_EQ(cuBool_Initialize(CUBOOL_HINT_NO), CUBOOL_STATUS_SUCCESS); + + for (int i = 0; i < 102; i++) { + int n = rand() % 10 + 1; + int m = rand() % 10 + 1; + + DataMatrix matrix(n, std::vector(m, 0)); + DataMatrix mask(n, std::vector(m, 0)); + + for (int i = 0; i < n; i++) { + for (int j = 0; j < m; j++) { + matrix[i][j] = rand() & 1; + mask[i][j] = rand() & 1; + } + } + + testApplyNotMask(matrix, mask); + } + + ASSERT_EQ(cuBool_Finalize(), CUBOOL_STATUS_SUCCESS); +} + + +CUBOOL_GTEST_MAIN From 2f5daace52f8ff9aab55924e2612fa62aedd1c76 Mon Sep 17 00:00:00 2001 From: Mitya <8863314@gmail.com> Date: Sun, 24 Nov 2024 23:49:19 +0300 Subject: [PATCH 05/69] Fixed mistake in matrix ewise add --- cubool/sources/cuda/kernels/spmerge.cuh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cubool/sources/cuda/kernels/spmerge.cuh b/cubool/sources/cuda/kernels/spmerge.cuh index 8a14f8d..7a5b62d 100644 --- a/cubool/sources/cuda/kernels/spmerge.cuh +++ b/cubool/sources/cuda/kernels/spmerge.cuh @@ -53,7 +53,7 @@ namespace cubool { assert(a.m_cols == b.m_cols); IndexType rows = a.m_rows; - IndexType cols = a.m_rows; + IndexType cols = a.m_cols; constexpr auto config_merge = make_bin_seq< From 99f1d8ef722a7b88be8fd61b15456f5658799d5e Mon Sep 17 00:00:00 2001 From: Dmitriy Kozenko <8863314@gmail.com> Date: Sun, 16 Feb 2025 23:43:32 +0300 Subject: [PATCH 06/69] Fixed errors in cmake: incorrect compile options for NVCC --- cubool/CMakeLists.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cubool/CMakeLists.txt b/cubool/CMakeLists.txt index cd80032..451efd0 100644 --- a/cubool/CMakeLists.txt +++ b/cubool/CMakeLists.txt @@ -206,11 +206,7 @@ target_compile_definitions(cubool PRIVATE CUBOOL_VERSION_MAJOR=${CUBOOL_VERSION_ target_compile_definitions(cubool PRIVATE CUBOOL_VERSION_MINOR=${CUBOOL_VERSION_MINOR}) target_compile_definitions(cubool PRIVATE CUBOOL_VERSION_SUB=${CUBOOL_VERSION_SUB}) -target_compile_features(cubool PUBLIC cxx_std_14) - -target_compile_options(cubool PRIVATE $<$: -Wall>) -target_compile_options(cubool PRIVATE $<$,$>: -O2>) -target_compile_options(cubool PRIVATE $<$,$>: -O0>) +target_compile_features(cubool PUBLIC cxx_std_17) set_target_properties(cubool PROPERTIES CXX_STANDARD 17) set_target_properties(cubool PROPERTIES CXX_STANDARD_REQUIRED ON) From b40486326dbd9bacb763afbf4956e23ecc61f9d3 Mon Sep 17 00:00:00 2001 From: Dmitriy Kozenko <8863314@gmail.com> Date: Wed, 19 Feb 2025 00:01:52 +0300 Subject: [PATCH 07/69] Added option for using matrix addition optimization from nsparse --- CMakeLists.txt | 2 + cubool/CMakeLists.txt | 2 + cubool/sources/cuda/kernels/spmerge.cuh | 71 +++++++++++++++++++++++++ 3 files changed, 75 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index da9e6d9..a432980 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,8 @@ option(CUBOOL_WITH_NAIVE "Build library with naive and naive-shared dens option(CUBOOL_BUILD_TESTS "Build project unit-tests with gtest" ON) option(CUBOOL_COPY_TO_PY_PACKAGE "Copy compiled shared library into python package folder (for package use purposes)" ON) +option(CUBOOL_USE_NSPARSE_MERGE_FUNCTOR "Use nsparse optimiztion for matrix addition function" ON) + set(CUBOOL_VERSION_MAJOR 1) set(CUBOOL_VERSION_MINOR 0) set(CUBOOL_VERSION_SUB 0) diff --git a/cubool/CMakeLists.txt b/cubool/CMakeLists.txt index 451efd0..5434c02 100644 --- a/cubool/CMakeLists.txt +++ b/cubool/CMakeLists.txt @@ -206,6 +206,8 @@ target_compile_definitions(cubool PRIVATE CUBOOL_VERSION_MAJOR=${CUBOOL_VERSION_ target_compile_definitions(cubool PRIVATE CUBOOL_VERSION_MINOR=${CUBOOL_VERSION_MINOR}) target_compile_definitions(cubool PRIVATE CUBOOL_VERSION_SUB=${CUBOOL_VERSION_SUB}) +target_compile_definitions(cubool PRIVATE CUBOOL_USE_NSPARSE_MERGE_FUNCTOR=$) + target_compile_features(cubool PUBLIC cxx_std_17) set_target_properties(cubool PROPERTIES CXX_STANDARD 17) diff --git a/cubool/sources/cuda/kernels/spmerge.cuh b/cubool/sources/cuda/kernels/spmerge.cuh index 7a5b62d..96805cb 100644 --- a/cubool/sources/cuda/kernels/spmerge.cuh +++ b/cubool/sources/cuda/kernels/spmerge.cuh @@ -29,9 +29,13 @@ #include #include +#include +#include + namespace cubool { namespace kernels { +#if CUBOOL_USE_NSPARSE_MERGE_FUNCTOR template class SpMergeFunctor { public: @@ -77,6 +81,73 @@ namespace cubool { private: nsparse::unique_merge_functor_t uniqueMergeFunctor; }; +#else + template + class SpMergeFunctor { + public: + template + using ContainerType = thrust::device_vector::other>; + using MatrixType = nsparse::matrix; + using LargeIndexType = unsigned long; + + static void fillIndices(const MatrixType& m, ContainerType& out) { + thrust::for_each(thrust::counting_iterator(0), thrust::counting_iterator(m.m_vals), + [rowOffset=m.m_row_index.data(), colIndex=m.m_col_index.data(), + outIndices=out.data(), nrows=m.m_rows, ncols=m.m_cols] __device__ (IndexType valueId) { + LargeIndexType row = findNearestRowIdx(valueId, nrows, rowOffset); + LargeIndexType col = colIndex[valueId]; + LargeIndexType index = row * ncols + col; + outIndices[valueId] = index; + }); + } + + MatrixType operator()(const MatrixType& a, const MatrixType& b) { + auto aNvals = a.m_vals; + auto bNvals = b.m_vals; + // auto worst = std::min(aNvals, bNvals); + auto worst = aNvals + bNvals; + + // Allocate memory for the worst case scenario + ContainerType inputA(aNvals); + ContainerType inputB(bNvals); + + fillIndices(a, inputA); + fillIndices(b, inputB); + + ContainerType intersected(worst); + + auto out = thrust::set_union(inputA.begin(), inputA.end(), + inputB.begin(), inputB.end(), + intersected.begin()); + + // Count result nvals count + auto nvals = thrust::distance(intersected.begin(), out); + + ContainerType rowOffsetTmp(a.m_rows + 1); + ContainerType colIndex(nvals); + + thrust::fill(rowOffsetTmp.begin(), rowOffsetTmp.end(), 0); + + thrust::for_each(thrust::counting_iterator(0), thrust::counting_iterator(nvals), + [rowOffset=rowOffsetTmp.data(), colIndex=colIndex.data(), intersected=intersected.data(), + nrows=a.m_rows, ncols=a.m_cols] __device__ (IndexType valueId) { + LargeIndexType i = intersected[valueId]; + LargeIndexType row = i / ncols; + LargeIndexType col = i % ncols; + atomicAdd((rowOffset + row).get(), 1); + colIndex[valueId] = (IndexType) col; + }); + + ContainerType rowOffset(a.m_rows + 1); + thrust::exclusive_scan(rowOffsetTmp.begin(), rowOffsetTmp.end(), rowOffset.begin(), 0, thrust::plus()); + + assert(nvals == rowOffset.back()); + + return MatrixType(std::move(colIndex), std::move(rowOffset), a.m_rows, a.m_cols, nvals); + } + }; + +#endif } } From f81ccea03657b19641776d4bdbe215c653c36cd4 Mon Sep 17 00:00:00 2001 From: Dmitriy Kozenko <8863314@gmail.com> Date: Sat, 19 Apr 2025 22:48:49 +0300 Subject: [PATCH 08/69] Updated gtest version to last release for compability with last cmake version --- deps/gtest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/gtest b/deps/gtest index d9c309f..6910c9d 160000 --- a/deps/gtest +++ b/deps/gtest @@ -1 +1 @@ -Subproject commit d9c309fdab807b716c2cf4d4a42989b8c34f712a +Subproject commit 6910c9d9165801d8827d628cb72eb7ea9dd538c5 From da6be345f8a3a0962887e7f1f934adc9b07058bb Mon Sep 17 00:00:00 2001 From: Dmitriy Kozenko <8863314@gmail.com> Date: Sat, 3 May 2025 15:49:38 +0300 Subject: [PATCH 09/69] Restored python package build --- CMakeLists.txt | 12 ++---------- cubool/CMakeLists.txt | 23 +++++++++++------------ 2 files changed, 13 insertions(+), 22 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a432980..a6404f7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,14 +34,6 @@ endif() # Configure cuda dependencies if (CUBOOL_WITH_CUDA) - message(STATUS "Add cub as cuda utility") - # set(CUB_ENABLE_HEADER_TESTING OFF CACHE BOOL "" FORCE) - # set(CUB_ENABLE_TESTING OFF CACHE BOOL "" FORCE) - # set(CUB_ENABLE_EXAMPLES OFF CACHE BOOL "" FORCE) - # add_subdirectory(deps/cub) - # add_library(cub INTERFACE IMPORTED) - # target_link_libraries(cub INTERFACE CUB::CUB) - message(STATUS "Add nsparse library as crs matrix multiplication backend") add_subdirectory(deps/nsparse-um) endif() @@ -60,7 +52,7 @@ endif() add_subdirectory(cubool) # Copy scripts into binary directory -# file(COPY scripts DESTINATION ${CMAKE_BINARY_DIR}/) +file(COPY scripts DESTINATION ${CMAKE_BINARY_DIR}/) # Copy python related stuff -# file(COPY python DESTINATION ${CMAKE_BINARY_DIR}/) +file(COPY python DESTINATION ${CMAKE_BINARY_DIR}/) diff --git a/cubool/CMakeLists.txt b/cubool/CMakeLists.txt index 5434c02..ed3cb41 100644 --- a/cubool/CMakeLists.txt +++ b/cubool/CMakeLists.txt @@ -222,7 +222,6 @@ endforeach() if (CUBOOL_WITH_CUDA) set_target_properties(cubool PROPERTIES CUDA_STANDARD 14) set_target_properties(cubool PROPERTIES CUDA_STANDARD_REQUIRED ON) - # set_target_properties(cubool PROPERTIES CUDA_SEPARABLE_COMPILATION ON) # Settings: https://arnon.dk/matching-sm-architectures-arch-and-gencode-for-various-nvidia-cards/ @@ -249,14 +248,14 @@ if (CUBOOL_BUILD_TESTS) endif() # Copy cubool library after build if allowed -# if (CUBOOL_COPY_TO_PY_PACKAGE) -# set(LIBRARY_FILE_NAME ${TARGET_FILE_NAME}) -# -# add_custom_command( -# TARGET cubool POST_BUILD -# COMMAND "${CMAKE_COMMAND}" -E -# copy -# "${CMAKE_BINARY_DIR}/cubool/${LIBRARY_FILE_NAME}" -# "${CMAKE_BINARY_DIR}/python/pycubool" -# COMMENT "Copy ${LIBRARY_FILE_NAME} compiled lib into python folder") -# endif() +if (CUBOOL_COPY_TO_PY_PACKAGE) + set(LIBRARY_FILE_NAME ${TARGET_FILE_NAME}) + + add_custom_command( + TARGET cubool POST_BUILD + COMMAND "${CMAKE_COMMAND}" -E + copy + "${CMAKE_BINARY_DIR}/cubool/${LIBRARY_FILE_NAME}" + "${CMAKE_BINARY_DIR}/python/pycubool" + COMMENT "Copy ${LIBRARY_FILE_NAME} compiled lib into python folder") +endif() From 69d38a67d74cc032ab9c313721f40cba2f79125c Mon Sep 17 00:00:00 2001 From: Dmitriy Kozenko <8863314@gmail.com> Date: Sat, 3 May 2025 17:07:42 +0300 Subject: [PATCH 10/69] Removed temporary solutions for preparing PR --- cubool/include/cubool/cubool.h | 14 +++++------ cubool/sources/core/matrix.cpp | 10 +++----- cubool/sources/cuda/kernels/spmerge.cuh | 1 - .../tests/test_matrix_ewisemult_inverted.cpp | 25 ------------------- deps/nsparse-um/CMakeLists.txt | 1 - .../include/nsparse/unified_allocator.h | 1 - deps/nsparse-um/test/CMakeLists.txt | 1 - deps/nsparse/test/CMakeLists.txt | 1 - 8 files changed, 11 insertions(+), 43 deletions(-) diff --git a/cubool/include/cubool/cubool.h b/cubool/include/cubool/cubool.h index ef7d9b5..2859542 100644 --- a/cubool/include/cubool/cubool.h +++ b/cubool/include/cubool/cubool.h @@ -926,20 +926,20 @@ CUBOOL_EXPORT CUBOOL_API cuBool_Status cuBool_Kronecker( ); /** - * Performs result = left + ~right, where - * '+' is boolean semiring 'or' operation + * Performs result = left * ~right, where + * '*' is boolean semiring 'and' operation * '~' is operation for invert matrix (0 swaps to 1 and 1 to 0) * - * @note Matrices must be compatible + * @note To perform this operation matrices must be compatible + * dim(left) = M x T + * dim(right) = T x N * dim(result) = M x N - * dim(left) = M x N - * dim(right) = M x N * * @note Pass `CUBOOL_HINT_TIME_CHECK` hint to measure operation time * * @param result[out] Destination matrix to store result - * @param left Source matrix to be added - * @param right Source matrix to be inverted and added + * @param left Source matrix to be multiplied + * @param right Source matrix to be inverted and multiplied * @param hints Hints for the operation * * @return Error code on this operation diff --git a/cubool/sources/core/matrix.cpp b/cubool/sources/core/matrix.cpp index 730fe63..49e35c3 100644 --- a/cubool/sources/core/matrix.cpp +++ b/cubool/sources/core/matrix.cpp @@ -370,21 +370,19 @@ namespace cubool { msk->commitCache(); this->releaseCache(); - /* if (checkTime) { - TIMER_ACTION(timer, mHnd->eWiseMult(*a->mHnd, *b->mHnd, false)); + TIMER_ACTION(timer, mHnd->eWiseMultInverted(*mat->mHnd, *msk->mHnd, false)); LogStream stream(*Library::getLogger()); stream << Logger::Level::Info << "Time: " << timer.getElapsedTimeMs() << " ms " - << "Matrix::eWiseMult: " + << "Matrix::eWiseMultInverted: " << this->getDebugMarker() << " = " - << a->getDebugMarker() << " + " - << b->getDebugMarker() << LogStream::cmt; + << mat->getDebugMarker() << " + " + << msk->getDebugMarker() << LogStream::cmt; return; } - */ mHnd->eWiseMultInverted(*mat->mHnd, *msk->mHnd, false); } diff --git a/cubool/sources/cuda/kernels/spmerge.cuh b/cubool/sources/cuda/kernels/spmerge.cuh index 96805cb..4734e35 100644 --- a/cubool/sources/cuda/kernels/spmerge.cuh +++ b/cubool/sources/cuda/kernels/spmerge.cuh @@ -104,7 +104,6 @@ namespace cubool { MatrixType operator()(const MatrixType& a, const MatrixType& b) { auto aNvals = a.m_vals; auto bNvals = b.m_vals; - // auto worst = std::min(aNvals, bNvals); auto worst = aNvals + bNvals; // Allocate memory for the worst case scenario diff --git a/cubool/tests/test_matrix_ewisemult_inverted.cpp b/cubool/tests/test_matrix_ewisemult_inverted.cpp index 2e32520..250ba55 100644 --- a/cubool/tests/test_matrix_ewisemult_inverted.cpp +++ b/cubool/tests/test_matrix_ewisemult_inverted.cpp @@ -4,31 +4,6 @@ using DataMatrix = std::vector>; -static void printTestingMatrix(const testing::Matrix &matrix, std::string name = "") { - if (name != "") { - std::cout << name << std::endl; - } - - for (int i = 0; i < matrix.nvals; i++) { - printf("(%d, %d)\n", matrix.rowsIndex[i], matrix.colsIndex[i]); - } -} - -static void printCuboolMatrix(cuBool_Matrix matrix, std::string name = "") { - if (name != "") { - std::cout << name << std::endl; - } - - cuBool_Index nvals; - cuBool_Matrix_Nvals(matrix, &nvals); - std::vector rows(nvals), cols(nvals); - cuBool_Matrix_ExtractPairs(matrix, rows.data(), cols.data(), &nvals); - - for (int i = 0; i < nvals; i++) { - printf("(%d, %d)\n", rows[i], cols[i]); - } -} - void testApplyNotMask(const DataMatrix &matrix_data, const DataMatrix &mask_data) { cuBool_Index nrows, ncols; nrows = matrix_data.size(); diff --git a/deps/nsparse-um/CMakeLists.txt b/deps/nsparse-um/CMakeLists.txt index cdc4463..91ed084 100644 --- a/deps/nsparse-um/CMakeLists.txt +++ b/deps/nsparse-um/CMakeLists.txt @@ -3,7 +3,6 @@ project(nsparse_um LANGUAGES CXX CUDA) add_library(nsparse_um INTERFACE) target_include_directories(nsparse_um INTERFACE include/) -# target_link_libraries(nsparse_um INTERFACE cub) target_link_libraries(nsparse_um INTERFACE) target_compile_options(nsparse_um INTERFACE $<$: --expt-relaxed-constexpr --expt-extended-lambda>) diff --git a/deps/nsparse-um/include/nsparse/unified_allocator.h b/deps/nsparse-um/include/nsparse/unified_allocator.h index c15f19e..9c3b2f8 100644 --- a/deps/nsparse-um/include/nsparse/unified_allocator.h +++ b/deps/nsparse-um/include/nsparse/unified_allocator.h @@ -3,7 +3,6 @@ #include #include #include -// #include #include #include diff --git a/deps/nsparse-um/test/CMakeLists.txt b/deps/nsparse-um/test/CMakeLists.txt index 6a29690..6a1aaff 100644 --- a/deps/nsparse-um/test/CMakeLists.txt +++ b/deps/nsparse-um/test/CMakeLists.txt @@ -3,7 +3,6 @@ project(nsparse_um_test CXX CUDA) add_executable(${PROJECT_NAME} src/nsparse_test.cu src/utils.cpp) -# target_link_libraries(${PROJECT_NAME} PRIVATE gtest_main nsparse_um cub) target_link_libraries(${PROJECT_NAME} PRIVATE gtest_main nsparse_um) target_compile_features(${PROJECT_NAME} PRIVATE cxx_std_14) diff --git a/deps/nsparse/test/CMakeLists.txt b/deps/nsparse/test/CMakeLists.txt index 9927725..5abd415 100644 --- a/deps/nsparse/test/CMakeLists.txt +++ b/deps/nsparse/test/CMakeLists.txt @@ -2,7 +2,6 @@ cmake_minimum_required(VERSION 3.11) project(nsparse_test CXX CUDA) add_executable(${PROJECT_NAME} src/nsparse_test.cu src/utils.cpp) -# target_link_libraries(${PROJECT_NAME} PRIVATE gtest_main nsparse cub) target_link_libraries(${PROJECT_NAME} PRIVATE gtest_main nsparse) target_compile_features(${PROJECT_NAME} PRIVATE cxx_std_14) From f304b58dc2cf61d7234d47399f1c13dafa255df2 Mon Sep 17 00:00:00 2001 From: Kostya Date: Wed, 12 Mar 2025 14:11:55 +0300 Subject: [PATCH 11/69] ci: run on self-hosted platform --- .github/workflows/ubuntu.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index cd33dd3..d448174 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -4,7 +4,7 @@ name: Ubuntu on: push: - branches: [ master ] + branches: [ master, self-hosted-runner ] pull_request: branches: [ master ] workflow_dispatch: @@ -12,7 +12,7 @@ on: jobs: build: name: Build ${{ matrix.os }} GCC ${{ matrix.gcc }} CUDA ${{ matrix.cuda }} - runs-on: ${{ matrix.os }} + runs-on: self-hosted strategy: fail-fast: false matrix: From fb6867b865cfc4ba4eb1872cb03062c14d6b55b9 Mon Sep 17 00:00:00 2001 From: Kostya Date: Wed, 12 Mar 2025 19:34:30 +0300 Subject: [PATCH 12/69] ci: update versions of packages --- .github/workflows/ubuntu.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index d448174..6f16225 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -26,10 +26,10 @@ jobs: artifact: "cubool-ubuntu-build.tar.xz" steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: submodules: true - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 with: python-version: '3.7' @@ -76,7 +76,7 @@ jobs: run: tar cfz ${{ env.artifact }} ${{ env.build_dir }} - name: Upload binary - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: ${{ env.artifact }} path: ${{ env.artifact }} \ No newline at end of file From 21de87c7b674dd7864db54f30a4ab01fd4ef2cb3 Mon Sep 17 00:00:00 2001 From: Kostya Date: Wed, 12 Mar 2025 20:02:33 +0300 Subject: [PATCH 13/69] ci: comment unnecessary python installation --- .github/workflows/ubuntu.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 6f16225..3622014 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -29,9 +29,9 @@ jobs: - uses: actions/checkout@v4 with: submodules: true - - uses: actions/setup-python@v5 - with: - python-version: '3.7' + # - uses: actions/setup-python@v5 + # with: + # python-version: '3.7' - name: Install CUDA env: From 4d5e940cc2969b4748725fa79212199c39c4482e Mon Sep 17 00:00:00 2001 From: Kostya Date: Mon, 17 Mar 2025 14:04:08 +0300 Subject: [PATCH 14/69] ci: left only the installation step --- .github/workflows/ubuntu.yml | 70 ++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 39 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 3622014..fae441e 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -17,9 +17,9 @@ jobs: fail-fast: false matrix: include: - - os: ubuntu-18.04 - cuda: "10.1" - gcc: 8 + - os: ubuntu-24.04 + cuda: "12.8" + gcc: 11 env: build_dir: "build" config: "Release" @@ -36,47 +36,39 @@ jobs: - name: Install CUDA env: cuda: ${{ matrix.cuda }} - run: | - source ./scripts/install_cuda_ubuntu.sh - if [[ $? -eq 0 ]]; then - # Set paths for subsequent steps, using ${CUDA_PATH} - echo "Adding CUDA to CUDA_PATH, PATH and LD_LIBRARY_PATH" - echo "CUDA_PATH=${CUDA_PATH}" >> $GITHUB_ENV - echo "${CUDA_PATH}/bin" >> $GITHUB_PATH - echo "LD_LIBRARY_PATH=${CUDA_PATH}/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV - fi + run: ./scripts/install_cuda_ubuntu.sh shell: bash - - name: Install and configure GCC and GXX - run: | - sudo apt-get install -y gcc-${{ matrix.gcc }} g++-${{ matrix.gcc }} - echo "СС=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV - echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV - echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV + # - name: Install and configure GCC and GXX + # run: | + # sudo apt-get install -y gcc-${{ matrix.gcc }} g++-${{ matrix.gcc }} + # echo "СС=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV + # echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV + # echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV - - name: Configure CMake build - run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=YES + # - name: Configure CMake build + # run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=YES - - name: Build library sources - working-directory: ${{ env.build_dir }} - run: cmake --build . --target all --verbose -j `nproc` + # - name: Build library sources + # working-directory: ${{ env.build_dir }} + # run: cmake --build . --target all --verbose -j `nproc` - - name: Run unit-tests (sequential backend) - working-directory: ${{ env.build_dir }} - run: bash scripts/run_tests_fallback.sh - shell: bash + # - name: Run unit-tests (sequential backend) + # working-directory: ${{ env.build_dir }} + # run: bash scripts/run_tests_fallback.sh + # shell: bash - - name: Run regression-tests (sequential backend) - working-directory: ${{ env.build_dir }}/python - run: bash run_tests.sh - shell: bash + # - name: Run regression-tests (sequential backend) + # working-directory: ${{ env.build_dir }}/python + # run: bash run_tests.sh + # shell: bash - - name: Prepare upload binary - shell: bash - run: tar cfz ${{ env.artifact }} ${{ env.build_dir }} + # - name: Prepare upload binary + # shell: bash + # run: tar cfz ${{ env.artifact }} ${{ env.build_dir }} - - name: Upload binary - uses: actions/upload-artifact@v4 - with: - name: ${{ env.artifact }} - path: ${{ env.artifact }} \ No newline at end of file + # - name: Upload binary + # uses: actions/upload-artifact@v4 + # with: + # name: ${{ env.artifact }} + # path: ${{ env.artifact }} \ No newline at end of file From 0c28baf0e4cf6a8803faf4b8f4e32c3a74f061af Mon Sep 17 00:00:00 2001 From: Kostya Date: Mon, 17 Mar 2025 14:04:40 +0300 Subject: [PATCH 15/69] feat(install_cuda): update the installation script --- scripts/install_cuda_ubuntu.sh | 90 +++++++++++++++++++++++++--------- 1 file changed, 68 insertions(+), 22 deletions(-) diff --git a/scripts/install_cuda_ubuntu.sh b/scripts/install_cuda_ubuntu.sh index 8c7e5ed..65ebe60 100644 --- a/scripts/install_cuda_ubuntu.sh +++ b/scripts/install_cuda_ubuntu.sh @@ -1,8 +1,12 @@ # Original script from https://github.com/ptheywood/cuda-cmake-github-actions CUDA_PACKAGES_IN=( - "command-line-tools" - "libraries-dev" + "cuda-compiler" + "cuda-cudart-dev" + "cuda-nvtx" + "cuda-nvrtc-dev" + "libcurand-dev" # 11-0+ + "cuda-cccl" # 11.4+, provides cub and thrust. On 11.3 known as cuda-thrust-11-3 ) # returns 0 (true) if a >= b @@ -32,6 +36,7 @@ function version_lt() { CUDA_VERSION_MAJOR_MINOR=${cuda} # Split the version. +# We (might/probably) don't know PATCH at this point - it depends which version gets installed. CUDA_MAJOR=$(echo "${CUDA_VERSION_MAJOR_MINOR}" | cut -d. -f1) CUDA_MINOR=$(echo "${CUDA_VERSION_MAJOR_MINOR}" | cut -d. -f2) CUDA_PATCH=$(echo "${CUDA_VERSION_MAJOR_MINOR}" | cut -d. -f3) @@ -64,41 +69,81 @@ fi ## Select CUDA packages to install CUDA_PACKAGES="" for package in "${CUDA_PACKAGES_IN[@]}" -do : +do : + # @todo This is not perfect. Should probably provide a separate list for diff versions # cuda-compiler-X-Y if CUDA >= 9.1 else cuda-nvcc-X-Y - if [[ "${package}" == "nvcc" ]] && version_ge "$CUDA_VERSION_MAJOR_MINOR" "9.1" ; then - package="compiler" - elif [[ "${package}" == "compiler" ]] && version_lt "$CUDA_VERSION_MAJOR_MINOR" "9.1" ; then - package="nvcc" + if [[ "${package}" == "cuda-nvcc" ]] && version_ge "$CUDA_VERSION_MAJOR_MINOR" "9.1" ; then + package="cuda-compiler" + elif [[ "${package}" == "cuda-compiler" ]] && version_lt "$CUDA_VERSION_MAJOR_MINOR" "9.1" ; then + package="cuda-nvcc" + # CUB/Thrust are packages in cuda-thrust in 11.3, but cuda-cccl in 11.4+ + elif [[ "${package}" == "cuda-thrust" || "${package}" == "cuda-cccl" ]]; then + # CUDA cuda-thrust >= 11.4 + if version_ge "$CUDA_VERSION_MAJOR_MINOR" "11.4" ; then + package="cuda-cccl" + # Use cuda-thrust > 11.2 + elif version_ge "$CUDA_VERSION_MAJOR_MINOR" "11.3" ; then + package="cuda-thrust" + # Do not include this pacakge < 11.3 + else + continue + fi + fi + # CUDA 11+ includes lib* / lib*-dev packages, which if they existed previously where cuda-cu*- / cuda-cu*-dev- + if [[ ${package} == libcu* ]] && version_lt "$CUDA_VERSION_MAJOR_MINOR" "11.0" ; then + package="${package/libcu/cuda-cu}" fi # Build the full package name and append to the string. - CUDA_PACKAGES+=" cuda-${package}-${CUDA_MAJOR}-${CUDA_MINOR}" + CUDA_PACKAGES+=" ${package}-${CUDA_MAJOR}-${CUDA_MINOR}" done echo "CUDA_PACKAGES ${CUDA_PACKAGES}" +CPU_ARCH="x86_64" PIN_FILENAME="cuda-ubuntu${UBUNTU_VERSION}.pin" -PIN_URL="https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/x86_64/${PIN_FILENAME}" -APT_KEY_URL="http://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/x86_64/7fa2af80.pub" -REPO_URL="http://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/x86_64/" +PIN_URL="https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/${CPU_ARCH}/${PIN_FILENAME}" +# apt keyring package now available https://developer.nvidia.com/blog/updating-the-cuda-linux-gpg-repository-key/ +KERYRING_PACKAGE_FILENAME="cuda-keyring_1.0-1_all.deb" +KEYRING_PACKAGE_URL="https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/${CPU_ARCH}/${KERYRING_PACKAGE_FILENAME}" +REPO_URL="https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/${CPU_ARCH}/" echo "PIN_FILENAME ${PIN_FILENAME}" echo "PIN_URL ${PIN_URL}" -echo "APT_KEY_URL ${APT_KEY_URL}" +echo "KEYRING_PACKAGE_URL ${KEYRING_PACKAGE_URL}" -sudo rm -r /var/lib/apt/lists/* -sudo apt-get clean -sudo apt-get update +# Detect if the script is being run as root, storing true/false in is_root. +is_root=false +if (( $EUID == 0)); then + is_root=true +fi +# Find if sudo is available +has_sudo=false +if command -v sudo &> /dev/null ; then + has_sudo=true +fi +# Decide if we can proceed or not (root or sudo is required) and if so store whether sudo should be used or not. +if [ "$is_root" = false ] && [ "$has_sudo" = false ]; then + echo "Root or sudo is required. Aborting." + exit 1 +elif [ "$is_root" = false ] ; then + USE_SUDO=sudo +else + USE_SUDO= +fi + +$USE_SUDO rm -r /var/lib/apt/lists/* +$USE_SUDO apt-get clean +$USE_SUDO apt-get update ## Install echo "Adding CUDA Repository" wget ${PIN_URL} -sudo mv ${PIN_FILENAME} /etc/apt/preferences.d/cuda-repository-pin-600 -sudo apt-key adv --fetch-keys ${APT_KEY_URL} -sudo add-apt-repository "deb ${REPO_URL} /" -sudo apt-get update +$USE_SUDO mv ${PIN_FILENAME} /etc/apt/preferences.d/cuda-repository-pin-600 +wget ${KEYRING_PACKAGE_URL} && ${USE_SUDO} dpkg -i ${KERYRING_PACKAGE_FILENAME} && rm ${KERYRING_PACKAGE_FILENAME} +$USE_SUDO add-apt-repository "deb ${REPO_URL} /" +$USE_SUDO apt-get update echo "Installing CUDA packages ${CUDA_PACKAGES}" -sudo apt-get -y install ${CUDA_PACKAGES} +$USE_SUDO apt-get -y install ${CUDA_PACKAGES} if [[ $? -ne 0 ]]; then echo "CUDA Installation Error." @@ -109,7 +154,8 @@ fi CUDA_PATH=/usr/local/cuda-${CUDA_MAJOR}.${CUDA_MINOR} echo "CUDA_PATH=${CUDA_PATH}" export CUDA_PATH=${CUDA_PATH} - export PATH="$CUDA_PATH/bin:$PATH" export LD_LIBRARY_PATH="$CUDA_PATH/lib:$LD_LIBRARY_PATH" -nvcc -V \ No newline at end of file +export LD_LIBRARY_PATH="$CUDA_PATH/lib64:$LD_LIBRARY_PATH" +# Check nvcc is now available. +nvcc -V From 444fac7eb475a083d60c62a5c837fd8a0fbc09ce Mon Sep 17 00:00:00 2001 From: Kostya Date: Mon, 17 Mar 2025 14:07:58 +0300 Subject: [PATCH 16/69] ci: add source command for installation script startup --- .github/workflows/ubuntu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index fae441e..c305a0e 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -36,7 +36,7 @@ jobs: - name: Install CUDA env: cuda: ${{ matrix.cuda }} - run: ./scripts/install_cuda_ubuntu.sh + run: source ./scripts/install_cuda_ubuntu.sh shell: bash # - name: Install and configure GCC and GXX From 71729151bddfc038231b05267776c8110642a8f6 Mon Sep 17 00:00:00 2001 From: Kostya Date: Mon, 17 Mar 2025 17:49:38 +0300 Subject: [PATCH 17/69] fix(install_cuda): add correct url --- scripts/install_cuda_ubuntu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install_cuda_ubuntu.sh b/scripts/install_cuda_ubuntu.sh index 65ebe60..f5d95ba 100644 --- a/scripts/install_cuda_ubuntu.sh +++ b/scripts/install_cuda_ubuntu.sh @@ -102,7 +102,7 @@ CPU_ARCH="x86_64" PIN_FILENAME="cuda-ubuntu${UBUNTU_VERSION}.pin" PIN_URL="https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/${CPU_ARCH}/${PIN_FILENAME}" # apt keyring package now available https://developer.nvidia.com/blog/updating-the-cuda-linux-gpg-repository-key/ -KERYRING_PACKAGE_FILENAME="cuda-keyring_1.0-1_all.deb" +KERYRING_PACKAGE_FILENAME="cuda-keyring_1.1-1_all.deb" KEYRING_PACKAGE_URL="https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/${CPU_ARCH}/${KERYRING_PACKAGE_FILENAME}" REPO_URL="https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/${CPU_ARCH}/" From b95f94cab33aa17106c9c1825dd1c31cdba1760e Mon Sep 17 00:00:00 2001 From: Kostya Date: Mon, 17 Mar 2025 18:37:59 +0300 Subject: [PATCH 18/69] ci: add gcc and g++ installation --- .github/workflows/ubuntu.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index c305a0e..500b2d1 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -19,7 +19,7 @@ jobs: include: - os: ubuntu-24.04 cuda: "12.8" - gcc: 11 + gcc: 14.2 env: build_dir: "build" config: "Release" @@ -39,12 +39,12 @@ jobs: run: source ./scripts/install_cuda_ubuntu.sh shell: bash - # - name: Install and configure GCC and GXX - # run: | - # sudo apt-get install -y gcc-${{ matrix.gcc }} g++-${{ matrix.gcc }} - # echo "СС=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV - # echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV - # echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV + - name: Install and configure GCC and GXX + run: | + sudo apt-get install -y gcc-${{ matrix.gcc }} g++-${{ matrix.gcc }} + export СС=/usr/bin/gcc-${{ matrix.gcc }} + export CXX=/usr/bin/g++-${{ matrix.gcc }} + export CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }} # - name: Configure CMake build # run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=YES From 4eb72e7fdb5abab1e58ce39d0d38b3b1d9ab3fe9 Mon Sep 17 00:00:00 2001 From: Kostya Date: Mon, 17 Mar 2025 18:54:42 +0300 Subject: [PATCH 19/69] ci: add cmake config --- .github/workflows/ubuntu.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 500b2d1..6ada5f8 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -19,7 +19,7 @@ jobs: include: - os: ubuntu-24.04 cuda: "12.8" - gcc: 14.2 + gcc: 14 env: build_dir: "build" config: "Release" @@ -39,19 +39,19 @@ jobs: run: source ./scripts/install_cuda_ubuntu.sh shell: bash - - name: Install and configure GCC and GXX + # sudo apt-get install -y gcc-${{ matrix.gcc }} g++-${{ matrix.gcc }} + - name: Export gcc and g++ variables run: | - sudo apt-get install -y gcc-${{ matrix.gcc }} g++-${{ matrix.gcc }} export СС=/usr/bin/gcc-${{ matrix.gcc }} export CXX=/usr/bin/g++-${{ matrix.gcc }} export CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }} - # - name: Configure CMake build - # run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=YES + - name: Configure CMake build + run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=YES - # - name: Build library sources - # working-directory: ${{ env.build_dir }} - # run: cmake --build . --target all --verbose -j `nproc` + - name: Build library sources + working-directory: ${{ env.build_dir }} + run: cmake --build . --target all --verbose -j `nproc` # - name: Run unit-tests (sequential backend) # working-directory: ${{ env.build_dir }} From 93d3c08059d746998f9cf481c7a9498b5acdb78b Mon Sep 17 00:00:00 2001 From: Kostya Date: Mon, 17 Mar 2025 18:55:57 +0300 Subject: [PATCH 20/69] fix: add correct version of gcc --- .github/workflows/ubuntu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 6ada5f8..c26f975 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -19,7 +19,7 @@ jobs: include: - os: ubuntu-24.04 cuda: "12.8" - gcc: 14 + gcc: 13 env: build_dir: "build" config: "Release" From 6143a65e543b8b6373cf08f3076de372c5c240cc Mon Sep 17 00:00:00 2001 From: Kostya Date: Mon, 17 Mar 2025 19:07:10 +0300 Subject: [PATCH 21/69] fix: correct env variables export --- .github/workflows/ubuntu.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index c26f975..d577ca4 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -42,9 +42,9 @@ jobs: # sudo apt-get install -y gcc-${{ matrix.gcc }} g++-${{ matrix.gcc }} - name: Export gcc and g++ variables run: | - export СС=/usr/bin/gcc-${{ matrix.gcc }} - export CXX=/usr/bin/g++-${{ matrix.gcc }} - export CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }} + echo "СС=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV + echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV + echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV - name: Configure CMake build run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=YES From 7e57342bf144628e0d5d50ada078794d8c862b50 Mon Sep 17 00:00:00 2001 From: Kostya Date: Mon, 17 Mar 2025 20:02:00 +0300 Subject: [PATCH 22/69] fix: add cudacxx flag to cmake --- .github/workflows/ubuntu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index d577ca4..936c5d4 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -47,7 +47,7 @@ jobs: echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV - name: Configure CMake build - run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=YES + run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=YES -CUDACXX=/usr/local/cuda-${{ matrix.cuda }}/bin/nvcc - name: Build library sources working-directory: ${{ env.build_dir }} From 1bc181163760372cf692ee458f6e72336c0723f5 Mon Sep 17 00:00:00 2001 From: Kostya Date: Mon, 17 Mar 2025 20:13:25 +0300 Subject: [PATCH 23/69] fix: add cuda architecture flag in cmake --- .github/workflows/ubuntu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 936c5d4..7ba59c7 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -47,7 +47,7 @@ jobs: echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV - name: Configure CMake build - run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=YES -CUDACXX=/usr/local/cuda-${{ matrix.cuda }}/bin/nvcc + run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=YES -D CMAKE_CUDA_ARCHITECTURES=native - name: Build library sources working-directory: ${{ env.build_dir }} From db88a3f0a28687b0cd3825ea6fb4b022753e83a1 Mon Sep 17 00:00:00 2001 From: Kostya Date: Mon, 17 Mar 2025 20:25:14 +0300 Subject: [PATCH 24/69] fix: add cudacxx env variable --- .github/workflows/ubuntu.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 7ba59c7..6e683c1 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -45,6 +45,7 @@ jobs: echo "СС=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV + echo "CUDACXX=/usr/local/cuda-${{ matrix.cuda }}/bin/nvcc" >> $GITHUB_ENV - name: Configure CMake build run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=YES -D CMAKE_CUDA_ARCHITECTURES=native From 66803695d81ee930378f44a77badd3bd1b1f1fb9 Mon Sep 17 00:00:00 2001 From: Kostya Date: Thu, 20 Mar 2025 16:57:34 +0300 Subject: [PATCH 25/69] ci: delete cuda architecture flag --- .github/workflows/ubuntu.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 6e683c1..bf98193 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -28,7 +28,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - submodules: true + submodules: recursive # - uses: actions/setup-python@v5 # with: # python-version: '3.7' @@ -48,7 +48,7 @@ jobs: echo "CUDACXX=/usr/local/cuda-${{ matrix.cuda }}/bin/nvcc" >> $GITHUB_ENV - name: Configure CMake build - run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=YES -D CMAKE_CUDA_ARCHITECTURES=native + run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=YES - name: Build library sources working-directory: ${{ env.build_dir }} From 2cd33f1814533a32dc3c0597464a170d724249db Mon Sep 17 00:00:00 2001 From: Kostya Date: Thu, 20 Mar 2025 17:12:31 +0300 Subject: [PATCH 26/69] ci: add unit and regression tests --- .github/workflows/ubuntu.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index bf98193..8821339 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -54,22 +54,22 @@ jobs: working-directory: ${{ env.build_dir }} run: cmake --build . --target all --verbose -j `nproc` - # - name: Run unit-tests (sequential backend) - # working-directory: ${{ env.build_dir }} - # run: bash scripts/run_tests_fallback.sh - # shell: bash + - name: Run unit-tests (sequential backend) + working-directory: ${{ env.build_dir }} + run: bash scripts/run_tests_fallback.sh + shell: bash - # - name: Run regression-tests (sequential backend) - # working-directory: ${{ env.build_dir }}/python - # run: bash run_tests.sh - # shell: bash + - name: Run regression-tests (sequential backend) + working-directory: ${{ env.build_dir }}/python + run: bash run_tests.sh + shell: bash - # - name: Prepare upload binary - # shell: bash - # run: tar cfz ${{ env.artifact }} ${{ env.build_dir }} + - name: Prepare upload binary + shell: bash + run: tar cfz ${{ env.artifact }} ${{ env.build_dir }} - # - name: Upload binary - # uses: actions/upload-artifact@v4 - # with: - # name: ${{ env.artifact }} - # path: ${{ env.artifact }} \ No newline at end of file + - name: Upload binary + uses: actions/upload-artifact@v4 + with: + name: ${{ env.artifact }} + path: ${{ env.artifact }} From 86e50673122b46ebabb2a44a86a20383b4432fee Mon Sep 17 00:00:00 2001 From: Kostya Date: Thu, 20 Mar 2025 17:49:07 +0300 Subject: [PATCH 27/69] ci: change working directory --- .github/workflows/ubuntu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 8821339..3f79427 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -55,7 +55,7 @@ jobs: run: cmake --build . --target all --verbose -j `nproc` - name: Run unit-tests (sequential backend) - working-directory: ${{ env.build_dir }} + # working-directory: ${{ env.build_dir }} run: bash scripts/run_tests_fallback.sh shell: bash From ea7798014733d5c5c0776419cc085222ad1d4126 Mon Sep 17 00:00:00 2001 From: Kostya Date: Thu, 20 Mar 2025 17:56:42 +0300 Subject: [PATCH 28/69] ci: change the way of running scripts --- .github/workflows/ubuntu.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 3f79427..591c25e 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -55,8 +55,8 @@ jobs: run: cmake --build . --target all --verbose -j `nproc` - name: Run unit-tests (sequential backend) - # working-directory: ${{ env.build_dir }} - run: bash scripts/run_tests_fallback.sh + working-directory: ${{ env.build_dir }} + run: bash ../scripts/run_tests_fallback.sh shell: bash - name: Run regression-tests (sequential backend) From 300a36aba7660526c9e19a0a10d45e99d8e4647b Mon Sep 17 00:00:00 2001 From: Kostya Date: Thu, 20 Mar 2025 18:24:08 +0300 Subject: [PATCH 29/69] ci: add python working directory toregression tests --- .github/workflows/ubuntu.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 591c25e..42de5f0 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -60,8 +60,8 @@ jobs: shell: bash - name: Run regression-tests (sequential backend) - working-directory: ${{ env.build_dir }}/python - run: bash run_tests.sh + working-directory: python + run: bash ./run_tests.sh shell: bash - name: Prepare upload binary From 850ab4ac5cd0d2794ea80f0c06505ec28a8d7ba9 Mon Sep 17 00:00:00 2001 From: Kostya Date: Thu, 20 Mar 2025 20:45:38 +0300 Subject: [PATCH 30/69] ci: call tests from build directory --- .github/workflows/ubuntu.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 42de5f0..eb7a4f3 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -54,14 +54,14 @@ jobs: working-directory: ${{ env.build_dir }} run: cmake --build . --target all --verbose -j `nproc` - - name: Run unit-tests (sequential backend) + - name: Run unit-tests working-directory: ${{ env.build_dir }} - run: bash ../scripts/run_tests_fallback.sh + run: bash scripts/run_tests_fallback.sh shell: bash - - name: Run regression-tests (sequential backend) - working-directory: python - run: bash ./run_tests.sh + - name: Run regression-tests + working-directory: ${{ env.build_dir }}/python + run: bash run_tests.sh shell: bash - name: Prepare upload binary From 934aa5c6968a2768c2aa00cdcecc3b53eae7a8cd Mon Sep 17 00:00:00 2001 From: Kostya Date: Thu, 20 Mar 2025 21:00:34 +0300 Subject: [PATCH 31/69] ci: turn on the flag with tests --- .github/workflows/ubuntu.yml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index eb7a4f3..3a9eb42 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -48,7 +48,7 @@ jobs: echo "CUDACXX=/usr/local/cuda-${{ matrix.cuda }}/bin/nvcc" >> $GITHUB_ENV - name: Configure CMake build - run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=YES + run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=ON - name: Build library sources working-directory: ${{ env.build_dir }} diff --git a/README.md b/README.md index 160a767..e0bcb23 100644 --- a/README.md +++ b/README.md @@ -214,7 +214,7 @@ $ bash ./scripts/run_tests_all.sh By default, the following cmake options will be automatically enabled: - `CUBOOL_WITH_CUDA` - build library with actual cuda backend -- `CUBOOL_WITH_SEQUENTIAL` - build library witt cpu based backend +- `CUBOOL_WITH_SEQUENTIAL` - build library with cpu based backend - `CUBOOL_WITH_TESTS` - build library unit-tests collection > Note: in order to provide correct GCC version for CUDA sources compiling, From 749cde1652dd2130711a712bcb42a5ec65c52f8c Mon Sep 17 00:00:00 2001 From: Kostya Date: Thu, 20 Mar 2025 21:13:05 +0300 Subject: [PATCH 32/69] ci: comment build stage --- .github/workflows/ubuntu.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 3a9eb42..3a0e1bc 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -28,16 +28,16 @@ jobs: steps: - uses: actions/checkout@v4 with: - submodules: recursive + submodules: true # - uses: actions/setup-python@v5 # with: # python-version: '3.7' - - name: Install CUDA - env: - cuda: ${{ matrix.cuda }} - run: source ./scripts/install_cuda_ubuntu.sh - shell: bash + # - name: Install CUDA + # env: + # cuda: ${{ matrix.cuda }} + # run: source ./scripts/install_cuda_ubuntu.sh + # shell: bash # sudo apt-get install -y gcc-${{ matrix.gcc }} g++-${{ matrix.gcc }} - name: Export gcc and g++ variables @@ -48,21 +48,21 @@ jobs: echo "CUDACXX=/usr/local/cuda-${{ matrix.cuda }}/bin/nvcc" >> $GITHUB_ENV - name: Configure CMake build - run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=ON + run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=YES - name: Build library sources working-directory: ${{ env.build_dir }} run: cmake --build . --target all --verbose -j `nproc` - - name: Run unit-tests - working-directory: ${{ env.build_dir }} - run: bash scripts/run_tests_fallback.sh - shell: bash + # - name: Run unit-tests + # working-directory: ${{ env.build_dir }} + # run: bash scripts/run_tests_fallback.sh + # shell: bash - - name: Run regression-tests - working-directory: ${{ env.build_dir }}/python - run: bash run_tests.sh - shell: bash + # - name: Run regression-tests + # working-directory: ${{ env.build_dir }}/python + # run: bash run_tests.sh + # shell: bash - name: Prepare upload binary shell: bash From 8130fed85b3d62730f80bae0bcdc3d0dd19741f6 Mon Sep 17 00:00:00 2001 From: Kostya Date: Thu, 20 Mar 2025 21:37:57 +0300 Subject: [PATCH 33/69] fix(cmake): uncomment test build --- .github/workflows/ubuntu.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 3a0e1bc..f54d5c4 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -28,7 +28,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - submodules: true + submodules: recursive # - uses: actions/setup-python@v5 # with: # python-version: '3.7' @@ -48,21 +48,21 @@ jobs: echo "CUDACXX=/usr/local/cuda-${{ matrix.cuda }}/bin/nvcc" >> $GITHUB_ENV - name: Configure CMake build - run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=YES + run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=ON - name: Build library sources working-directory: ${{ env.build_dir }} run: cmake --build . --target all --verbose -j `nproc` - # - name: Run unit-tests - # working-directory: ${{ env.build_dir }} - # run: bash scripts/run_tests_fallback.sh - # shell: bash + - name: Run unit-tests + working-directory: ${{ env.build_dir }} + run: bash scripts/run_tests_fallback.sh + shell: bash - # - name: Run regression-tests - # working-directory: ${{ env.build_dir }}/python - # run: bash run_tests.sh - # shell: bash + - name: Run regression-tests + working-directory: ${{ env.build_dir }}/python + run: bash run_tests.sh + shell: bash - name: Prepare upload binary shell: bash From ce82e3c26ab14c5ded0999c388e64882aa433aaa Mon Sep 17 00:00:00 2001 From: Kostya Date: Thu, 10 Apr 2025 19:36:43 +0300 Subject: [PATCH 34/69] ci: add job with test in pipeline --- .github/workflows/ubuntu.yml | 77 ++++++++++++++++++++++++------------ 1 file changed, 51 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index f54d5c4..bfd12c0 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -9,10 +9,15 @@ on: branches: [ master ] workflow_dispatch: +env: + build_dir: "build" + config: "Release" + artifact: "cubool-ubuntu-build.tar.xz" + jobs: build: name: Build ${{ matrix.os }} GCC ${{ matrix.gcc }} CUDA ${{ matrix.cuda }} - runs-on: self-hosted + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: @@ -20,50 +25,37 @@ jobs: - os: ubuntu-24.04 cuda: "12.8" gcc: 13 - env: - build_dir: "build" - config: "Release" - artifact: "cubool-ubuntu-build.tar.xz" steps: - uses: actions/checkout@v4 with: submodules: recursive - # - uses: actions/setup-python@v5 - # with: - # python-version: '3.7' - # - name: Install CUDA - # env: - # cuda: ${{ matrix.cuda }} - # run: source ./scripts/install_cuda_ubuntu.sh - # shell: bash + - uses: actions/setup-python@v5 + with: + python-version: '3.7' + + - name: Install CUDA + env: + cuda: ${{ matrix.cuda }} + run: source ./scripts/install_cuda_ubuntu.sh + shell: bash - # sudo apt-get install -y gcc-${{ matrix.gcc }} g++-${{ matrix.gcc }} - name: Export gcc and g++ variables run: | + sudo apt-get install -y gcc-${{ matrix.gcc }} g++-${{ matrix.gcc }} echo "СС=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV - echo "CUDACXX=/usr/local/cuda-${{ matrix.cuda }}/bin/nvcc" >> $GITHUB_ENV +# echo "CUDACXX=/usr/local/cuda-${{ matrix.cuda }}/bin/nvcc" >> $GITHUB_ENV - name: Configure CMake build run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=ON - - name: Build library sources + - name: Build librarnamey sources working-directory: ${{ env.build_dir }} run: cmake --build . --target all --verbose -j `nproc` - - name: Run unit-tests - working-directory: ${{ env.build_dir }} - run: bash scripts/run_tests_fallback.sh - shell: bash - - - name: Run regression-tests - working-directory: ${{ env.build_dir }}/python - run: bash run_tests.sh - shell: bash - - name: Prepare upload binary shell: bash run: tar cfz ${{ env.artifact }} ${{ env.build_dir }} @@ -73,3 +65,36 @@ jobs: with: name: ${{ env.artifact }} path: ${{ env.artifact }} + + + test: + name: Test ${{ matrix.os }} GCC ${{ matrix.gcc }} CUDA ${{ matrix.cuda }} + runs-on: self-hosted + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-24.04 + cuda: "12.8" + gcc: 13 + + steps: + - name: Download binary artifact + uses: actions/download-artifact@v4 + with: + name: ${{ env.artifact }} + + - name: Unarchive artifact + run: | + tar xzf ${{ env.artifact }} + rm ${{ env.artifact }} + + - name: Run unit-tests + working-directory: ${{ env.build_dir }} + run: bash scripts/run_tests_fallback.sh + shell: bash + + - name: Run regression-tests + working-directory: ${{ env.build_dir }}/python + run: bash run_tests.sh + shell: bash From 76cebaa542499704cebe79059b213d1a7fb0adea Mon Sep 17 00:00:00 2001 From: Kostya Date: Thu, 10 Apr 2025 19:40:53 +0300 Subject: [PATCH 35/69] fix: change python version --- .github/workflows/ubuntu.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index bfd12c0..2377c1f 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -33,7 +33,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: '3.7' + python-version: '3.12' - name: Install CUDA env: @@ -69,6 +69,7 @@ jobs: test: name: Test ${{ matrix.os }} GCC ${{ matrix.gcc }} CUDA ${{ matrix.cuda }} + needs: build runs-on: self-hosted strategy: fail-fast: false From d7c00887e83e834f3d2ee9f487b672477a21b213 Mon Sep 17 00:00:00 2001 From: Kostya Date: Thu, 10 Apr 2025 19:44:30 +0300 Subject: [PATCH 36/69] ci: add cudacxx env variable --- .github/workflows/ubuntu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 2377c1f..5c2a558 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -47,7 +47,7 @@ jobs: echo "СС=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV -# echo "CUDACXX=/usr/local/cuda-${{ matrix.cuda }}/bin/nvcc" >> $GITHUB_ENV + echo "CUDACXX=/usr/local/cuda-${{ matrix.cuda }}/bin/nvcc" >> $GITHUB_ENV - name: Configure CMake build run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=ON From aa64a88a1673e45dab962330855a759502667742 Mon Sep 17 00:00:00 2001 From: Kostya Date: Mon, 21 Apr 2025 12:42:08 +0300 Subject: [PATCH 37/69] ci: add separate pipeline for self-hosted --- .github/workflows/self-hosted.yml | 79 +++++++++++++++++++++++++++++++ .github/workflows/ubuntu.yml | 65 +++++++------------------ 2 files changed, 97 insertions(+), 47 deletions(-) create mode 100644 .github/workflows/self-hosted.yml diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml new file mode 100644 index 0000000..6651d6d --- /dev/null +++ b/.github/workflows/self-hosted.yml @@ -0,0 +1,79 @@ +# Original script from https://github.com/ptheywood/cuda-cmake-github-actions + +name: Ubuntu + +on: + push: + branches: [ master, self-hosted-runner ] + pull_request: + branches: [ master ] + workflow_dispatch: + +env: + build_dir: "build" + config: "Release" + +jobs: + build: + name: Build ${{ matrix.self-hosted }} GCC ${{ matrix.gcc }} CUDA ${{ matrix.cuda }} + runs-on: ${{ matrix.self-hosted }} + strategy: + fail-fast: false + matrix: + include: + - self-hosted: ubuntu-24.04 + cuda: "12.8" + gcc: 13 + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install CUDA + env: + cuda: ${{ matrix.cuda }} + run: source ./scripts/install_cuda_ubuntu.sh + shell: bash + + - name: Export gcc and g++ variables + run: | + sudo apt-get install -y gcc-${{ matrix.gcc }} g++-${{ matrix.gcc }} + echo "СС=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV + echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV + echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV + echo "CUDACXX=/usr/local/cuda-${{ matrix.cuda }}/bin/nvcc" >> $GITHUB_ENV + + - name: Configure CMake build + run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=ON + + - name: Build library sources + working-directory: ${{ env.build_dir }} + run: cmake --build . --target all --verbose -j `nproc` + + test: + name: Test ${{ matrix.self-hosted }} GCC ${{ matrix.gcc }} CUDA ${{ matrix.cuda }} + needs: build + runs-on: ${{ matrix.self-hosted }} + strategy: + fail-fast: false + matrix: + include: + - self-hosted: ubuntu-24.04 + cuda: "12.8" + gcc: 13 + + steps: + - name: Run unit-tests + working-directory: ${{ env.build_dir }} + run: bash scripts/run_tests_fallback.sh + shell: bash + + - name: Run regression-tests + working-directory: ${{ env.build_dir }}/python + run: bash run_tests.sh + shell: bash diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 5c2a558..7bd0334 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -9,11 +9,6 @@ on: branches: [ master ] workflow_dispatch: -env: - build_dir: "build" - config: "Release" - artifact: "cubool-ubuntu-build.tar.xz" - jobs: build: name: Build ${{ matrix.os }} GCC ${{ matrix.gcc }} CUDA ${{ matrix.cuda }} @@ -22,9 +17,13 @@ jobs: fail-fast: false matrix: include: - - os: ubuntu-24.04 - cuda: "12.8" - gcc: 13 + - os: [ubuntu-18.04, ubuntu-24.04] + cuda: ["10.1", "12.8"] + gcc: [8, 13] + env: + build_dir: "build" + config: "Release" + artifact: "cubool-${{ matrix.os }}-build.tar.xz" steps: - uses: actions/checkout@v4 @@ -36,12 +35,18 @@ jobs: python-version: '3.12' - name: Install CUDA - env: - cuda: ${{ matrix.cuda }} - run: source ./scripts/install_cuda_ubuntu.sh + run: | + source ./scripts/install_cuda_ubuntu.sh + if [[ $? -eq 0 ]]; then + # Set paths for subsequent steps, using ${CUDA_PATH} + echo "Adding CUDA to CUDA_PATH, PATH and LD_LIBRARY_PATH" + echo "CUDA_PATH=${CUDA_PATH}" >> $GITHUB_ENV + echo "${CUDA_PATH}/bin" >> $GITHUB_PATH + echo "LD_LIBRARY_PATH=${CUDA_PATH}/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV + fi shell: bash - - name: Export gcc and g++ variables + - name: Install and configure GCC and GXX run: | sudo apt-get install -y gcc-${{ matrix.gcc }} g++-${{ matrix.gcc }} echo "СС=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV @@ -52,7 +57,7 @@ jobs: - name: Configure CMake build run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=ON - - name: Build librarnamey sources + - name: Build library sources working-directory: ${{ env.build_dir }} run: cmake --build . --target all --verbose -j `nproc` @@ -65,37 +70,3 @@ jobs: with: name: ${{ env.artifact }} path: ${{ env.artifact }} - - - test: - name: Test ${{ matrix.os }} GCC ${{ matrix.gcc }} CUDA ${{ matrix.cuda }} - needs: build - runs-on: self-hosted - strategy: - fail-fast: false - matrix: - include: - - os: ubuntu-24.04 - cuda: "12.8" - gcc: 13 - - steps: - - name: Download binary artifact - uses: actions/download-artifact@v4 - with: - name: ${{ env.artifact }} - - - name: Unarchive artifact - run: | - tar xzf ${{ env.artifact }} - rm ${{ env.artifact }} - - - name: Run unit-tests - working-directory: ${{ env.build_dir }} - run: bash scripts/run_tests_fallback.sh - shell: bash - - - name: Run regression-tests - working-directory: ${{ env.build_dir }}/python - run: bash run_tests.sh - shell: bash From b4167d92727384064cad0b5e2803e2d901af7599 Mon Sep 17 00:00:00 2001 From: Kostya Date: Mon, 21 Apr 2025 13:33:46 +0300 Subject: [PATCH 38/69] ci: add correct platform --- .github/workflows/self-hosted.yml | 14 +++++++------- .github/workflows/ubuntu.yml | 16 ++++++++-------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index 6651d6d..21b5dc5 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -1,6 +1,6 @@ # Original script from https://github.com/ptheywood/cuda-cmake-github-actions -name: Ubuntu +name: Self-Hosted on: push: @@ -15,13 +15,13 @@ env: jobs: build: - name: Build ${{ matrix.self-hosted }} GCC ${{ matrix.gcc }} CUDA ${{ matrix.cuda }} - runs-on: ${{ matrix.self-hosted }} + name: Build ${{ matrix.os }} GCC ${{ matrix.gcc }} CUDA ${{ matrix.cuda }} + runs-on: self-hosted strategy: fail-fast: false matrix: include: - - self-hosted: ubuntu-24.04 + - os: ubuntu-24.04 cuda: "12.8" gcc: 13 @@ -56,14 +56,14 @@ jobs: run: cmake --build . --target all --verbose -j `nproc` test: - name: Test ${{ matrix.self-hosted }} GCC ${{ matrix.gcc }} CUDA ${{ matrix.cuda }} + name: Test ${{ matrix.os }} GCC ${{ matrix.gcc }} CUDA ${{ matrix.cuda }} needs: build - runs-on: ${{ matrix.self-hosted }} + runs-on: self-hosted strategy: fail-fast: false matrix: include: - - self-hosted: ubuntu-24.04 + - os: ubuntu-24.04 cuda: "12.8" gcc: 13 diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 7bd0334..a9a6932 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -4,7 +4,7 @@ name: Ubuntu on: push: - branches: [ master, self-hosted-runner ] + branches: [ master ] pull_request: branches: [ master ] workflow_dispatch: @@ -17,24 +17,25 @@ jobs: fail-fast: false matrix: include: - - os: [ubuntu-18.04, ubuntu-24.04] - cuda: ["10.1", "12.8"] - gcc: [8, 13] + - os: ubuntu-18.04 + cuda: "10.1" + gcc: 8 env: build_dir: "build" config: "Release" - artifact: "cubool-${{ matrix.os }}-build.tar.xz" + artifact: "cubool-ubuntu-build.tar.xz" steps: - uses: actions/checkout@v4 with: - submodules: recursive - + submodules: true - uses: actions/setup-python@v5 with: python-version: '3.12' - name: Install CUDA + env: + cuda: ${{ matrix.cuda }} run: | source ./scripts/install_cuda_ubuntu.sh if [[ $? -eq 0 ]]; then @@ -52,7 +53,6 @@ jobs: echo "СС=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV - echo "CUDACXX=/usr/local/cuda-${{ matrix.cuda }}/bin/nvcc" >> $GITHUB_ENV - name: Configure CMake build run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=ON From 071309e57dc80ba5e597957c83640af5ecc4d666 Mon Sep 17 00:00:00 2001 From: Kostya Date: Mon, 21 Apr 2025 22:47:32 +0300 Subject: [PATCH 39/69] ci: add environment variables in test stage --- .github/workflows/self-hosted.yml | 16 ++++------------ .github/workflows/ubuntu.yml | 4 +--- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index 21b5dc5..12f7876 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -30,19 +30,11 @@ jobs: with: submodules: recursive - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Install CUDA - env: - cuda: ${{ matrix.cuda }} - run: source ./scripts/install_cuda_ubuntu.sh - shell: bash - - - name: Export gcc and g++ variables + - name: Set environment variables run: | - sudo apt-get install -y gcc-${{ matrix.gcc }} g++-${{ matrix.gcc }} + echo "CUDA_PATH=/usr/local/cuda-12.8" >> $GITHUB_ENV + echo "${CUDA_PATH}/bin" >> $GITHUB_PATH + echo "LD_LIBRARY_PATH=${CUDA_PATH}/lib64:${LD_LIBRARY_PATH} >> $GITHUB_ENV echo "СС=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index a9a6932..cadc199 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -4,7 +4,7 @@ name: Ubuntu on: push: - branches: [ master ] + branches: [ master, self-hosted-runner ] pull_request: branches: [ master ] workflow_dispatch: @@ -34,8 +34,6 @@ jobs: python-version: '3.12' - name: Install CUDA - env: - cuda: ${{ matrix.cuda }} run: | source ./scripts/install_cuda_ubuntu.sh if [[ $? -eq 0 ]]; then From fc4296deb8ccba182ec894ab4b0453b382110497 Mon Sep 17 00:00:00 2001 From: Kostya Date: Mon, 21 Apr 2025 23:09:38 +0300 Subject: [PATCH 40/69] fix: delete extra quotes --- .github/workflows/self-hosted.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index 12f7876..cd60f9e 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -34,7 +34,7 @@ jobs: run: | echo "CUDA_PATH=/usr/local/cuda-12.8" >> $GITHUB_ENV echo "${CUDA_PATH}/bin" >> $GITHUB_PATH - echo "LD_LIBRARY_PATH=${CUDA_PATH}/lib64:${LD_LIBRARY_PATH} >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${CUDA_PATH}/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV echo "СС=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV From f98ea6c8fdff23f732e38e0dddf4e78b6dfb1c72 Mon Sep 17 00:00:00 2001 From: Kostya Date: Mon, 21 Apr 2025 23:41:22 +0300 Subject: [PATCH 41/69] ci: add error status of pipeline --- .github/workflows/self-hosted.yml | 6 ++++++ .github/workflows/ubuntu.yml | 8 ++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index cd60f9e..0ec6df7 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -61,11 +61,17 @@ jobs: steps: - name: Run unit-tests + id: unit-test working-directory: ${{ env.build_dir }} run: bash scripts/run_tests_fallback.sh shell: bash - name: Run regression-tests + id: regression-test working-directory: ${{ env.build_dir }}/python run: bash run_tests.sh shell: bash + + - name: Return test code + if: ${{ steps.unit-test.outcome == 'failure' || steps.regression-test.outcome == 'failure' }} + run: exit 1 diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index cadc199..d2c25b9 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -4,7 +4,7 @@ name: Ubuntu on: push: - branches: [ master, self-hosted-runner ] + branches: [ master, self-hosted-runnernner ] pull_request: branches: [ master ] workflow_dispatch: @@ -17,9 +17,9 @@ jobs: fail-fast: false matrix: include: - - os: ubuntu-18.04 - cuda: "10.1" - gcc: 8 + - os: ubuntu-24.04 + cuda: "12.8" + gcc: 13 env: build_dir: "build" config: "Release" From 374e060804d690623dbc555721c394ba10193a34 Mon Sep 17 00:00:00 2001 From: Kostya Date: Wed, 23 Apr 2025 17:55:56 +0300 Subject: [PATCH 42/69] ci: add test script start up --- .github/workflows/self-hosted.yml | 14 +++----------- .github/workflows/ubuntu.yml | 2 +- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index 0ec6df7..c3b1ee8 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -61,17 +61,9 @@ jobs: steps: - name: Run unit-tests - id: unit-test - working-directory: ${{ env.build_dir }} - run: bash scripts/run_tests_fallback.sh - shell: bash + working-directory: ${{ env.build_dir }}/scripts + run: chmod +x scripts/run_tests_fallback.sh && ./scripts/run_tests_fallback.sh - name: Run regression-tests - id: regression-test working-directory: ${{ env.build_dir }}/python - run: bash run_tests.sh - shell: bash - - - name: Return test code - if: ${{ steps.unit-test.outcome == 'failure' || steps.regression-test.outcome == 'failure' }} - run: exit 1 + run: chmod +x run_tests.sh && ./run_tests.sh diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index d2c25b9..5766fdd 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -4,7 +4,7 @@ name: Ubuntu on: push: - branches: [ master, self-hosted-runnernner ] + branches: [ master, self-hosted-runner ] pull_request: branches: [ master ] workflow_dispatch: From 176bf18f1a176dabc093d933e068294371e5bfd1 Mon Sep 17 00:00:00 2001 From: Kostya Date: Wed, 23 Apr 2025 18:40:08 +0300 Subject: [PATCH 43/69] fix: change working directory --- .github/workflows/self-hosted.yml | 2 +- .github/workflows/ubuntu.yml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index c3b1ee8..fc4e756 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -61,7 +61,7 @@ jobs: steps: - name: Run unit-tests - working-directory: ${{ env.build_dir }}/scripts + working-directory: ${{ env.build_dir }} run: chmod +x scripts/run_tests_fallback.sh && ./scripts/run_tests_fallback.sh - name: Run regression-tests diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 5766fdd..9c622d6 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -29,11 +29,14 @@ jobs: - uses: actions/checkout@v4 with: submodules: true + - uses: actions/setup-python@v5 with: python-version: '3.12' - name: Install CUDA + env: + cuda: ${{ matrix.cuda }} run: | source ./scripts/install_cuda_ubuntu.sh if [[ $? -eq 0 ]]; then From 1660aabffe1e64ca30d7c763f5e99a8abb912a0f Mon Sep 17 00:00:00 2001 From: Kostya Date: Thu, 24 Apr 2025 12:50:01 +0300 Subject: [PATCH 44/69] ci: change way of test start up --- .github/workflows/self-hosted.yml | 8 ++++++-- .github/workflows/ubuntu.yml | 6 +++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index fc4e756..5539e15 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -62,8 +62,12 @@ jobs: steps: - name: Run unit-tests working-directory: ${{ env.build_dir }} - run: chmod +x scripts/run_tests_fallback.sh && ./scripts/run_tests_fallback.sh + run: chmod +x scripts/run_tests_all.sh && ./scripts/run_tests_all.sh - name: Run regression-tests working-directory: ${{ env.build_dir }}/python - run: chmod +x run_tests.sh && ./run_tests.sh + run: | + echo "PYTHONPATH=`pwd`:$PYTHONPATH" >> $GITHUB_ENV + cd tests + python3 -m unittest discover -v + # chmod +x run_tests.sh && ./run_tests.sh diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 9c622d6..92ff42e 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -17,9 +17,9 @@ jobs: fail-fast: false matrix: include: - - os: ubuntu-24.04 - cuda: "12.8" - gcc: 13 + - os: ubuntu-20.04 + cuda: "10.1" + gcc: 8 env: build_dir: "build" config: "Release" From 1e156a8fce2c00f49bc542cd0fd800a8eec53b33 Mon Sep 17 00:00:00 2001 From: Kostya Date: Thu, 24 Apr 2025 20:57:43 +0300 Subject: [PATCH 45/69] ci: change tests run --- .github/workflows/self-hosted.yml | 2 +- .github/workflows/ubuntu.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index 5539e15..642a8e8 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -62,7 +62,7 @@ jobs: steps: - name: Run unit-tests working-directory: ${{ env.build_dir }} - run: chmod +x scripts/run_tests_all.sh && ./scripts/run_tests_all.sh + run: chmod +x scripts/run_tests_fallback.sh && ./scripts/run_tests_fallback.sh - name: Run regression-tests working-directory: ${{ env.build_dir }}/python diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 92ff42e..55302b8 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -17,7 +17,7 @@ jobs: fail-fast: false matrix: include: - - os: ubuntu-20.04 + - os: ubuntu-22.04 cuda: "10.1" gcc: 8 env: From 0101c7e8f087e86e2cb9ac9a7c593b7c857698de Mon Sep 17 00:00:00 2001 From: Kostya Date: Thu, 24 Apr 2025 22:39:25 +0300 Subject: [PATCH 46/69] ci: add check for failed tests --- .github/workflows/self-hosted.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index 642a8e8..1c25c5d 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -46,7 +46,7 @@ jobs: - name: Build library sources working-directory: ${{ env.build_dir }} run: cmake --build . --target all --verbose -j `nproc` - + test: name: Test ${{ matrix.os }} GCC ${{ matrix.gcc }} CUDA ${{ matrix.cuda }} needs: build @@ -62,12 +62,18 @@ jobs: steps: - name: Run unit-tests working-directory: ${{ env.build_dir }} - run: chmod +x scripts/run_tests_fallback.sh && ./scripts/run_tests_fallback.sh + run: | + output=$(chmod +x scripts/run_tests_fallback.sh && ./scripts/run_tests_fallback.sh) + echo $output + if echo "$output" | grep -qE "FAILED|ERROR"; then + exit 1 + fi - name: Run regression-tests working-directory: ${{ env.build_dir }}/python run: | - echo "PYTHONPATH=`pwd`:$PYTHONPATH" >> $GITHUB_ENV - cd tests - python3 -m unittest discover -v - # chmod +x run_tests.sh && ./run_tests.sh + output=$(chmod +x run_tests.sh && ./run_tests.sh) + echo $output + if echo "$output" | grep -qE "FAILED|ERROR"; then + exit 1 + fi From e0ce1cb9bb2f7f2ab09c2eea352b8077f77ff29f Mon Sep 17 00:00:00 2001 From: Kostya Date: Thu, 24 Apr 2025 23:11:15 +0300 Subject: [PATCH 47/69] ci: add newlines in output --- .github/workflows/self-hosted.yml | 12 +++++------- .github/workflows/ubuntu.yml | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index 1c25c5d..1386d73 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -62,18 +62,16 @@ jobs: steps: - name: Run unit-tests working-directory: ${{ env.build_dir }} - run: | - output=$(chmod +x scripts/run_tests_fallback.sh && ./scripts/run_tests_fallback.sh) - echo $output - if echo "$output" | grep -qE "FAILED|ERROR"; then + run: + chmod +x scripts/run_tests_fallback.sh + if ./scripts/run_tests_fallback.sh | tee /dev/tty | grep -qE "FAILED|ERROR"; then exit 1 fi - name: Run regression-tests working-directory: ${{ env.build_dir }}/python run: | - output=$(chmod +x run_tests.sh && ./run_tests.sh) - echo $output - if echo "$output" | grep -qE "FAILED|ERROR"; then + chmod +x run_tests.sh + if ./run_tests.sh | tee /dev/tty | grep -qE "FAILED|ERROR"; then exit 1 fi diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 55302b8..4bbc9ca 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -18,7 +18,7 @@ jobs: matrix: include: - os: ubuntu-22.04 - cuda: "10.1" + cuda: "11.7" gcc: 8 env: build_dir: "build" From e3416cc6db2f9bbaaad7cf43ae243d1804ba8853 Mon Sep 17 00:00:00 2001 From: Kostya Date: Thu, 24 Apr 2025 23:19:37 +0300 Subject: [PATCH 48/69] fix: add minimal version of gcc in ubuntu --- .github/workflows/self-hosted.yml | 2 +- .github/workflows/ubuntu.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index 1386d73..e1ea7c6 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -62,7 +62,7 @@ jobs: steps: - name: Run unit-tests working-directory: ${{ env.build_dir }} - run: + run: | chmod +x scripts/run_tests_fallback.sh if ./scripts/run_tests_fallback.sh | tee /dev/tty | grep -qE "FAILED|ERROR"; then exit 1 diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 4bbc9ca..6eba06f 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -18,8 +18,8 @@ jobs: matrix: include: - os: ubuntu-22.04 - cuda: "11.7" - gcc: 8 + cuda: "11.6" + gcc: 11 env: build_dir: "build" config: "Release" From faf132daea6cd0b2ba07558557668903c3f512ff Mon Sep 17 00:00:00 2001 From: Kostya Date: Fri, 25 Apr 2025 01:32:10 +0300 Subject: [PATCH 49/69] fix: change tty --- .github/workflows/self-hosted.yml | 6 ++++-- .github/workflows/ubuntu.yml | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index e1ea7c6..babc75e 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -64,7 +64,8 @@ jobs: working-directory: ${{ env.build_dir }} run: | chmod +x scripts/run_tests_fallback.sh - if ./scripts/run_tests_fallback.sh | tee /dev/tty | grep -qE "FAILED|ERROR"; then + tty=$(readlink /proc/$$/fd/2) + if ./scripts/run_tests_fallback.sh | tee $tty | grep -qE "FAILED|ERROR"; then exit 1 fi @@ -72,6 +73,7 @@ jobs: working-directory: ${{ env.build_dir }}/python run: | chmod +x run_tests.sh - if ./run_tests.sh | tee /dev/tty | grep -qE "FAILED|ERROR"; then + tty=$(readlink /proc/$$/fd/2) + if ./run_tests.sh | tee $tty | grep -qE "FAILED|ERROR"; then exit 1 fi diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 6eba06f..8d92764 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -18,7 +18,7 @@ jobs: matrix: include: - os: ubuntu-22.04 - cuda: "11.6" + cuda: "11.7" gcc: 11 env: build_dir: "build" From fbab4039c03c65df4da09f8f7ba68e8418838d64 Mon Sep 17 00:00:00 2001 From: Kostya Date: Fri, 25 Apr 2025 02:11:29 +0300 Subject: [PATCH 50/69] ci: use tee to generate pipeline --- .github/workflows/self-hosted.yml | 8 ++++---- .github/workflows/ubuntu.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index babc75e..0b96bc7 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -64,8 +64,8 @@ jobs: working-directory: ${{ env.build_dir }} run: | chmod +x scripts/run_tests_fallback.sh - tty=$(readlink /proc/$$/fd/2) - if ./scripts/run_tests_fallback.sh | tee $tty | grep -qE "FAILED|ERROR"; then + ./scripts/run_tests_fallback.sh | tee >(grep -E "FAILED|ERROR" && echo "HAS_ERRORS=true" >> $GITHUB_ENV) + if [ $HAS_ERRORS == true ]; then exit 1 fi @@ -73,7 +73,7 @@ jobs: working-directory: ${{ env.build_dir }}/python run: | chmod +x run_tests.sh - tty=$(readlink /proc/$$/fd/2) - if ./run_tests.sh | tee $tty | grep -qE "FAILED|ERROR"; then + ./run_tests.sh | tee >(grep -E "FAILED|ERROR" && echo "HAS_ERRORS=true" >> $GITHUB_ENV) + if [ $HAS_ERRORS == true ]; then exit 1 fi diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 8d92764..d590eab 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -19,7 +19,7 @@ jobs: include: - os: ubuntu-22.04 cuda: "11.7" - gcc: 11 + gcc: 10 env: build_dir: "build" config: "Release" From 7431fb53990c82e31c864030eada1afcd0ac5a75 Mon Sep 17 00:00:00 2001 From: Kostya Date: Fri, 25 Apr 2025 02:37:44 +0300 Subject: [PATCH 51/69] ci: add double brackets --- .github/workflows/self-hosted.yml | 8 ++++---- .github/workflows/ubuntu.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index 0b96bc7..e46eb5a 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -64,8 +64,8 @@ jobs: working-directory: ${{ env.build_dir }} run: | chmod +x scripts/run_tests_fallback.sh - ./scripts/run_tests_fallback.sh | tee >(grep -E "FAILED|ERROR" && echo "HAS_ERRORS=true" >> $GITHUB_ENV) - if [ $HAS_ERRORS == true ]; then + ./scripts/run_tests_fallback.sh | tee >(grep -qE "FAILED|ERROR" && echo "HAS_ERRORS=true" >> $GITHUB_ENV) + if [[ $HAS_ERRORS == "true" ]]; then exit 1 fi @@ -73,7 +73,7 @@ jobs: working-directory: ${{ env.build_dir }}/python run: | chmod +x run_tests.sh - ./run_tests.sh | tee >(grep -E "FAILED|ERROR" && echo "HAS_ERRORS=true" >> $GITHUB_ENV) - if [ $HAS_ERRORS == true ]; then + ./run_tests.sh | tee >(grep -qE "FAILED|ERROR" && echo "HAS_ERRORS=true" >> $GITHUB_ENV) + if [[ $HAS_ERRORS == "true" ]]; then exit 1 fi diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index d590eab..54871c9 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -19,7 +19,7 @@ jobs: include: - os: ubuntu-22.04 cuda: "11.7" - gcc: 10 + gcc: 9 env: build_dir: "build" config: "Release" From d80229c566bacd57827d712f6bef2d51e0e17858 Mon Sep 17 00:00:00 2001 From: Kostya Date: Fri, 25 Apr 2025 03:05:12 +0300 Subject: [PATCH 52/69] test: add test print --- .github/workflows/self-hosted.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index e46eb5a..58f423a 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -66,6 +66,7 @@ jobs: chmod +x scripts/run_tests_fallback.sh ./scripts/run_tests_fallback.sh | tee >(grep -qE "FAILED|ERROR" && echo "HAS_ERRORS=true" >> $GITHUB_ENV) if [[ $HAS_ERRORS == "true" ]]; then + echo Test exit 1 fi From 78a1ab78e392b7953ec2ce35a733ed252be07ad5 Mon Sep 17 00:00:00 2001 From: Kostya Date: Fri, 25 Apr 2025 03:26:04 +0300 Subject: [PATCH 53/69] ci: delete unnecessary print --- .github/workflows/self-hosted.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index 58f423a..e46eb5a 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -66,7 +66,6 @@ jobs: chmod +x scripts/run_tests_fallback.sh ./scripts/run_tests_fallback.sh | tee >(grep -qE "FAILED|ERROR" && echo "HAS_ERRORS=true" >> $GITHUB_ENV) if [[ $HAS_ERRORS == "true" ]]; then - echo Test exit 1 fi From 2320e7813588c7cb12bc9bd0ac1dbb8a843c48ec Mon Sep 17 00:00:00 2001 From: Kostya Date: Fri, 25 Apr 2025 13:12:58 +0300 Subject: [PATCH 54/69] ci: add tests result artifact --- .github/workflows/self-hosted.yml | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index e46eb5a..a3eea7a 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -51,6 +51,9 @@ jobs: name: Test ${{ matrix.os }} GCC ${{ matrix.gcc }} CUDA ${{ matrix.cuda }} needs: build runs-on: self-hosted + env: + unit-test-file: test_fallback.txt + regression-test-file: test_regression.txt strategy: fail-fast: false matrix: @@ -64,16 +67,28 @@ jobs: working-directory: ${{ env.build_dir }} run: | chmod +x scripts/run_tests_fallback.sh - ./scripts/run_tests_fallback.sh | tee >(grep -qE "FAILED|ERROR" && echo "HAS_ERRORS=true" >> $GITHUB_ENV) - if [[ $HAS_ERRORS == "true" ]]; then + ./scripts/run_tests_fallback.sh | tee ${{ env.unit-test-file }} + if cat ${{ env.unit-test-file }} | grep -q "FAILED"; then exit 1 fi + - name: Upload unit tests resutls + uses: actions/upload-artifact@v4 + with: + name: ${{ env.unit-test-file }} + path: ${{ env.build_dir }}/${{ env.unit-test-file }} + - name: Run regression-tests working-directory: ${{ env.build_dir }}/python run: | chmod +x run_tests.sh - ./run_tests.sh | tee >(grep -qE "FAILED|ERROR" && echo "HAS_ERRORS=true" >> $GITHUB_ENV) - if [[ $HAS_ERRORS == "true" ]]; then + ./run_tests.sh | tee ${{ env.regression-test-file }} + if cat ${{ env.regression-test-file }} | grep -q "FAILED"; then exit 1 fi + + - name: Upload regression tests resutls + uses: actions/upload-artifact@v4 + with: + name: ${{ env.regression-test-file }} + path: ${{ env.build_dir }}/python/${{ env.regression-test-file }} From 7d9a039224c39098501f450d2b2a0bbf43fbc0e8 Mon Sep 17 00:00:00 2001 From: Kostya Date: Fri, 25 Apr 2025 13:28:24 +0300 Subject: [PATCH 55/69] ci: add check for result as separate step --- .github/workflows/self-hosted.yml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index a3eea7a..ad64773 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -68,27 +68,35 @@ jobs: run: | chmod +x scripts/run_tests_fallback.sh ./scripts/run_tests_fallback.sh | tee ${{ env.unit-test-file }} - if cat ${{ env.unit-test-file }} | grep -q "FAILED"; then - exit 1 - fi - name: Upload unit tests resutls uses: actions/upload-artifact@v4 with: name: ${{ env.unit-test-file }} path: ${{ env.build_dir }}/${{ env.unit-test-file }} + + - name: Check for unit tests results + working-directory: ${{ env.build_dir }} + run: | + if cat ${{ env.unit-test-file }} | grep -q "FAILED"; then + exit 1 + fi - name: Run regression-tests working-directory: ${{ env.build_dir }}/python run: | chmod +x run_tests.sh ./run_tests.sh | tee ${{ env.regression-test-file }} - if cat ${{ env.regression-test-file }} | grep -q "FAILED"; then - exit 1 - fi - name: Upload regression tests resutls uses: actions/upload-artifact@v4 with: name: ${{ env.regression-test-file }} path: ${{ env.build_dir }}/python/${{ env.regression-test-file }} + + - name: Check for regression tests results + working-directory: ${{ env.build_dir }}/python + run: | + if cat ${{ env.regression-test-file }} | grep -q "FAILED"; then + exit 1 + fi From c547aa0e139a8ac8bb713dd55f10e387c3c3bc5a Mon Sep 17 00:00:00 2001 From: Kostya Date: Sun, 4 May 2025 00:25:21 +0300 Subject: [PATCH 56/69] test: check for workspace variable --- .github/workflows/self-hosted.yml | 134 +++++++++++++++--------------- 1 file changed, 68 insertions(+), 66 deletions(-) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index ad64773..364bf4e 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -1,5 +1,3 @@ -# Original script from https://github.com/ptheywood/cuda-cmake-github-actions - name: Self-Hosted on: @@ -26,77 +24,81 @@ jobs: gcc: 13 steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Set environment variables + - name: Cleanup workspace run: | - echo "CUDA_PATH=/usr/local/cuda-12.8" >> $GITHUB_ENV - echo "${CUDA_PATH}/bin" >> $GITHUB_PATH - echo "LD_LIBRARY_PATH=${CUDA_PATH}/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV - echo "СС=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV - echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV - echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV - echo "CUDACXX=/usr/local/cuda-${{ matrix.cuda }}/bin/nvcc" >> $GITHUB_ENV + echo ${{ github.workspace }} - - name: Configure CMake build - run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=ON + # - uses: actions/checkout@v4 + # with: + # submodules: recursive - - name: Build library sources - working-directory: ${{ env.build_dir }} - run: cmake --build . --target all --verbose -j `nproc` + # - name: Set environment variables + # run: | + # echo "CUDA_PATH=/usr/local/cuda-12.8" >> $GITHUB_ENV + # echo "${CUDA_PATH}/bin" >> $GITHUB_PATH + # echo "LD_LIBRARY_PATH=${CUDA_PATH}/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV + # echo "CC=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV + # echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV + # echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV + # echo "CUDACXX=/usr/local/cuda-${{ matrix.cuda }}/bin/nvcc" >> $GITHUB_ENV - test: - name: Test ${{ matrix.os }} GCC ${{ matrix.gcc }} CUDA ${{ matrix.cuda }} - needs: build - runs-on: self-hosted - env: - unit-test-file: test_fallback.txt - regression-test-file: test_regression.txt - strategy: - fail-fast: false - matrix: - include: - - os: ubuntu-24.04 - cuda: "12.8" - gcc: 13 + # - name: Configure CMake build + # run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=ON - steps: - - name: Run unit-tests - working-directory: ${{ env.build_dir }} - run: | - chmod +x scripts/run_tests_fallback.sh - ./scripts/run_tests_fallback.sh | tee ${{ env.unit-test-file }} + # - name: Build library sources + # working-directory: ${{ env.build_dir }} + # run: cmake --build . --target all --verbose -j `nproc` + + # test: + # name: Test ${{ matrix.os }} GCC ${{ matrix.gcc }} CUDA ${{ matrix.cuda }} + # needs: build + # runs-on: self-hosted + # env: + # unit-test-file: test_fallback.txt + # regression-test-file: test_regression.txt + # strategy: + # fail-fast: false + # matrix: + # include: + # - os: ubuntu-24.04 + # cuda: "12.8" + # gcc: 13 - - name: Upload unit tests resutls - uses: actions/upload-artifact@v4 - with: - name: ${{ env.unit-test-file }} - path: ${{ env.build_dir }}/${{ env.unit-test-file }} + # steps: + # - name: Run unit-tests + # working-directory: ${{ env.build_dir }} + # run: | + # chmod +x scripts/run_tests_fallback.sh + # ./scripts/run_tests_fallback.sh | tee ${{ env.unit-test-file }} + + # - name: Upload unit tests resutls + # uses: actions/upload-artifact@v4 + # with: + # name: ${{ env.unit-test-file }} + # path: ${{ env.build_dir }}/${{ env.unit-test-file }} - - name: Check for unit tests results - working-directory: ${{ env.build_dir }} - run: | - if cat ${{ env.unit-test-file }} | grep -q "FAILED"; then - exit 1 - fi + # - name: Check for unit tests results + # working-directory: ${{ env.build_dir }} + # run: | + # if cat ${{ env.unit-test-file }} | grep -q "FAILED"; then + # exit 1 + # fi - - name: Run regression-tests - working-directory: ${{ env.build_dir }}/python - run: | - chmod +x run_tests.sh - ./run_tests.sh | tee ${{ env.regression-test-file }} + # - name: Run regression-tests + # working-directory: ${{ env.build_dir }}/python + # run: | + # chmod +x run_tests.sh + # ./run_tests.sh | tee ${{ env.regression-test-file }} - - name: Upload regression tests resutls - uses: actions/upload-artifact@v4 - with: - name: ${{ env.regression-test-file }} - path: ${{ env.build_dir }}/python/${{ env.regression-test-file }} + # - name: Upload regression tests resutls + # uses: actions/upload-artifact@v4 + # with: + # name: ${{ env.regression-test-file }} + # path: ${{ env.build_dir }}/python/${{ env.regression-test-file }} - - name: Check for regression tests results - working-directory: ${{ env.build_dir }}/python - run: | - if cat ${{ env.regression-test-file }} | grep -q "FAILED"; then - exit 1 - fi + # - name: Check for regression tests results + # working-directory: ${{ env.build_dir }}/python + # run: | + # if cat ${{ env.regression-test-file }} | grep -q "FAILED"; then + # exit 1 + # fi From 66dead93e18c5e82c117d4e64d2f214dc51b818f Mon Sep 17 00:00:00 2001 From: Kostya Date: Sun, 4 May 2025 02:28:59 +0300 Subject: [PATCH 57/69] ci: add clean up of repo --- .github/workflows/self-hosted.yml | 131 +++++++++++++++--------------- 1 file changed, 66 insertions(+), 65 deletions(-) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index 364bf4e..21e9cef 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -26,79 +26,80 @@ jobs: steps: - name: Cleanup workspace run: | - echo ${{ github.workspace }} + rm -rf ${{ github.workspace }}/* + rm -rf ${{ github.workspace }}/.* - # - uses: actions/checkout@v4 - # with: - # submodules: recursive + - uses: actions/checkout@v4 + with: + submodules: recursive - # - name: Set environment variables - # run: | - # echo "CUDA_PATH=/usr/local/cuda-12.8" >> $GITHUB_ENV - # echo "${CUDA_PATH}/bin" >> $GITHUB_PATH - # echo "LD_LIBRARY_PATH=${CUDA_PATH}/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV - # echo "CC=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV - # echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV - # echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV - # echo "CUDACXX=/usr/local/cuda-${{ matrix.cuda }}/bin/nvcc" >> $GITHUB_ENV + - name: Set environment variables + run: | + echo "CUDA_PATH=/usr/local/cuda-12.8" >> $GITHUB_ENV + echo "${CUDA_PATH}/bin" >> $GITHUB_PATH + echo "LD_LIBRARY_PATH=${CUDA_PATH}/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV + echo "CC=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV + echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV + echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV + echo "CUDACXX=/usr/local/cuda-${{ matrix.cuda }}/bin/nvcc" >> $GITHUB_ENV - # - name: Configure CMake build - # run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=ON + - name: Configure CMake build + run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=ON - # - name: Build library sources - # working-directory: ${{ env.build_dir }} - # run: cmake --build . --target all --verbose -j `nproc` + - name: Build library sources + working-directory: ${{ env.build_dir }} + run: cmake --build . --target all --verbose -j `nproc` - # test: - # name: Test ${{ matrix.os }} GCC ${{ matrix.gcc }} CUDA ${{ matrix.cuda }} - # needs: build - # runs-on: self-hosted - # env: - # unit-test-file: test_fallback.txt - # regression-test-file: test_regression.txt - # strategy: - # fail-fast: false - # matrix: - # include: - # - os: ubuntu-24.04 - # cuda: "12.8" - # gcc: 13 + test: + name: Test ${{ matrix.os }} GCC ${{ matrix.gcc }} CUDA ${{ matrix.cuda }} + needs: build + runs-on: self-hosted + env: + unit-test-file: test_fallback.txt + regression-test-file: test_regression.txt + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-24.04 + cuda: "12.8" + gcc: 13 - # steps: - # - name: Run unit-tests - # working-directory: ${{ env.build_dir }} - # run: | - # chmod +x scripts/run_tests_fallback.sh - # ./scripts/run_tests_fallback.sh | tee ${{ env.unit-test-file }} + steps: + - name: Run unit-tests + working-directory: ${{ env.build_dir }} + run: | + chmod +x scripts/run_tests_fallback.sh + ./scripts/run_tests_fallback.sh | tee ${{ env.unit-test-file }} - # - name: Upload unit tests resutls - # uses: actions/upload-artifact@v4 - # with: - # name: ${{ env.unit-test-file }} - # path: ${{ env.build_dir }}/${{ env.unit-test-file }} + - name: Upload unit tests resutls + uses: actions/upload-artifact@v4 + with: + name: ${{ env.unit-test-file }} + path: ${{ env.build_dir }}/${{ env.unit-test-file }} - # - name: Check for unit tests results - # working-directory: ${{ env.build_dir }} - # run: | - # if cat ${{ env.unit-test-file }} | grep -q "FAILED"; then - # exit 1 - # fi + - name: Check for unit tests results + working-directory: ${{ env.build_dir }} + run: | + if cat ${{ env.unit-test-file }} | grep -q "FAILED"; then + exit 1 + fi - # - name: Run regression-tests - # working-directory: ${{ env.build_dir }}/python - # run: | - # chmod +x run_tests.sh - # ./run_tests.sh | tee ${{ env.regression-test-file }} + - name: Run regression-tests + working-directory: ${{ env.build_dir }}/python + run: | + chmod +x run_tests.sh + ./run_tests.sh | tee ${{ env.regression-test-file }} - # - name: Upload regression tests resutls - # uses: actions/upload-artifact@v4 - # with: - # name: ${{ env.regression-test-file }} - # path: ${{ env.build_dir }}/python/${{ env.regression-test-file }} + - name: Upload regression tests resutls + uses: actions/upload-artifact@v4 + with: + name: ${{ env.regression-test-file }} + path: ${{ env.build_dir }}/python/${{ env.regression-test-file }} - # - name: Check for regression tests results - # working-directory: ${{ env.build_dir }}/python - # run: | - # if cat ${{ env.regression-test-file }} | grep -q "FAILED"; then - # exit 1 - # fi + - name: Check for regression tests results + working-directory: ${{ env.build_dir }}/python + run: | + if cat ${{ env.regression-test-file }} | grep -q "FAILED"; then + exit 1 + fi From 63e2384d931efb66621af241f2622d4fbb2f9f29 Mon Sep 17 00:00:00 2001 From: Kostya Date: Sun, 4 May 2025 03:18:24 +0300 Subject: [PATCH 58/69] ci: add tests on cpu --- .github/workflows/self-hosted-cpu.yaml | 103 ++++++++++++++++++ .../{self-hosted.yml => self-hosted-gpu.yml} | 16 ++- .github/workflows/ubuntu.yml | 2 +- 3 files changed, 111 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/self-hosted-cpu.yaml rename .github/workflows/{self-hosted.yml => self-hosted-gpu.yml} (89%) diff --git a/.github/workflows/self-hosted-cpu.yaml b/.github/workflows/self-hosted-cpu.yaml new file mode 100644 index 0000000..b3dd0e1 --- /dev/null +++ b/.github/workflows/self-hosted-cpu.yaml @@ -0,0 +1,103 @@ +name: Self-Hosted-CPU + +on: + push: + branches: [ master, self-hosted-runner ] + pull_request: + branches: [ master ] + workflow_dispatch: + +env: + build_dir: "build" + config: "Release" + +jobs: + build: + name: Build ${{ matrix.os }} GCC ${{ matrix.gcc }} CUDA ${{ matrix.cuda }} + runs-on: self-hosted + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-24.04 + cuda: "12.8" + gcc: 13 + + steps: + - name: Cleanup workspace + run: | + rm -rf ${{ github.workspace }}/* + rm -rf ${{ github.workspace }}/.* + + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Set environment variables + run: | + echo "CUDA_PATH=/usr/local/cuda-12.8" >> $GITHUB_ENV + echo "${CUDA_PATH}/bin" >> $GITHUB_PATH + echo "LD_LIBRARY_PATH=${CUDA_PATH}/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV + echo "CC=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV + echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV + echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV + echo "CUDACXX=/usr/local/cuda-${{ matrix.cuda }}/bin/nvcc" >> $GITHUB_ENV + + - name: Configure CMake build + run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=ON -DCUBOOL_WITH_CUDA=OFF + + - name: Build library sources + working-directory: ${{ env.build_dir }} + run: cmake --build . --target all --verbose -j `nproc` + + test: + name: Test ${{ matrix.cpu }} + needs: build + runs-on: self-hosted + env: + unit-test-file: test_all.txt + regression-test-file: test_regression.txt + strategy: + fail-fast: false + matrix: + include: + - cpu: Intel-Core-i7-4790 + + steps: + - name: Run unit-tests + working-directory: ${{ env.build_dir }} + run: | + chmod +x scripts/run_tests_all.sh + ./scripts/run_tests_all.sh | tee ${{ env.unit-test-file }} + + - name: Upload unit tests resutls + uses: actions/upload-artifact@v4 + with: + name: ${{ env.unit-test-file }} + path: ${{ env.build_dir }}/${{ env.unit-test-file }} + + - name: Check for unit tests results + working-directory: ${{ env.build_dir }} + run: | + if cat ${{ env.unit-test-file }} | grep -q "FAILED"; then + exit 1 + fi + + - name: Run regression-tests + working-directory: ${{ env.build_dir }}/python + run: | + chmod +x run_tests.sh + ./run_tests.sh | tee ${{ env.regression-test-file }} + + - name: Upload regression tests resutls + uses: actions/upload-artifact@v4 + with: + name: ${{ env.regression-test-file }} + path: ${{ env.build_dir }}/python/${{ env.regression-test-file }} + + - name: Check for regression tests results + working-directory: ${{ env.build_dir }}/python + run: | + if cat ${{ env.regression-test-file }} | grep -q "FAILED"; then + exit 1 + fi diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted-gpu.yml similarity index 89% rename from .github/workflows/self-hosted.yml rename to .github/workflows/self-hosted-gpu.yml index 21e9cef..da81265 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted-gpu.yml @@ -1,4 +1,4 @@ -name: Self-Hosted +name: Self-Hosted-GPU on: push: @@ -51,33 +51,31 @@ jobs: run: cmake --build . --target all --verbose -j `nproc` test: - name: Test ${{ matrix.os }} GCC ${{ matrix.gcc }} CUDA ${{ matrix.cuda }} + name: Test ${{ matrix.gpu }} needs: build runs-on: self-hosted env: - unit-test-file: test_fallback.txt + unit-test-file: test_all.txt regression-test-file: test_regression.txt strategy: fail-fast: false matrix: include: - - os: ubuntu-24.04 - cuda: "12.8" - gcc: 13 + - gpu: NVIDIA-GeForce-GT-1030 steps: - name: Run unit-tests working-directory: ${{ env.build_dir }} run: | - chmod +x scripts/run_tests_fallback.sh - ./scripts/run_tests_fallback.sh | tee ${{ env.unit-test-file }} + chmod +x scripts/run_tests_all.sh + ./scripts/run_tests_all.sh | tee ${{ env.unit-test-file }} - name: Upload unit tests resutls uses: actions/upload-artifact@v4 with: name: ${{ env.unit-test-file }} path: ${{ env.build_dir }}/${{ env.unit-test-file }} - + - name: Check for unit tests results working-directory: ${{ env.build_dir }} run: | diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 54871c9..a644c9b 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -51,7 +51,7 @@ jobs: - name: Install and configure GCC and GXX run: | sudo apt-get install -y gcc-${{ matrix.gcc }} g++-${{ matrix.gcc }} - echo "СС=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV + echo "CC=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV From 3a7049fa460a23c0f4de7add5eb8fd148aa67689 Mon Sep 17 00:00:00 2001 From: Kostya Date: Sun, 4 May 2025 04:17:21 +0300 Subject: [PATCH 59/69] ci: delete run on test branches --- .github/workflows/self-hosted-cpu.yaml | 6 +++--- .github/workflows/self-hosted-gpu.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/self-hosted-cpu.yaml b/.github/workflows/self-hosted-cpu.yaml index b3dd0e1..b560846 100644 --- a/.github/workflows/self-hosted-cpu.yaml +++ b/.github/workflows/self-hosted-cpu.yaml @@ -2,7 +2,7 @@ name: Self-Hosted-CPU on: push: - branches: [ master, self-hosted-runner ] + branches: [ master ] pull_request: branches: [ master ] workflow_dispatch: @@ -79,7 +79,7 @@ jobs: - name: Check for unit tests results working-directory: ${{ env.build_dir }} run: | - if cat ${{ env.unit-test-file }} | grep -q "FAILED"; then + if grep -q "FAILED" ${{ env.unit-test-file }}; then exit 1 fi @@ -98,6 +98,6 @@ jobs: - name: Check for regression tests results working-directory: ${{ env.build_dir }}/python run: | - if cat ${{ env.regression-test-file }} | grep -q "FAILED"; then + if grep -q "FAILED" ${{ env.regression-test-file }}; then exit 1 fi diff --git a/.github/workflows/self-hosted-gpu.yml b/.github/workflows/self-hosted-gpu.yml index da81265..b130906 100644 --- a/.github/workflows/self-hosted-gpu.yml +++ b/.github/workflows/self-hosted-gpu.yml @@ -2,7 +2,7 @@ name: Self-Hosted-GPU on: push: - branches: [ master, self-hosted-runner ] + branches: [ master ] pull_request: branches: [ master ] workflow_dispatch: @@ -79,7 +79,7 @@ jobs: - name: Check for unit tests results working-directory: ${{ env.build_dir }} run: | - if cat ${{ env.unit-test-file }} | grep -q "FAILED"; then + if grep -q "FAILED" ${{ env.unit-test-file }}; then exit 1 fi @@ -98,6 +98,6 @@ jobs: - name: Check for regression tests results working-directory: ${{ env.build_dir }}/python run: | - if cat ${{ env.regression-test-file }} | grep -q "FAILED"; then + if grep -q "FAILED" ${{ env.regression-test-file }}; then exit 1 fi From 36a4776a93cabc7e9211ce77a0fe3b1b585a6a09 Mon Sep 17 00:00:00 2001 From: Kostya Date: Sun, 4 May 2025 04:19:17 +0300 Subject: [PATCH 60/69] ci: delete run on self-hosted branch --- .github/workflows/ubuntu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index a644c9b..2470f9a 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -4,7 +4,7 @@ name: Ubuntu on: push: - branches: [ master, self-hosted-runner ] + branches: [ master ] pull_request: branches: [ master ] workflow_dispatch: From 3ea12e7b94c98939a820c645c0248d7a216dadf1 Mon Sep 17 00:00:00 2001 From: Kostya Date: Tue, 6 May 2025 00:11:56 +0300 Subject: [PATCH 61/69] ci: delete cpu pipeline --- .github/workflows/self-hosted-cpu.yaml | 103 ------------------ .../{self-hosted-gpu.yml => self-hosted.yml} | 22 ++-- 2 files changed, 13 insertions(+), 112 deletions(-) delete mode 100644 .github/workflows/self-hosted-cpu.yaml rename .github/workflows/{self-hosted-gpu.yml => self-hosted.yml} (92%) diff --git a/.github/workflows/self-hosted-cpu.yaml b/.github/workflows/self-hosted-cpu.yaml deleted file mode 100644 index b560846..0000000 --- a/.github/workflows/self-hosted-cpu.yaml +++ /dev/null @@ -1,103 +0,0 @@ -name: Self-Hosted-CPU - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - workflow_dispatch: - -env: - build_dir: "build" - config: "Release" - -jobs: - build: - name: Build ${{ matrix.os }} GCC ${{ matrix.gcc }} CUDA ${{ matrix.cuda }} - runs-on: self-hosted - strategy: - fail-fast: false - matrix: - include: - - os: ubuntu-24.04 - cuda: "12.8" - gcc: 13 - - steps: - - name: Cleanup workspace - run: | - rm -rf ${{ github.workspace }}/* - rm -rf ${{ github.workspace }}/.* - - - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Set environment variables - run: | - echo "CUDA_PATH=/usr/local/cuda-12.8" >> $GITHUB_ENV - echo "${CUDA_PATH}/bin" >> $GITHUB_PATH - echo "LD_LIBRARY_PATH=${CUDA_PATH}/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV - echo "CC=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV - echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV - echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV - echo "CUDACXX=/usr/local/cuda-${{ matrix.cuda }}/bin/nvcc" >> $GITHUB_ENV - - - name: Configure CMake build - run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=ON -DCUBOOL_WITH_CUDA=OFF - - - name: Build library sources - working-directory: ${{ env.build_dir }} - run: cmake --build . --target all --verbose -j `nproc` - - test: - name: Test ${{ matrix.cpu }} - needs: build - runs-on: self-hosted - env: - unit-test-file: test_all.txt - regression-test-file: test_regression.txt - strategy: - fail-fast: false - matrix: - include: - - cpu: Intel-Core-i7-4790 - - steps: - - name: Run unit-tests - working-directory: ${{ env.build_dir }} - run: | - chmod +x scripts/run_tests_all.sh - ./scripts/run_tests_all.sh | tee ${{ env.unit-test-file }} - - - name: Upload unit tests resutls - uses: actions/upload-artifact@v4 - with: - name: ${{ env.unit-test-file }} - path: ${{ env.build_dir }}/${{ env.unit-test-file }} - - - name: Check for unit tests results - working-directory: ${{ env.build_dir }} - run: | - if grep -q "FAILED" ${{ env.unit-test-file }}; then - exit 1 - fi - - - name: Run regression-tests - working-directory: ${{ env.build_dir }}/python - run: | - chmod +x run_tests.sh - ./run_tests.sh | tee ${{ env.regression-test-file }} - - - name: Upload regression tests resutls - uses: actions/upload-artifact@v4 - with: - name: ${{ env.regression-test-file }} - path: ${{ env.build_dir }}/python/${{ env.regression-test-file }} - - - name: Check for regression tests results - working-directory: ${{ env.build_dir }}/python - run: | - if grep -q "FAILED" ${{ env.regression-test-file }}; then - exit 1 - fi diff --git a/.github/workflows/self-hosted-gpu.yml b/.github/workflows/self-hosted.yml similarity index 92% rename from .github/workflows/self-hosted-gpu.yml rename to .github/workflows/self-hosted.yml index b130906..bee36bb 100644 --- a/.github/workflows/self-hosted-gpu.yml +++ b/.github/workflows/self-hosted.yml @@ -24,11 +24,6 @@ jobs: gcc: 13 steps: - - name: Cleanup workspace - run: | - rm -rf ${{ github.workspace }}/* - rm -rf ${{ github.workspace }}/.* - - uses: actions/checkout@v4 with: submodules: recursive @@ -67,8 +62,7 @@ jobs: - name: Run unit-tests working-directory: ${{ env.build_dir }} run: | - chmod +x scripts/run_tests_all.sh - ./scripts/run_tests_all.sh | tee ${{ env.unit-test-file }} + source scripts/run_tests_all.sh | tee ${{ env.unit-test-file }} - name: Upload unit tests resutls uses: actions/upload-artifact@v4 @@ -86,8 +80,7 @@ jobs: - name: Run regression-tests working-directory: ${{ env.build_dir }}/python run: | - chmod +x run_tests.sh - ./run_tests.sh | tee ${{ env.regression-test-file }} + source run_tests.sh | tee ${{ env.regression-test-file }} - name: Upload regression tests resutls uses: actions/upload-artifact@v4 @@ -101,3 +94,14 @@ jobs: if grep -q "FAILED" ${{ env.regression-test-file }}; then exit 1 fi + + clean: + name: Cleanup workspace + if: always() + runs-on: self-hosted + + steps: + - name: Cleanup workspace + run: | + rm -rf ${{ github.workspace }}/* + rm -rf ${{ github.workspace }}/.* From 033366abd4aaaab0ebce65f08bac005e51912124 Mon Sep 17 00:00:00 2001 From: Kostya Date: Tue, 6 May 2025 00:58:35 +0300 Subject: [PATCH 62/69] test: cleanup in failed pipeline --- .github/workflows/self-hosted.yml | 65 ++++++++++++++++--------------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index bee36bb..7a9adcb 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -50,8 +50,8 @@ jobs: needs: build runs-on: self-hosted env: - unit-test-file: test_all.txt - regression-test-file: test_regression.txt + unit-test-file: test_all.log + regression-test-file: test_regression.log strategy: fail-fast: false matrix: @@ -59,41 +59,44 @@ jobs: - gpu: NVIDIA-GeForce-GT-1030 steps: - - name: Run unit-tests - working-directory: ${{ env.build_dir }} + - name: Fail run: | - source scripts/run_tests_all.sh | tee ${{ env.unit-test-file }} + exit 1 + # - name: Run unit-tests + # working-directory: ${{ env.build_dir }} + # run: | + # source scripts/run_tests_all.sh | tee ${{ env.unit-test-file }} - - name: Upload unit tests resutls - uses: actions/upload-artifact@v4 - with: - name: ${{ env.unit-test-file }} - path: ${{ env.build_dir }}/${{ env.unit-test-file }} + # - name: Upload unit tests resutls + # uses: actions/upload-artifact@v4 + # with: + # name: ${{ env.unit-test-file }} + # path: ${{ env.build_dir }}/${{ env.unit-test-file }} - - name: Check for unit tests results - working-directory: ${{ env.build_dir }} - run: | - if grep -q "FAILED" ${{ env.unit-test-file }}; then - exit 1 - fi + # - name: Check for unit tests results + # working-directory: ${{ env.build_dir }} + # run: | + # if grep -q "FAILED" ${{ env.unit-test-file }}; then + # exit 1 + # fi - - name: Run regression-tests - working-directory: ${{ env.build_dir }}/python - run: | - source run_tests.sh | tee ${{ env.regression-test-file }} + # - name: Run regression-tests + # working-directory: ${{ env.build_dir }}/python + # run: | + # source run_tests.sh 2>&1 | tee ${{ env.regression-test-file }} - - name: Upload regression tests resutls - uses: actions/upload-artifact@v4 - with: - name: ${{ env.regression-test-file }} - path: ${{ env.build_dir }}/python/${{ env.regression-test-file }} + # - name: Upload regression tests resutls + # uses: actions/upload-artifact@v4 + # with: + # name: ${{ env.regression-test-file }} + # path: ${{ env.build_dir }}/python/${{ env.regression-test-file }} - - name: Check for regression tests results - working-directory: ${{ env.build_dir }}/python - run: | - if grep -q "FAILED" ${{ env.regression-test-file }}; then - exit 1 - fi + # - name: Check for regression tests results + # working-directory: ${{ env.build_dir }}/python + # run: | + # if grep -q "FAILED" ${{ env.regression-test-file }}; then + # exit 1 + # fi clean: name: Cleanup workspace From d85d41a4cdbddafadbf7e7ffe2c49b1689ffbf01 Mon Sep 17 00:00:00 2001 From: Kostya Date: Tue, 6 May 2025 01:03:23 +0300 Subject: [PATCH 63/69] ci: add needs in clean job --- .github/workflows/self-hosted.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index 7a9adcb..d623928 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -1,4 +1,4 @@ -name: Self-Hosted-GPU +name: Self-Hosted on: push: @@ -100,6 +100,7 @@ jobs: clean: name: Cleanup workspace + needs: test if: always() runs-on: self-hosted From 5b0b33e4c8f38a3c0e76cfb493a82afc02dcfcd6 Mon Sep 17 00:00:00 2001 From: Kostya Date: Tue, 6 May 2025 01:09:15 +0300 Subject: [PATCH 64/69] ci: uncomment test pipeline --- .github/workflows/self-hosted.yml | 61 +++++++++++++++---------------- 1 file changed, 29 insertions(+), 32 deletions(-) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index d623928..ee547b5 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -59,44 +59,41 @@ jobs: - gpu: NVIDIA-GeForce-GT-1030 steps: - - name: Fail + - name: Run unit-tests + working-directory: ${{ env.build_dir }} run: | - exit 1 - # - name: Run unit-tests - # working-directory: ${{ env.build_dir }} - # run: | - # source scripts/run_tests_all.sh | tee ${{ env.unit-test-file }} + source scripts/run_tests_all.sh | tee ${{ env.unit-test-file }} - # - name: Upload unit tests resutls - # uses: actions/upload-artifact@v4 - # with: - # name: ${{ env.unit-test-file }} - # path: ${{ env.build_dir }}/${{ env.unit-test-file }} + - name: Upload unit tests resutls + uses: actions/upload-artifact@v4 + with: + name: ${{ env.unit-test-file }} + path: ${{ env.build_dir }}/${{ env.unit-test-file }} - # - name: Check for unit tests results - # working-directory: ${{ env.build_dir }} - # run: | - # if grep -q "FAILED" ${{ env.unit-test-file }}; then - # exit 1 - # fi + - name: Check for unit tests results + working-directory: ${{ env.build_dir }} + run: | + if grep -q "FAILED" ${{ env.unit-test-file }}; then + exit 1 + fi - # - name: Run regression-tests - # working-directory: ${{ env.build_dir }}/python - # run: | - # source run_tests.sh 2>&1 | tee ${{ env.regression-test-file }} + - name: Run regression-tests + working-directory: ${{ env.build_dir }}/python + run: | + source run_tests.sh 2>&1 | tee ${{ env.regression-test-file }} - # - name: Upload regression tests resutls - # uses: actions/upload-artifact@v4 - # with: - # name: ${{ env.regression-test-file }} - # path: ${{ env.build_dir }}/python/${{ env.regression-test-file }} + - name: Upload regression tests resutls + uses: actions/upload-artifact@v4 + with: + name: ${{ env.regression-test-file }} + path: ${{ env.build_dir }}/python/${{ env.regression-test-file }} - # - name: Check for regression tests results - # working-directory: ${{ env.build_dir }}/python - # run: | - # if grep -q "FAILED" ${{ env.regression-test-file }}; then - # exit 1 - # fi + - name: Check for regression tests results + working-directory: ${{ env.build_dir }}/python + run: | + if grep -q "FAILED" ${{ env.regression-test-file }}; then + exit 1 + fi clean: name: Cleanup workspace From a82a79173a7eb489a7cdfaa4f3a936b21798adb1 Mon Sep 17 00:00:00 2001 From: Kostya Date: Wed, 7 May 2025 00:52:14 +0300 Subject: [PATCH 65/69] test: check cpu version of github runner --- .github/workflows/self-hosted.yml | 12 +++-- .github/workflows/ubuntu.yml | 81 +++++++++++++++++++++---------- 2 files changed, 62 insertions(+), 31 deletions(-) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index ee547b5..d7ff6aa 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -9,7 +9,6 @@ on: env: build_dir: "build" - config: "Release" jobs: build: @@ -22,6 +21,8 @@ jobs: - os: ubuntu-24.04 cuda: "12.8" gcc: 13 + env: + config: "Release" steps: - uses: actions/checkout@v4 @@ -46,17 +47,18 @@ jobs: run: cmake --build . --target all --verbose -j `nproc` test: - name: Test ${{ matrix.gpu }} + name: Test GPU ${{ matrix.gpu }} CUDA ${{ matrix.cuda }} needs: build runs-on: self-hosted env: - unit-test-file: test_all.log - regression-test-file: test_regression.log + unit-test-file: gpu_test_all.log + regression-test-file: gpu_test_regression.log strategy: fail-fast: false matrix: include: - gpu: NVIDIA-GeForce-GT-1030 + cuda: "12.9" steps: - name: Run unit-tests @@ -94,7 +96,7 @@ jobs: if grep -q "FAILED" ${{ env.regression-test-file }}; then exit 1 fi - + clean: name: Cleanup workspace needs: test diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 2470f9a..01313c2 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -9,6 +9,9 @@ on: branches: [ master ] workflow_dispatch: +env: + build_dir: "build" + jobs: build: name: Build ${{ matrix.os }} GCC ${{ matrix.gcc }} CUDA ${{ matrix.cuda }} @@ -21,7 +24,6 @@ jobs: cuda: "11.7" gcc: 9 env: - build_dir: "build" config: "Release" artifact: "cubool-ubuntu-build.tar.xz" @@ -30,33 +32,11 @@ jobs: with: submodules: true - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Install CUDA - env: - cuda: ${{ matrix.cuda }} - run: | - source ./scripts/install_cuda_ubuntu.sh - if [[ $? -eq 0 ]]; then - # Set paths for subsequent steps, using ${CUDA_PATH} - echo "Adding CUDA to CUDA_PATH, PATH and LD_LIBRARY_PATH" - echo "CUDA_PATH=${CUDA_PATH}" >> $GITHUB_ENV - echo "${CUDA_PATH}/bin" >> $GITHUB_PATH - echo "LD_LIBRARY_PATH=${CUDA_PATH}/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV - fi - shell: bash - - - name: Install and configure GCC and GXX - run: | - sudo apt-get install -y gcc-${{ matrix.gcc }} g++-${{ matrix.gcc }} - echo "CC=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV - echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV - echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV + - name: Check CPU + run: cat /proc/cpuinfo - name: Configure CMake build - run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=ON + run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=ON -DCUBOOL_WITH_CUDA=OFF - name: Build library sources working-directory: ${{ env.build_dir }} @@ -71,3 +51,52 @@ jobs: with: name: ${{ env.artifact }} path: ${{ env.artifact }} + + test: + name: Test CPU + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-22.04 + env: + unit-test-file: cpu_test_all.log + regression-test-file: cpu_test_regression.log + + steps: + - name: Run unit-tests + working-directory: ${{ env.build_dir }} + run: | + source scripts/run_tests_all.sh | tee ${{ env.unit-test-file }} + + - name: Upload unit tests resutls + uses: actions/upload-artifact@v4 + with: + name: ${{ env.unit-test-file }} + path: ${{ env.build_dir }}/${{ env.unit-test-file }} + + - name: Check for unit tests results + working-directory: ${{ env.build_dir }} + run: | + if grep -q "FAILED" ${{ env.unit-test-file }}; then + exit 1 + fi + + - name: Run regression-tests + working-directory: ${{ env.build_dir }}/python + run: | + source run_tests.sh 2>&1 | tee ${{ env.regression-test-file }} + + - name: Upload regression tests resutls + uses: actions/upload-artifact@v4 + with: + name: ${{ env.regression-test-file }} + path: ${{ env.build_dir }}/python/${{ env.regression-test-file }} + + - name: Check for regression tests results + working-directory: ${{ env.build_dir }}/python + run: | + if grep -q "FAILED" ${{ env.regression-test-file }}; then + exit 1 + fi From aec5bb93d61636247af272fa263ffa051991fae7 Mon Sep 17 00:00:00 2001 From: Kostya Date: Wed, 7 May 2025 01:05:43 +0300 Subject: [PATCH 66/69] ci: add tests for cpu use case --- .github/workflows/self-hosted.yml | 10 ++++++++-- .github/workflows/ubuntu.yml | 13 +++++++++---- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index d7ff6aa..8f3d08d 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -3,8 +3,6 @@ name: Self-Hosted on: push: branches: [ master ] - pull_request: - branches: [ master ] workflow_dispatch: env: @@ -30,6 +28,7 @@ jobs: submodules: recursive - name: Set environment variables + shell: bash run: | echo "CUDA_PATH=/usr/local/cuda-12.8" >> $GITHUB_ENV echo "${CUDA_PATH}/bin" >> $GITHUB_PATH @@ -40,10 +39,12 @@ jobs: echo "CUDACXX=/usr/local/cuda-${{ matrix.cuda }}/bin/nvcc" >> $GITHUB_ENV - name: Configure CMake build + shell: bash run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=ON - name: Build library sources working-directory: ${{ env.build_dir }} + shell: bash run: cmake --build . --target all --verbose -j `nproc` test: @@ -63,6 +64,7 @@ jobs: steps: - name: Run unit-tests working-directory: ${{ env.build_dir }} + shell: bash run: | source scripts/run_tests_all.sh | tee ${{ env.unit-test-file }} @@ -74,6 +76,7 @@ jobs: - name: Check for unit tests results working-directory: ${{ env.build_dir }} + shell: bash run: | if grep -q "FAILED" ${{ env.unit-test-file }}; then exit 1 @@ -81,6 +84,7 @@ jobs: - name: Run regression-tests working-directory: ${{ env.build_dir }}/python + shell: bash run: | source run_tests.sh 2>&1 | tee ${{ env.regression-test-file }} @@ -92,6 +96,7 @@ jobs: - name: Check for regression tests results working-directory: ${{ env.build_dir }}/python + shell: bash run: | if grep -q "FAILED" ${{ env.regression-test-file }}; then exit 1 @@ -105,6 +110,7 @@ jobs: steps: - name: Cleanup workspace + shell: bash run: | rm -rf ${{ github.workspace }}/* rm -rf ${{ github.workspace }}/.* diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 01313c2..384fcc4 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -32,14 +32,13 @@ jobs: with: submodules: true - - name: Check CPU - run: cat /proc/cpuinfo - - name: Configure CMake build + shell: bash run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=ON -DCUBOOL_WITH_CUDA=OFF - name: Build library sources working-directory: ${{ env.build_dir }} + shell: bash run: cmake --build . --target all --verbose -j `nproc` - name: Prepare upload binary @@ -53,13 +52,15 @@ jobs: path: ${{ env.artifact }} test: - name: Test CPU + name: Test CPU ${{ matrix.cpu }} + needs: build runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: include: - os: ubuntu-22.04 + cpu: AMD-EPYC-7763 env: unit-test-file: cpu_test_all.log regression-test-file: cpu_test_regression.log @@ -67,6 +68,7 @@ jobs: steps: - name: Run unit-tests working-directory: ${{ env.build_dir }} + shell: bash run: | source scripts/run_tests_all.sh | tee ${{ env.unit-test-file }} @@ -78,6 +80,7 @@ jobs: - name: Check for unit tests results working-directory: ${{ env.build_dir }} + shell: bash run: | if grep -q "FAILED" ${{ env.unit-test-file }}; then exit 1 @@ -85,6 +88,7 @@ jobs: - name: Run regression-tests working-directory: ${{ env.build_dir }}/python + shell: bash run: | source run_tests.sh 2>&1 | tee ${{ env.regression-test-file }} @@ -96,6 +100,7 @@ jobs: - name: Check for regression tests results working-directory: ${{ env.build_dir }}/python + shell: bash run: | if grep -q "FAILED" ${{ env.regression-test-file }}; then exit 1 From 412f0a68594ccba40374f4b626fd3b2378b5a156 Mon Sep 17 00:00:00 2001 From: Kostya Date: Wed, 7 May 2025 01:12:54 +0300 Subject: [PATCH 67/69] ci: delete shell from steps --- .github/workflows/self-hosted.yml | 8 -------- .github/workflows/ubuntu.yml | 7 ------- 2 files changed, 15 deletions(-) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index 8f3d08d..3f0912f 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -28,7 +28,6 @@ jobs: submodules: recursive - name: Set environment variables - shell: bash run: | echo "CUDA_PATH=/usr/local/cuda-12.8" >> $GITHUB_ENV echo "${CUDA_PATH}/bin" >> $GITHUB_PATH @@ -39,12 +38,10 @@ jobs: echo "CUDACXX=/usr/local/cuda-${{ matrix.cuda }}/bin/nvcc" >> $GITHUB_ENV - name: Configure CMake build - shell: bash run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=ON - name: Build library sources working-directory: ${{ env.build_dir }} - shell: bash run: cmake --build . --target all --verbose -j `nproc` test: @@ -64,7 +61,6 @@ jobs: steps: - name: Run unit-tests working-directory: ${{ env.build_dir }} - shell: bash run: | source scripts/run_tests_all.sh | tee ${{ env.unit-test-file }} @@ -76,7 +72,6 @@ jobs: - name: Check for unit tests results working-directory: ${{ env.build_dir }} - shell: bash run: | if grep -q "FAILED" ${{ env.unit-test-file }}; then exit 1 @@ -84,7 +79,6 @@ jobs: - name: Run regression-tests working-directory: ${{ env.build_dir }}/python - shell: bash run: | source run_tests.sh 2>&1 | tee ${{ env.regression-test-file }} @@ -96,7 +90,6 @@ jobs: - name: Check for regression tests results working-directory: ${{ env.build_dir }}/python - shell: bash run: | if grep -q "FAILED" ${{ env.regression-test-file }}; then exit 1 @@ -110,7 +103,6 @@ jobs: steps: - name: Cleanup workspace - shell: bash run: | rm -rf ${{ github.workspace }}/* rm -rf ${{ github.workspace }}/.* diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 384fcc4..18812b6 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -33,16 +33,13 @@ jobs: submodules: true - name: Configure CMake build - shell: bash run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=ON -DCUBOOL_WITH_CUDA=OFF - name: Build library sources working-directory: ${{ env.build_dir }} - shell: bash run: cmake --build . --target all --verbose -j `nproc` - name: Prepare upload binary - shell: bash run: tar cfz ${{ env.artifact }} ${{ env.build_dir }} - name: Upload binary @@ -68,7 +65,6 @@ jobs: steps: - name: Run unit-tests working-directory: ${{ env.build_dir }} - shell: bash run: | source scripts/run_tests_all.sh | tee ${{ env.unit-test-file }} @@ -80,7 +76,6 @@ jobs: - name: Check for unit tests results working-directory: ${{ env.build_dir }} - shell: bash run: | if grep -q "FAILED" ${{ env.unit-test-file }}; then exit 1 @@ -88,7 +83,6 @@ jobs: - name: Run regression-tests working-directory: ${{ env.build_dir }}/python - shell: bash run: | source run_tests.sh 2>&1 | tee ${{ env.regression-test-file }} @@ -100,7 +94,6 @@ jobs: - name: Check for regression tests results working-directory: ${{ env.build_dir }}/python - shell: bash run: | if grep -q "FAILED" ${{ env.regression-test-file }}; then exit 1 From 6ab461bd8967160a1c6d737cda3a0dafdfe05ca4 Mon Sep 17 00:00:00 2001 From: Kostya Date: Wed, 7 May 2025 01:21:01 +0300 Subject: [PATCH 68/69] ci: add artifact download --- .github/workflows/ubuntu.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 18812b6..f5a49ff 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -11,6 +11,7 @@ on: env: build_dir: "build" + artifact: "cubool-ubuntu-build.tar.xz" jobs: build: @@ -25,7 +26,6 @@ jobs: gcc: 9 env: config: "Release" - artifact: "cubool-ubuntu-build.tar.xz" steps: - uses: actions/checkout@v4 @@ -63,6 +63,16 @@ jobs: regression-test-file: cpu_test_regression.log steps: + - uses: actions/download-artifact@v4 + with: + name: ${{ env.artifact }} + + - name: Unarchive artifact + run: | + tar xzf ${{ env.artifact }} + rm ${{ env.artifact }} + ls + - name: Run unit-tests working-directory: ${{ env.build_dir }} run: | From f782c5f2737d9215b1c18d84510f0edcf114b796 Mon Sep 17 00:00:00 2001 From: Kostya Date: Wed, 7 May 2025 01:56:36 +0300 Subject: [PATCH 69/69] ci: add cpu and gpu tests run --- .github/workflows/self-hosted.yml | 14 +++++++++----- .github/workflows/ubuntu.yml | 16 +++++++--------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index 3f0912f..57ac676 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -3,6 +3,8 @@ name: Self-Hosted on: push: branches: [ master ] + pull_request: + branches: [ master ] workflow_dispatch: env: @@ -38,25 +40,27 @@ jobs: echo "CUDACXX=/usr/local/cuda-${{ matrix.cuda }}/bin/nvcc" >> $GITHUB_ENV - name: Configure CMake build - run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=ON + run: | + cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=ON - name: Build library sources working-directory: ${{ env.build_dir }} - run: cmake --build . --target all --verbose -j `nproc` + run: | + cmake --build . --target all --verbose -j `nproc` test: name: Test GPU ${{ matrix.gpu }} CUDA ${{ matrix.cuda }} needs: build runs-on: self-hosted - env: - unit-test-file: gpu_test_all.log - regression-test-file: gpu_test_regression.log strategy: fail-fast: false matrix: include: - gpu: NVIDIA-GeForce-GT-1030 cuda: "12.9" + env: + unit-test-file: gpu_test_all.log + regression-test-file: gpu_test_regression.log steps: - name: Run unit-tests diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index f5a49ff..6e020ca 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -1,5 +1,3 @@ -# Original script from https://github.com/ptheywood/cuda-cmake-github-actions - name: Ubuntu on: @@ -15,15 +13,13 @@ env: jobs: build: - name: Build ${{ matrix.os }} GCC ${{ matrix.gcc }} CUDA ${{ matrix.cuda }} + name: Build ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: include: - os: ubuntu-22.04 - cuda: "11.7" - gcc: 9 env: config: "Release" @@ -33,14 +29,17 @@ jobs: submodules: true - name: Configure CMake build - run: cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=ON -DCUBOOL_WITH_CUDA=OFF + run: | + cmake . -B ${{ env.build_dir }} -DCMAKE_BUILD_TYPE=${{ env.config }} -DCUBOOL_BUILD_TESTS=ON -DCUBOOL_WITH_CUDA=OFF - name: Build library sources working-directory: ${{ env.build_dir }} - run: cmake --build . --target all --verbose -j `nproc` + run: | + cmake --build . --target all --verbose -j `nproc` - name: Prepare upload binary - run: tar cfz ${{ env.artifact }} ${{ env.build_dir }} + run: | + tar cfz ${{ env.artifact }} ${{ env.build_dir }} - name: Upload binary uses: actions/upload-artifact@v4 @@ -71,7 +70,6 @@ jobs: run: | tar xzf ${{ env.artifact }} rm ${{ env.artifact }} - ls - name: Run unit-tests working-directory: ${{ env.build_dir }}