From c091f4e1d7ae331761e72996d9fd7c645db9c7d1 Mon Sep 17 00:00:00 2001 From: phoebusm Date: Fri, 22 Mar 2024 13:16:01 +0000 Subject: [PATCH 1/3] Dynmically link openssl --- .github/workflows/build_steps.yml | 9 +++++++++ README.md | 10 ++++++++++ cpp/CMakePresets.json | 16 ++++++++++++++-- .../vcpkg_overlays/triplets/x64-linux-mix.cmake | 9 +++++++++ docs/mkdocs/docs/error_messages.md | 5 +++++ 5 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 cpp/third_party/vcpkg_overlays/triplets/x64-linux-mix.cmake diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index 8e39400c37..53afc50275 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -134,6 +134,10 @@ jobs: if: inputs.job_type == 'cpp-tests' run: cd cpp; cmake --build --preset $ARCTIC_CMAKE_PRESET --target install + - name: Install dynamically linked library for tests + if: inputs.job_type == 'cpp-tests' && matrix.os == 'linux' + run: yum install -y https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/o/openssl11-1.1.1k-7.el7.x86_64.rpm + - name: C++ Rapidcheck if: inputs.job_type == 'cpp-tests' run: cpp/out/install/arcticdb_rapidcheck_tests @@ -164,6 +168,7 @@ jobs: run: pipx run cibuildwheel==${{inputs.cibw_version}} env: CIBW_MANYLINUX_X86_64_IMAGE: ${{inputs.cibw_image_tag}} + CIBW_REPAIR_WHEEL_COMMAND_LINUX: auditwheel repair -L . -w {dest_dir} {wheel} --exclude libssl-a0734ced.so.1.1 - name: Store wheel artifact if: inputs.job_type == 'build-python-wheels' @@ -298,6 +303,10 @@ jobs: ${{vars.EXTRA_TEST_PREPARE_CMD || ''}} env: CMAKE_BUILD_PARALLEL_LEVEL: ${{vars.CMAKE_BUILD_PARALLEL_LEVEL}} + + - name: Install dynamically linked library for tests + if: matrix.os == 'linux' + run: yum install -y https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/o/openssl11-1.1.1k-7.el7.x86_64.rpm - name: Set persistent storage variables if: inputs.persistent_storage == 'true' diff --git a/README.md b/README.md index 538f2ed296..75fdeebd8b 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,16 @@ Install ArcticDB: ```bash $ pip install arcticdb +# For linux-pypi prebuilt binary, since v4.5.0, libssl is no longer statically linked. +# openssl v1.1.1 is needed to be manually installed. +# openssl v1.1.1 can be installed with precompiled package online, if not available in the OS: +# RHEL: +$ yum install -y compat-openssl11 +# or +$ yum install -y https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/o/openssl11-1.1.1k-7.el7.x86_64.rpm +# Ubuntu: +# Please go to http://security.ubuntu.com/ubuntu/pool/main/o/openssl/ to +# download and install latest openssl_1.1.1 and libssl1.1_1.1.1 deb files in sequence ``` or using conda-forge ```bash diff --git a/cpp/CMakePresets.json b/cpp/CMakePresets.json index 005e3b16be..29e2bac188 100644 --- a/cpp/CMakePresets.json +++ b/cpp/CMakePresets.json @@ -110,7 +110,11 @@ }, { "name": "linux-debug", - "inherits": ["common_vcpkg", "linux"] + "inherits": ["common_vcpkg", "linux"], + "cacheVariables": { + "VCPKG_OVERLAY_TRIPLETS": "${sourceDir}/third_party/vcpkg_overlays/triplets/", + "VCPKG_TARGET_TRIPLET": "x64-linux-mix" + } }, { "name": "linux-debug-clang", @@ -145,7 +149,15 @@ { "name": "windows-cl-release", "inherits": "windows-cl-debug", "cacheVariables": { "CMAKE_BUILD_TYPE": "Release" }}, { "name": "windows-cl-conda-release", "inherits": "windows-cl-conda-debug", "cacheVariables": { "CMAKE_BUILD_TYPE": "Release" } }, - { "name": "linux-release", "inherits": "linux-debug", "cacheVariables": { "CMAKE_BUILD_TYPE": "RelWithDebInfo" } }, + { + "name": "linux-release", + "inherits": "linux-debug", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "RelWithDebInfo", + "VCPKG_OVERLAY_TRIPLETS": "${sourceDir}/third_party/vcpkg_overlays/triplets/", + "VCPKG_TARGET_TRIPLET": "x64-linux-mix" + } + }, { "name": "linux-conda-release", "inherits": "linux-conda-debug", "cacheVariables": { "CMAKE_BUILD_TYPE": "RelWithDebInfo" } }, { "name": "darwin-conda-release", "inherits": "darwin-conda-debug", "cacheVariables": { "CMAKE_BUILD_TYPE": "RelWithDebInfo" } } ], diff --git a/cpp/third_party/vcpkg_overlays/triplets/x64-linux-mix.cmake b/cpp/third_party/vcpkg_overlays/triplets/x64-linux-mix.cmake new file mode 100644 index 0000000000..071f60aa88 --- /dev/null +++ b/cpp/third_party/vcpkg_overlays/triplets/x64-linux-mix.cmake @@ -0,0 +1,9 @@ +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_CMAKE_SYSTEM_NAME Linux) + +if(${PORT} MATCHES "openssl") + set(VCPKG_LIBRARY_LINKAGE dynamic) +else() + set(VCPKG_LIBRARY_LINKAGE static) +endif() \ No newline at end of file diff --git a/docs/mkdocs/docs/error_messages.md b/docs/mkdocs/docs/error_messages.md index 4ba610cbb4..130fc4e02e 100644 --- a/docs/mkdocs/docs/error_messages.md +++ b/docs/mkdocs/docs/error_messages.md @@ -183,3 +183,8 @@ RuntimeError |-- StreamDescriptorMismatch └-- InternalException ``` + +## Import Error +| Error messages | Cause | Resolution | +|:--------------|:-------|:-----------| +| libssl.so.1.1: cannot open shared object file | For linux-pypi prebuilt binary, since v4.5.0, libssl is no longer statically linked | Install libssl v1.1.1 manually; Please refer to https://github.com/man-group/ArcticDB | \ No newline at end of file From cae6960b278a9ffeb243eb683f2d41be73f50bda Mon Sep 17 00:00:00 2001 From: phoebusm Date: Wed, 10 Apr 2024 16:27:50 +0100 Subject: [PATCH 2/3] Upgrade to openssl to v3 in pypi --- .github/workflows/build_steps.yml | 10 +--------- README.md | 10 ---------- cpp/vcpkg.json | 2 +- docs/mkdocs/docs/error_messages.md | 2 +- 4 files changed, 3 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build_steps.yml b/.github/workflows/build_steps.yml index 53afc50275..bfaacea33e 100644 --- a/.github/workflows/build_steps.yml +++ b/.github/workflows/build_steps.yml @@ -134,10 +134,6 @@ jobs: if: inputs.job_type == 'cpp-tests' run: cd cpp; cmake --build --preset $ARCTIC_CMAKE_PRESET --target install - - name: Install dynamically linked library for tests - if: inputs.job_type == 'cpp-tests' && matrix.os == 'linux' - run: yum install -y https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/o/openssl11-1.1.1k-7.el7.x86_64.rpm - - name: C++ Rapidcheck if: inputs.job_type == 'cpp-tests' run: cpp/out/install/arcticdb_rapidcheck_tests @@ -168,7 +164,7 @@ jobs: run: pipx run cibuildwheel==${{inputs.cibw_version}} env: CIBW_MANYLINUX_X86_64_IMAGE: ${{inputs.cibw_image_tag}} - CIBW_REPAIR_WHEEL_COMMAND_LINUX: auditwheel repair -L . -w {dest_dir} {wheel} --exclude libssl-a0734ced.so.1.1 + CIBW_REPAIR_WHEEL_COMMAND_LINUX: auditwheel repair -L . -w {dest_dir} {wheel} --exclude libssl.so.3 - name: Store wheel artifact if: inputs.job_type == 'build-python-wheels' @@ -303,10 +299,6 @@ jobs: ${{vars.EXTRA_TEST_PREPARE_CMD || ''}} env: CMAKE_BUILD_PARALLEL_LEVEL: ${{vars.CMAKE_BUILD_PARALLEL_LEVEL}} - - - name: Install dynamically linked library for tests - if: matrix.os == 'linux' - run: yum install -y https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/o/openssl11-1.1.1k-7.el7.x86_64.rpm - name: Set persistent storage variables if: inputs.persistent_storage == 'true' diff --git a/README.md b/README.md index 75fdeebd8b..538f2ed296 100644 --- a/README.md +++ b/README.md @@ -70,16 +70,6 @@ Install ArcticDB: ```bash $ pip install arcticdb -# For linux-pypi prebuilt binary, since v4.5.0, libssl is no longer statically linked. -# openssl v1.1.1 is needed to be manually installed. -# openssl v1.1.1 can be installed with precompiled package online, if not available in the OS: -# RHEL: -$ yum install -y compat-openssl11 -# or -$ yum install -y https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/o/openssl11-1.1.1k-7.el7.x86_64.rpm -# Ubuntu: -# Please go to http://security.ubuntu.com/ubuntu/pool/main/o/openssl/ to -# download and install latest openssl_1.1.1 and libssl1.1_1.1.1 deb files in sequence ``` or using conda-forge ```bash diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json index d532d8305b..30832112d9 100644 --- a/cpp/vcpkg.json +++ b/cpp/vcpkg.json @@ -67,7 +67,7 @@ "benchmark" ], "overrides": [ - { "name": "openssl", "version-string": "1.1.1n#1" }, + { "name": "openssl", "version-string": "3.2.1" }, { "name": "aws-sdk-cpp", "version": "1.11.201" }, { "name": "aws-c-s3", "version": "0.3.24" }, { "name": "bitmagic", "version": "7.12.3" }, diff --git a/docs/mkdocs/docs/error_messages.md b/docs/mkdocs/docs/error_messages.md index 130fc4e02e..122ec8a170 100644 --- a/docs/mkdocs/docs/error_messages.md +++ b/docs/mkdocs/docs/error_messages.md @@ -187,4 +187,4 @@ RuntimeError ## Import Error | Error messages | Cause | Resolution | |:--------------|:-------|:-----------| -| libssl.so.1.1: cannot open shared object file | For linux-pypi prebuilt binary, since v4.5.0, libssl is no longer statically linked | Install libssl v1.1.1 manually; Please refer to https://github.com/man-group/ArcticDB | \ No newline at end of file +| libssl.so.3: cannot open shared object file | For linux-pypi prebuilt binary, since v4.5.0, libssl is no longer statically linked | Install libssl v3 manually | \ No newline at end of file From cbba9460251d8e7ec012c52b8216db76f3239f74 Mon Sep 17 00:00:00 2001 From: phoebusm Date: Thu, 11 Apr 2024 07:40:41 +0100 Subject: [PATCH 3/3] Fix --- cpp/vcpkg.json | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json index 30832112d9..c58f526f83 100644 --- a/cpp/vcpkg.json +++ b/cpp/vcpkg.json @@ -67,7 +67,6 @@ "benchmark" ], "overrides": [ - { "name": "openssl", "version-string": "3.2.1" }, { "name": "aws-sdk-cpp", "version": "1.11.201" }, { "name": "aws-c-s3", "version": "0.3.24" }, { "name": "bitmagic", "version": "7.12.3" },