Expose a private method in aggregator

.gitignore

@@ -12,6 +12,7 @@
 /build
 /build_*
 /build-*
+/builds
 /tests/venv
 /obj-x86_64-linux-gnu/
 
@@ -169,3 +170,8 @@ tests/integration/**/_gen
 # It is autogenerated from *.in
 /rust/**/.cargo/config.toml
 /rust/**/vendor
+
+utils/local-engine/tests/testConfig.h
+utils/extern-local-engine
+
+.history

CMakeLists.txt

@@ -21,8 +21,11 @@ include (cmake/clang_tidy.cmake)
 include (cmake/git.cmake)
 include (cmake/utils.cmake)
 
+# This is needed to set up the CMAKE_INSTALL_BINDIR variable.
+include (GNUInstallDirs)
+
 # Ignore export() since we don't use it,
-# but it gets broken with a global targets via link_libraries()
+# but it gets broken with global targets via link_libraries()
 macro (export)
 endmacro ()
 
@@ -144,6 +147,9 @@ if (COMPILER_CLANG)
         set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xclang -fuse-ctor-homing")
     endif()
 
+    # uncommment this to analyze flame graph
+    # set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -fno-optimize-sibling-calls")
+    # set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -fno-optimize-sibling-calls")
     no_warning(enum-constexpr-conversion) # breaks Protobuf in clang-16
 endif ()
 
@@ -380,6 +386,8 @@ endif ()
 
 include(cmake/dbms_glob_sources.cmake)
 
+set (CMAKE_POSITION_INDEPENDENT_CODE ON)
+
 add_library(global-group INTERFACE)
 if (OS_LINUX OR OS_ANDROID)
     include(cmake/linux/default_libs.cmake)
@@ -422,10 +430,6 @@ endif ()
 
 set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX")
 
-if (NOT SANITIZE)
-    set (CMAKE_POSITION_INDEPENDENT_CODE OFF)
-endif()
-
 if (OS_LINUX AND NOT (ARCH_AARCH64 OR ARCH_S390X) AND NOT SANITIZE)
     # Slightly more efficient code can be generated
     # It's disabled for ARM because otherwise ClickHouse cannot run on Android.
@@ -460,14 +464,6 @@ endif ()
 
 message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE}")
 
-include (GNUInstallDirs)
-
-# When testing for memory leaks with Valgrind, don't link tcmalloc or jemalloc.
-
-if (TARGET global-group)
-    install (EXPORT global DESTINATION cmake)
-endif ()
-
 add_subdirectory (contrib EXCLUDE_FROM_ALL)
 
 if (NOT ENABLE_JEMALLOC)
@@ -488,8 +484,10 @@ macro (clickhouse_add_executable target)
     if (${type} STREQUAL EXECUTABLE)
         # disabled for TSAN and gcc since libtsan.a provides overrides too
         if (TARGET clickhouse_new_delete)
-            # operator::new/delete for executables (MemoryTracker stuff)
-            target_link_libraries (${target} PRIVATE clickhouse_new_delete)
+            if (NOT ${target} STREQUAL Git::Git)
+                # operator::new/delete for executables (MemoryTracker stuff)
+                target_link_libraries (${target} PRIVATE clickhouse_new_delete ${MALLOC_LIBRARIES})
+            endif()
         endif()
 
         # In case of static jemalloc, because zone_register() is located in zone.c and
@@ -561,7 +559,6 @@ option(CHECK_LARGE_OBJECT_SIZES "Check that there are no large object files afte
 add_subdirectory (base)
 add_subdirectory (src)
 add_subdirectory (programs)
-add_subdirectory (tests)
 add_subdirectory (utils)
 
 if (FUZZER)

base/glibc-compatibility/CMakeLists.txt

@@ -35,12 +35,6 @@ if (GLIBC_COMPATIBILITY)
 
     target_link_libraries(global-libs INTERFACE glibc-compatibility ${MEMCPY_LIBRARY})
 
-    install(
-        TARGETS glibc-compatibility ${MEMCPY_LIBRARY}
-        EXPORT global
-        ARCHIVE DESTINATION lib
-    )
-
     message (STATUS "Some symbols from glibc will be replaced for compatibility")
 
 elseif (CLICKHOUSE_OFFICIAL_BUILD)

base/harmful/CMakeLists.txt

@@ -1,2 +1 @@
 add_library(harmful harmful.c)
-install(TARGETS harmful EXPORT global ARCHIVE DESTINATION lib)

base/poco/Net/src/HTTPServerSession.cpp

@@ -26,7 +26,6 @@ HTTPServerSession::HTTPServerSession(const StreamSocket& socket, HTTPServerParam
 	_maxKeepAliveRequests(pParams->getMaxKeepAliveRequests())
 {
 	setTimeout(pParams->getTimeout());
-	this->socket().setReceiveTimeout(pParams->getTimeout());
 }
 
 

base/poco/Net/src/HTTPSession.cpp

@@ -93,9 +93,34 @@ void HTTPSession::setTimeout(const Poco::Timespan& timeout)
 
 void HTTPSession::setTimeout(const Poco::Timespan& connectionTimeout, const Poco::Timespan& sendTimeout, const Poco::Timespan& receiveTimeout)
 {
-	 _connectionTimeout = connectionTimeout;
-	 _sendTimeout = sendTimeout;
-	 _receiveTimeout = receiveTimeout;
+     try
+     {
+         _connectionTimeout = connectionTimeout;
+
+         if (_sendTimeout.totalMicroseconds() != sendTimeout.totalMicroseconds()) {
+             _sendTimeout = sendTimeout;
+
+             if (connected())
+                 _socket.setSendTimeout(_sendTimeout);
+         }
+
+         if (_receiveTimeout.totalMicroseconds() != receiveTimeout.totalMicroseconds()) {
+             _receiveTimeout = receiveTimeout;
+
+             if (connected())
+                 _socket.setReceiveTimeout(_receiveTimeout);
+         }
+     }
+     catch (NetException &)
+     {
+#ifndef NDEBUG
+         throw;
+#else
+         // mute exceptions in release
+         // just in case when changing settings on socket is not allowed
+         // however it should be OK for timeouts
+#endif
+     }
 }
 
 

cmake/ccache.cmake

@@ -9,10 +9,10 @@ if (CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" OR CMAKE_C_COMPILER_LAUNCHER MA
     return()
 endif()
 
-set(COMPILER_CACHE "auto" CACHE STRING "Speedup re-compilations using the caching tools; valid options are 'auto' (ccache, then sccache), 'ccache', 'sccache', or 'disabled'")
+set(COMPILER_CACHE "auto" CACHE STRING "Speedup re-compilations using the caching tools; valid options are 'auto' (sccache, then ccache), 'ccache', 'sccache', or 'disabled'")
 
 if(COMPILER_CACHE STREQUAL "auto")
-    find_program (CCACHE_EXECUTABLE NAMES ccache sccache)
+    find_program (CCACHE_EXECUTABLE NAMES sccache ccache)
 elseif (COMPILER_CACHE STREQUAL "ccache")
     find_program (CCACHE_EXECUTABLE ccache)
 elseif(COMPILER_CACHE STREQUAL "sccache")
@@ -21,7 +21,7 @@ elseif(COMPILER_CACHE STREQUAL "disabled")
     message(STATUS "Using *ccache: no (disabled via configuration)")
     return()
 else()
-    message(${RECONFIGURE_MESSAGE_LEVEL} "The COMPILER_CACHE must be one of (auto|ccache|sccache|disabled), value: '${COMPILER_CACHE}'")
+    message(${RECONFIGURE_MESSAGE_LEVEL} "The COMPILER_CACHE must be one of (auto|sccache|ccache|disabled), value: '${COMPILER_CACHE}'")
 endif()
 
 

cmake/cpu_features.cmake

@@ -134,60 +134,52 @@ elseif (ARCH_AMD64)
     # ClickHouse can be cross-compiled (e.g. on an ARM host for x86) but it is also possible to build ClickHouse on x86 w/o AVX for x86 w/
     # AVX. We only assume that the compiler can emit certain SIMD instructions, we don't care if the host system is able to run the binary.
 
-    SET (HAVE_SSSE3 1)
-    SET (HAVE_SSE41 1)
-    SET (HAVE_SSE42 1)
-    SET (HAVE_PCLMULQDQ 1)
-    SET (HAVE_POPCNT 1)
-    SET (HAVE_AVX 1)
-    SET (HAVE_AVX2 1)
-    SET (HAVE_AVX512 1)
-    SET (HAVE_AVX512_VBMI 1)
-    SET (HAVE_BMI 1)
-    SET (HAVE_BMI2 1)
-
-    if (HAVE_SSSE3 AND ENABLE_SSSE3)
+    if (ENABLE_SSSE3)
         set (COMPILER_FLAGS "${COMPILER_FLAGS} -mssse3")
     endif ()
-    if (HAVE_SSE41 AND ENABLE_SSE41)
+
+    if (ENABLE_SSE41)
         set (COMPILER_FLAGS "${COMPILER_FLAGS} -msse4.1")
     endif ()
-    if (HAVE_SSE42 AND ENABLE_SSE42)
+
+    if (ENABLE_SSE42)
         set (COMPILER_FLAGS "${COMPILER_FLAGS} -msse4.2")
     endif ()
-    if (HAVE_PCLMULQDQ AND ENABLE_PCLMULQDQ)
+
+    if (ENABLE_PCLMULQDQ)
         set (COMPILER_FLAGS "${COMPILER_FLAGS} -mpclmul")
     endif ()
-    if (HAVE_POPCNT AND ENABLE_POPCNT)
+
+    if (ENABLE_BMI)
+        set (COMPILER_FLAGS "${COMPILER_FLAGS} -mbmi")
+    endif ()
+
+    if (ENABLE_POPCNT)
         set (COMPILER_FLAGS "${COMPILER_FLAGS} -mpopcnt")
     endif ()
-    if (HAVE_AVX AND ENABLE_AVX)
+
+    if (ENABLE_AVX)
         set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx")
     endif ()
-    if (HAVE_AVX2 AND ENABLE_AVX2)
+
+    if (ENABLE_AVX2)
         set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx2")
+        if (ENABLE_BMI2)
+            set (COMPILER_FLAGS "${COMPILER_FLAGS} -mbmi2")
+        endif ()
     endif ()
-    if (HAVE_AVX512 AND ENABLE_AVX512)
+
+    if (ENABLE_AVX512)
         set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx512f -mavx512bw -mavx512vl")
+        if (ENABLE_AVX512_VBMI)
+            set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx512vbmi")
+        endif ()
     endif ()
-    if (HAVE_AVX512 AND ENABLE_AVX512 AND HAVE_AVX512_VBMI AND ENABLE_AVX512_VBMI)
-        set (COMPILER_FLAGS "${COMPILER_FLAGS} -mavx512vbmi")
-    endif ()
-    if (HAVE_BMI AND ENABLE_BMI)
-        set (COMPILER_FLAGS "${COMPILER_FLAGS} -mbmi")
-    endif ()
-    if (HAVE_BMI2 AND HAVE_AVX2 AND ENABLE_AVX2 AND ENABLE_BMI2)
-        set (COMPILER_FLAGS "${COMPILER_FLAGS} -mbmi2")
-    endif ()
+
     if (ENABLE_AVX512_FOR_SPEC_OP)
-        set (X86_INTRINSICS_FLAGS "")
-        if (HAVE_BMI)
-            set (X86_INTRINSICS_FLAGS "${X86_INTRINSICS_FLAGS} -mbmi")
-        endif ()
-        if (HAVE_AVX512)
-            set (X86_INTRINSICS_FLAGS "${X86_INTRINSICS_FLAGS} -mavx512f -mavx512bw -mavx512vl -mprefer-vector-width=256")
-        endif ()
+        set (X86_INTRINSICS_FLAGS "-mbmi -mavx512f -mavx512bw -mavx512vl -mprefer-vector-width=256")
     endif ()
+
 else ()
     # RISC-V + exotic platforms
 endif ()

cmake/darwin/default_libs.cmake

@@ -22,9 +22,3 @@ link_libraries(global-group)
 target_link_libraries(global-group INTERFACE
     $<TARGET_PROPERTY:global-libs,INTERFACE_LINK_LIBRARIES>
 )
-
-# FIXME: remove when all contribs will get custom cmake lists
-install(
-    TARGETS global-group global-libs
-    EXPORT global
-)

cmake/darwin/toolchain-aarch64.cmake

@@ -9,9 +9,3 @@ set (CMAKE_ASM_COMPILER_TARGET "aarch64-apple-darwin")
 set (CMAKE_OSX_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/darwin-aarch64")
 
 set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)  # disable linkage check - it doesn't work in CMake
-
-set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
-set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
-
-set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
-set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)

cmake/darwin/toolchain-x86_64.cmake

@@ -9,9 +9,3 @@ set (CMAKE_ASM_COMPILER_TARGET "x86_64-apple-darwin")
 set (CMAKE_OSX_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/darwin-x86_64")
 
 set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)  # disable linkage check - it doesn't work in CMake
-
-set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
-set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
-
-set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
-set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)

cmake/freebsd/default_libs.cmake

@@ -25,9 +25,3 @@ link_libraries(global-group)
 target_link_libraries(global-group INTERFACE
     $<TARGET_PROPERTY:global-libs,INTERFACE_LINK_LIBRARIES>
 )
-
-# FIXME: remove when all contribs will get custom cmake lists
-install(
-    TARGETS global-group global-libs
-    EXPORT global
-)

cmake/freebsd/toolchain-aarch64.cmake

@@ -9,13 +9,3 @@ set (CMAKE_ASM_COMPILER_TARGET "aarch64-unknown-freebsd12")
 set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/freebsd-aarch64")
 
 set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)  # disable linkage check - it doesn't work in CMake
-
-# Will be changed later, but somehow needed to be set here.
-set (CMAKE_AR "ar")
-set (CMAKE_RANLIB "ranlib")
-
-set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
-set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
-
-set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
-set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)

cmake/freebsd/toolchain-ppc64le.cmake

@@ -9,13 +9,3 @@ set (CMAKE_ASM_COMPILER_TARGET "powerpc64le-unknown-freebsd13")
 set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/freebsd-ppc64le")
 
 set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)  # disable linkage check - it doesn't work in CMake
-
-# Will be changed later, but somehow needed to be set here.
-set (CMAKE_AR "ar")
-set (CMAKE_RANLIB "ranlib")
-
-set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
-set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
-
-set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
-set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)

cmake/freebsd/toolchain-x86_64.cmake

@@ -9,13 +9,3 @@ set (CMAKE_ASM_COMPILER_TARGET "x86_64-pc-freebsd11")
 set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/freebsd-x86_64")
 
 set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)  # disable linkage check - it doesn't work in CMake
-
-# Will be changed later, but somehow needed to be set here.
-set (CMAKE_AR "ar")
-set (CMAKE_RANLIB "ranlib")
-
-set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
-set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
-
-set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
-set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)

cmake/limit_jobs.cmake

@@ -21,7 +21,7 @@ if (NOT PARALLEL_COMPILE_JOBS AND MAX_COMPILER_MEMORY)
         set (PARALLEL_COMPILE_JOBS 1)
     endif ()
     if (PARALLEL_COMPILE_JOBS LESS NUMBER_OF_LOGICAL_CORES)
-        message(WARNING "The auto-calculated compile jobs limit (${PARALLEL_COMPILE_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_COMPILE_JOBS to override.")
+        message("The auto-calculated compile jobs limit (${PARALLEL_COMPILE_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_COMPILE_JOBS to override.")
     endif()
 endif ()
 
@@ -32,7 +32,7 @@ if (NOT PARALLEL_LINK_JOBS AND MAX_LINKER_MEMORY)
         set (PARALLEL_LINK_JOBS 1)
     endif ()
     if (PARALLEL_LINK_JOBS LESS NUMBER_OF_LOGICAL_CORES)
-        message(WARNING "The auto-calculated link jobs limit (${PARALLEL_LINK_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_LINK_JOBS to override.")
+        message("The auto-calculated link jobs limit (${PARALLEL_LINK_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_LINK_JOBS to override.")
     endif()
 endif ()
 

cmake/linux/default_libs.cmake

@@ -50,9 +50,3 @@ target_link_libraries(global-group INTERFACE
     $<TARGET_PROPERTY:global-libs,INTERFACE_LINK_LIBRARIES>
     -Wl,--end-group
 )
-
-# FIXME: remove when all contribs will get custom cmake lists
-install(
-    TARGETS global-group global-libs
-    EXPORT global
-)