From 5d7882bb601573d0929ff3578c321469284b3eec Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 11 Apr 2024 16:50:01 +0200 Subject: [PATCH 1/7] Always use new analyzer in perf tests --- tests/performance/scripts/eqmed.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/performance/scripts/eqmed.sql b/tests/performance/scripts/eqmed.sql index 372534f6f48b..94e6733a3d71 100644 --- a/tests/performance/scripts/eqmed.sql +++ b/tests/performance/scripts/eqmed.sql @@ -67,5 +67,4 @@ from select throwIf(uniq((test, query)) != 1) from table ) check_single_query -- this subselect checks that there is only one query in the input table; -- written this way so that it is not optimized away (#10523) -SETTINGS allow_experimental_analyzer = 0 ; From 1eaa21ff8c7148341717c2f53f69f315bc8a87da Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Fri, 12 Apr 2024 15:10:34 +0800 Subject: [PATCH 2/7] improve utc timestamp transform --- src/Functions/UTCTimestampTransform.cpp | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/Functions/UTCTimestampTransform.cpp b/src/Functions/UTCTimestampTransform.cpp index 77cc1826941a..ba9be88fd2bf 100644 --- a/src/Functions/UTCTimestampTransform.cpp +++ b/src/Functions/UTCTimestampTransform.cpp @@ -67,7 +67,7 @@ namespace return date_time_type; } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) const override { if (arguments.size() != 2) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {}'s arguments number must be 2.", name); @@ -77,24 +77,33 @@ namespace if (!time_zone_const_col) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of 2nd argument of function {}. Excepted const(String).", arg2.column->getName(), name); String time_zone_val = time_zone_const_col->getDataAt(0).toString(); - auto column = result_type->createColumn(); if (WhichDataType(arg1.type).isDateTime()) { const auto * date_time_col = checkAndGetColumn(arg1.column.get()); - for (size_t i = 0; i < date_time_col->size(); ++i) + size_t col_size = date_time_col->size(); + using ColVecTo = DataTypeDateTime::ColumnType; + typename ColVecTo::MutablePtr result_column = ColVecTo::create(col_size); + typename ColVecTo::Container & result_data = result_column->getData(); + for (size_t i = 0; i < col_size; ++i) { UInt32 date_time_val = date_time_col->getElement(i); LocalDateTime date_time(date_time_val, Name::to ? DateLUT::instance("UTC") : DateLUT::instance(time_zone_val)); time_t time_val = date_time.to_time_t(Name::from ? DateLUT::instance("UTC") : DateLUT::instance(time_zone_val)); - column->insert(time_val); + result_data[i] = static_cast(time_val); } + return result_column; } else if (WhichDataType(arg1.type).isDateTime64()) { const auto * date_time_col = checkAndGetColumn(arg1.column.get()); + size_t col_size = date_time_col->size(); const DataTypeDateTime64 * date_time_type = static_cast(arg1.type.get()); - Int64 scale_multiplier = DecimalUtils::scaleMultiplier(date_time_type->getScale()); - for (size_t i = 0; i < date_time_col->size(); ++i) + UInt32 col_scale = date_time_type->getScale(); + Int64 scale_multiplier = DecimalUtils::scaleMultiplier(col_scale); + using ColDecimalTo = DataTypeDateTime64::ColumnType; + typename ColDecimalTo::MutablePtr result_column = ColDecimalTo::create(col_size, col_scale); + typename ColDecimalTo::Container & result_data = result_column->getData(); + for (size_t i = 0; i < col_size; ++i) { DateTime64 date_time_val = date_time_col->getElement(i); Int64 seconds = date_time_val.value / scale_multiplier; @@ -102,12 +111,12 @@ namespace LocalDateTime date_time(seconds, Name::to ? DateLUT::instance("UTC") : DateLUT::instance(time_zone_val)); time_t time_val = date_time.to_time_t(Name::from ? DateLUT::instance("UTC") : DateLUT::instance(time_zone_val)); DateTime64 date_time_64(time_val * scale_multiplier + micros); - column->insert(date_time_64); + result_data[i] = date_time_64; } + return result_column; } else throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function {}'s 1st argument can only be datetime/datatime64. ", name); - return column; } }; From 6a96ee12e9de3081443f4652593192f67c221142 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Fri, 12 Apr 2024 16:21:51 +0800 Subject: [PATCH 3/7] Improve utc timestamp transfrom --- src/Functions/UTCTimestampTransform.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Functions/UTCTimestampTransform.cpp b/src/Functions/UTCTimestampTransform.cpp index ba9be88fd2bf..b2398a978178 100644 --- a/src/Functions/UTCTimestampTransform.cpp +++ b/src/Functions/UTCTimestampTransform.cpp @@ -77,6 +77,8 @@ namespace if (!time_zone_const_col) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of 2nd argument of function {}. Excepted const(String).", arg2.column->getName(), name); String time_zone_val = time_zone_const_col->getDataAt(0).toString(); + time_t date_time_val = LocalDateTime(0, DateLUT::instance(time_zone_val)).to_time_t(); + time_t utc_time_val = LocalDateTime(0, DateLUT::instance("UTC")).to_time_t(); if (WhichDataType(arg1.type).isDateTime()) { const auto * date_time_col = checkAndGetColumn(arg1.column.get()); @@ -86,9 +88,8 @@ namespace typename ColVecTo::Container & result_data = result_column->getData(); for (size_t i = 0; i < col_size; ++i) { - UInt32 date_time_val = date_time_col->getElement(i); - LocalDateTime date_time(date_time_val, Name::to ? DateLUT::instance("UTC") : DateLUT::instance(time_zone_val)); - time_t time_val = date_time.to_time_t(Name::from ? DateLUT::instance("UTC") : DateLUT::instance(time_zone_val)); + UInt32 val = date_time_col->getElement(i); + time_t time_val = Name::to ? val + utc_time_val - date_time_val : val + date_time_val - utc_time_val; result_data[i] = static_cast(time_val); } return result_column; @@ -105,12 +106,11 @@ namespace typename ColDecimalTo::Container & result_data = result_column->getData(); for (size_t i = 0; i < col_size; ++i) { - DateTime64 date_time_val = date_time_col->getElement(i); - Int64 seconds = date_time_val.value / scale_multiplier; - Int64 micros = date_time_val.value % scale_multiplier; - LocalDateTime date_time(seconds, Name::to ? DateLUT::instance("UTC") : DateLUT::instance(time_zone_val)); - time_t time_val = date_time.to_time_t(Name::from ? DateLUT::instance("UTC") : DateLUT::instance(time_zone_val)); - DateTime64 date_time_64(time_val * scale_multiplier + micros); + DateTime64 val = date_time_col->getElement(i); + Int64 seconds = val.value / scale_multiplier; + Int64 mills = val.value % scale_multiplier; + time_t time_val = Name::from ? seconds + date_time_val - utc_time_val : seconds + utc_time_val - date_time_val; + DateTime64 date_time_64(time_val * scale_multiplier + mills); result_data[i] = date_time_64; } return result_column; From 878e0ed26b889ae60c83409e4d326f43363fd8b6 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Fri, 12 Apr 2024 16:37:35 +0800 Subject: [PATCH 4/7] add performance test --- tests/performance/utc_timestamp_transform.xml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 tests/performance/utc_timestamp_transform.xml diff --git a/tests/performance/utc_timestamp_transform.xml b/tests/performance/utc_timestamp_transform.xml new file mode 100644 index 000000000000..b3b8777bdda5 --- /dev/null +++ b/tests/performance/utc_timestamp_transform.xml @@ -0,0 +1,15 @@ + + CREATE TABLE test1(d DateTime) ENGINE Memory + CREATE TABLE test2(d DateTime64) ENGINE Memory + INSERT INTO test1 SELECT toDateTime('2023-03-16 11:22:33') + number from numbers(10000000) + INSERT INTO test2 SELECT toDateTime64('2023-03-16 11:22:33', 3) + number from numbers(10000000); + + + select count(1) from test1 where to_utc_timestamp(d, 'Etc/GMT+1') > '1990-01-01 12:00:00' SETTINGS max_threads=1 + select count(1) from test1 where from_utc_timestamp(d, 'Etc/GMT+1') > '1990-01-01 12:00:00' SETTINGS max_threads=1 + select count(1) from test2 where to_utc_timestamp(d, 'Etc/GMT+1') > '1990-01-01 12:00:00' SETTINGS max_threads=1 + select count(1) from test2 where from_utc_timestamp(d, 'Etc/GMT+1') > '1990-01-01 12:00:00' SETTINGS max_threads=1 + + DROP TABLE IF EXISTS test1 + DROP TABLE IF EXISTS test2 + \ No newline at end of file From a9711c4bec60d01d99d5b9f29b55da9989f34ccc Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Fri, 12 Apr 2024 16:57:34 +0800 Subject: [PATCH 5/7] modify test --- tests/performance/utc_timestamp_transform.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/performance/utc_timestamp_transform.xml b/tests/performance/utc_timestamp_transform.xml index b3b8777bdda5..b3ec21129d5e 100644 --- a/tests/performance/utc_timestamp_transform.xml +++ b/tests/performance/utc_timestamp_transform.xml @@ -2,8 +2,7 @@ CREATE TABLE test1(d DateTime) ENGINE Memory CREATE TABLE test2(d DateTime64) ENGINE Memory INSERT INTO test1 SELECT toDateTime('2023-03-16 11:22:33') + number from numbers(10000000) - INSERT INTO test2 SELECT toDateTime64('2023-03-16 11:22:33', 3) + number from numbers(10000000); - + INSERT INTO test2 SELECT toDateTime64('2023-03-16 11:22:33', 3) + number from numbers(10000000) select count(1) from test1 where to_utc_timestamp(d, 'Etc/GMT+1') > '1990-01-01 12:00:00' SETTINGS max_threads=1 select count(1) from test1 where from_utc_timestamp(d, 'Etc/GMT+1') > '1990-01-01 12:00:00' SETTINGS max_threads=1 From 4a50999fcc48274737c841e31591a48c148450da Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Mon, 15 Apr 2024 14:12:54 +0800 Subject: [PATCH 6/7] remove logical of cache value of different timezone --- src/Functions/UTCTimestampTransform.cpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/Functions/UTCTimestampTransform.cpp b/src/Functions/UTCTimestampTransform.cpp index b2398a978178..0fcba91f49fc 100644 --- a/src/Functions/UTCTimestampTransform.cpp +++ b/src/Functions/UTCTimestampTransform.cpp @@ -77,8 +77,7 @@ namespace if (!time_zone_const_col) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of 2nd argument of function {}. Excepted const(String).", arg2.column->getName(), name); String time_zone_val = time_zone_const_col->getDataAt(0).toString(); - time_t date_time_val = LocalDateTime(0, DateLUT::instance(time_zone_val)).to_time_t(); - time_t utc_time_val = LocalDateTime(0, DateLUT::instance("UTC")).to_time_t(); + const DateLUTImpl & utc_time_zone = DateLUT::instance("UTC"); if (WhichDataType(arg1.type).isDateTime()) { const auto * date_time_col = checkAndGetColumn(arg1.column.get()); @@ -88,8 +87,9 @@ namespace typename ColVecTo::Container & result_data = result_column->getData(); for (size_t i = 0; i < col_size; ++i) { - UInt32 val = date_time_col->getElement(i); - time_t time_val = Name::to ? val + utc_time_val - date_time_val : val + date_time_val - utc_time_val; + UInt32 date_time_val = date_time_col->getElement(i); + LocalDateTime date_time(date_time_val, Name::to ? utc_time_zone : DateLUT::instance(time_zone_val)); + time_t time_val = date_time.to_time_t(Name::from ? utc_time_zone : DateLUT::instance(time_zone_val)); result_data[i] = static_cast(time_val); } return result_column; @@ -106,11 +106,12 @@ namespace typename ColDecimalTo::Container & result_data = result_column->getData(); for (size_t i = 0; i < col_size; ++i) { - DateTime64 val = date_time_col->getElement(i); - Int64 seconds = val.value / scale_multiplier; - Int64 mills = val.value % scale_multiplier; - time_t time_val = Name::from ? seconds + date_time_val - utc_time_val : seconds + utc_time_val - date_time_val; - DateTime64 date_time_64(time_val * scale_multiplier + mills); + DateTime64 date_time_val = date_time_col->getElement(i); + Int64 seconds = date_time_val.value / scale_multiplier; + Int64 micros = date_time_val.value % scale_multiplier; + LocalDateTime date_time(seconds, Name::to ? utc_time_zone : DateLUT::instance(time_zone_val)); + time_t time_val = date_time.to_time_t(Name::from ? utc_time_zone : DateLUT::instance(time_zone_val)); + DateTime64 date_time_64(time_val * scale_multiplier + micros); result_data[i] = date_time_64; } return result_column; From 42e441d27a95ac868e333f88bd50bf12fb6c7d56 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 15 Apr 2024 17:32:28 +0200 Subject: [PATCH 7/7] Update TransactionLog.cpp --- src/Interpreters/TransactionLog.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Interpreters/TransactionLog.cpp b/src/Interpreters/TransactionLog.cpp index e0c5d01e1538..fb7d60a7a2c0 100644 --- a/src/Interpreters/TransactionLog.cpp +++ b/src/Interpreters/TransactionLog.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -451,6 +452,7 @@ CSN TransactionLog::commitTransaction(const MergeTreeTransactionPtr & txn, bool CSN TransactionLog::finalizeCommittedTransaction(MergeTreeTransaction * txn, CSN allocated_csn, scope_guard & state_guard) noexcept { LockMemoryExceptionInThread memory_tracker_lock(VariableContext::Global); + auto blocker = CannotAllocateThreadFaultInjector::blockFaultInjections(); chassert(!allocated_csn == txn->isReadOnly()); if (allocated_csn) {