<feat&fix>(txpool,executor): enable feature_calculate_gasPrice,featur…

…e_evm_timestamp, fix web3 tx memory not remove after commit.

CMakeLists.txt

@@ -6,7 +6,7 @@ endif()
 
 list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake)
 
-set(VERSION "3.11.0")
+set(VERSION "3.12.0")
 set(VERSION_SUFFIX "")
 include(Options)
 configure_project()

bcos-executor/src/executive/ExecutiveSerialFlow.cpp

@@ -67,10 +67,11 @@ void ExecutiveSerialFlow::asyncRun(std::function<void(CallParameters::UniquePtr)
 std::shared_ptr<TransactionExecutive> ExecutiveSerialFlow::buildExecutive(
     CallParameters::UniquePtr& input)
 {
-    bool isFeatureBalancePolicy1 =
-        m_executiveFactory->getBlockContext().features().get(ledger::Features::Flag::feature_balance_policy1);
+    auto const features = m_executiveFactory->getBlockContext().features();
+    const bool enableBilling = features.get(ledger::Features::Flag::feature_balance_policy1) ||
+                               features.get(ledger::Features::Flag::feature_calculate_gasPrice);
     return m_executiveFactory->build(input->codeAddress, input->contextID, input->seq,
-                isFeatureBalancePolicy1 ? ExecutiveType::billing : ExecutiveType::common);
+        enableBilling ? ExecutiveType::billing : ExecutiveType::common);
 }
 
 void ExecutiveSerialFlow::run(std::function<void(CallParameters::UniquePtr)> onTxReturn,

bcos-executor/src/executive/TransactionExecutive.cpp

@@ -307,8 +307,8 @@ CallParameters::UniquePtr TransactionExecutive::execute(CallParameters::UniquePt
                     co_await ledger::account::create(addr);
                 }
                 auto const nonceInStorage = co_await ledger::account::nonce(addr);
-                auto const baseNonce = u256(nonceInStorage.value_or("0"));
-                auto const newNonce = std::max(callNonce, baseNonce) + 1;
+                auto const storageNonce = u256(nonceInStorage.value_or("0"));
+                auto const newNonce = std::max(callNonce, storageNonce) + 1;
                 co_await ledger::account::setNonce(addr, newNonce.convert_to<std::string>());
             }(std::move(address), callParameters->nonce));
         }

bcos-executor/src/vm/HostContext.cpp

@@ -702,6 +702,11 @@ uint32_t HostContext::blockVersion() const
 
 uint64_t HostContext::timestamp() const
 {
+    if (m_executive->blockContext().features().get(ledger::Features::Flag::feature_evm_timestamp))
+    {
+        // millisecond to second
+        return m_executive->blockContext().timestamp() / 1000;
+    }
     return m_executive->blockContext().timestamp();
 }
 

bcos-framework/bcos-framework/ledger/Features.h

@@ -58,8 +58,9 @@ class Features
         feature_paillier_add_raw,
         feature_evm_cancun,
         feature_calculate_gasPrice,
+        feature_evm_timestamp,
         feature_evm_address,
-        feature_ethereum_compatible,  // will enbale all bugfixes, all features about evm
+        feature_ethereum_compatible,  // will enable all bugfixes, all features about evm
         feature_rpbft_term_weight,
     };
 
@@ -77,6 +78,11 @@ class Features
         return *value;
     }
 
+    static bool contains(std::string_view flag)
+    {
+        return magic_enum::enum_cast<Flag>(flag).has_value();
+    }
+
     void validate(std::string_view flag) const
     {
         auto value = magic_enum::enum_cast<Flag>(flag);
@@ -100,6 +106,13 @@ class Features
             BOOST_THROW_EXCEPTION(bcos::tool::InvalidSetFeature{}
                                   << errinfo_comment("must set feature_balance_precompiled first"));
         }
+        if ((flag == Flag::feature_calculate_gasPrice && get(Flag::feature_balance_policy1)) ||
+            (flag == Flag::feature_balance_policy1 && get(Flag::feature_calculate_gasPrice)))
+        {
+            BOOST_THROW_EXCEPTION(
+                bcos::tool::InvalidSetFeature{} << errinfo_comment(
+                    "feature_calculate_gasPrice can not be set with feature_balance_policy1"));
+        }
     }
 
     bool get(Flag flag) const
@@ -114,38 +127,48 @@ class Features
     }
     bool get(std::string_view flag) const { return get(string2Flag(flag)); }
 
-    void set(Flag flag)
-    {
-        auto index = magic_enum::enum_index(flag);
-        if (!index)
-        {
-            BOOST_THROW_EXCEPTION(NoSuchFeatureError{});
-        }
-
-        validate(flag);
-        m_flags[*index] = true;
-        turnOnMainSwitch(flag);
-    }
-
-    void turnOnMainSwitch(Flag flag)
+    // DO NOT use now, there is some action after set feature in systemPrecompiled
+    static auto getFeatureDependencies(Flag flag)
     {
+        /// NOTE：请不要在此处添加旧版本feature依赖！否则会出现数据不一致!
+        /// Do NOT add old version feature dependencies here! Otherwise, data inconsistency will
+        /// occur!
         const auto mainSwitchDependence = std::unordered_map<Flag, std::set<Flag>>(
             {{Flag::feature_ethereum_compatible, {
                                                      Flag::feature_balance,
                                                      Flag::feature_balance_precompiled,
                                                      Flag::feature_calculate_gasPrice,
+                                                     Flag::feature_evm_timestamp,
                                                      Flag::feature_evm_address,
                                                      Flag::feature_evm_cancun,
                                                  }}});
         if (mainSwitchDependence.contains(flag))
         {
-            for (const auto& dependence : mainSwitchDependence.at(flag))
-            {
-                set(dependence);
-            }
+            return mainSwitchDependence.at(flag);
+        }
+        return std::set<Flag>();
+    }
+    void enableDependencyFeatures(Flag flag)
+    {
+        for (const auto& dependence : getFeatureDependencies(flag))
+        {
+            set(dependence);
         }
     }
 
+    void set(Flag flag)
+    {
+        auto index = magic_enum::enum_index(flag);
+        if (!index)
+        {
+            BOOST_THROW_EXCEPTION(NoSuchFeatureError{});
+        }
+
+        validate(flag);
+        m_flags[*index] = true;
+        // enableDependencyFeatures(flag);
+    }
+
     void set(std::string_view flag) { set(string2Flag(flag)); }
 
     void setToShardingDefault(protocol::BlockVersion version)

bcos-framework/bcos-framework/protocol/Protocol.h

@@ -114,6 +114,7 @@ enum ProtocolVersion : uint32_t
 
 enum class BlockVersion : uint32_t
 {
+    V3_12_0_VERSION = 0x030c0000,  // 3.12.0
     V3_11_0_VERSION = 0x030b0000,
     V3_10_0_VERSION = 0x030a0000,
     V3_9_0_VERSION = 0x03090000,
@@ -153,10 +154,10 @@ const std::string RC4_VERSION_STR = "3.0.0-rc4";
 const std::string RC_VERSION_PREFIX = "3.0.0-rc";
 const std::string V3_9_VERSION_STR = "3.9.0";
 
-const BlockVersion DEFAULT_VERSION = bcos::protocol::BlockVersion::V3_11_0_VERSION;
+constexpr BlockVersion DEFAULT_VERSION = bcos::protocol::BlockVersion::V3_12_0_VERSION;
 const std::string DEFAULT_VERSION_STR = V3_9_VERSION_STR;
-const uint8_t MAX_MAJOR_VERSION = std::numeric_limits<uint8_t>::max();
-const uint8_t MIN_MAJOR_VERSION = 3;
+constexpr uint8_t MAX_MAJOR_VERSION = std::numeric_limits<uint8_t>::max();
+constexpr uint8_t MIN_MAJOR_VERSION = 3;
 
 [[nodiscard]] inline int versionCompareTo(
     std::variant<uint32_t, BlockVersion> const& _v1, BlockVersion const& _v2)

bcos-framework/test/unittests/interfaces/FeaturesTest.cpp

@@ -157,6 +157,7 @@ BOOST_AUTO_TEST_CASE(feature)
         "feature_paillier_add_raw",
         "feature_evm_cancun",
         "feature_calculate_gasPrice",
+        "feature_evm_timestamp",
         "feature_evm_address",
         "feature_ethereum_compatible",
         "feature_rpbft_term_weight",
@@ -394,4 +395,16 @@ BOOST_AUTO_TEST_CASE(genesis)
     }
 }
 
+
+BOOST_AUTO_TEST_CASE(testDependenciesFeatures)
+{
+    Features features;
+    features.set(ledger::Features::Flag::feature_ethereum_compatible);
+    // BOOST_CHECK(features.get(ledger::Features::Flag::feature_balance));
+    // BOOST_CHECK(features.get(ledger::Features::Flag::feature_balance_precompiled));
+    // BOOST_CHECK(features.get(ledger::Features::Flag::feature_calculate_gasPrice));
+    // BOOST_CHECK(features.get(ledger::Features::Flag::feature_evm_address));
+    // BOOST_CHECK(features.get(ledger::Features::Flag::feature_evm_cancun));
+}
+
 BOOST_AUTO_TEST_SUITE_END()

bcos-ledger/bcos-ledger/Ledger.cpp

@@ -438,14 +438,14 @@ task::Task<std::optional<ledger::StorageState>> Ledger::getStorageState(
     {
         co_return std::nullopt;
     }
-    state.nonce = std::stoull(std::string(nonceEntry->get()));
+    state.nonce = std::string(nonceEntry->get());
     auto const balanceEntry =
         co_await getStorageAt(_address, ACCOUNT_TABLE_FIELDS::BALANCE, _blockNumber);
     if (!balanceEntry.has_value())
     {
         co_return std::nullopt;
     }
-    state.balance = std::stoull(std::string(balanceEntry->get()));
+    state.balance = std::string(balanceEntry->get());
     co_return state;
 }
 

bcos-txpool/bcos-txpool/txpool/storage/MemoryStorage.cpp

@@ -694,8 +694,8 @@ void MemoryStorage::batchRemove(BlockNumber batchId, TransactionSubmitResults co
     auto updateLedgerNonceT = utcTime() - startT;
 
     startT = utcTime();
-    m_config->txValidator()->web3NonceChecker()->batchInsert(
-        RANGES::views::keys(web3NonceMap), RANGES::views::values(web3NonceMap));
+    task::wait(
+        m_config->txValidator()->web3NonceChecker()->updateNonceCache(std::move(web3NonceMap)));
     auto updateWeb3NonceT = utcTime() - startT;
 
     startT = utcTime();
@@ -720,7 +720,7 @@ void MemoryStorage::batchRemove(BlockNumber batchId, TransactionSubmitResults co
                      << LOG_KV("batchId", batchId) << LOG_KV("timecost", (utcTime() - recordT))
                      << LOG_KV("lockT", lockT) << LOG_KV("removeT", removeT)
                      << LOG_KV("updateLedgerNonceT", updateLedgerNonceT)
-                     << LOG_KV("updateWeb3NonceT", updateLedgerNonceT)
+                     << LOG_KV("updateWeb3NonceT", updateWeb3NonceT)
                      << LOG_KV("updateTxPoolNonceT", updateTxPoolNonceT);
 }
 
@@ -954,8 +954,8 @@ void MemoryStorage::removeInvalidTxs(bool lock)
             }
         }
         m_config->txPoolNonceChecker()->batchRemove(invalidNonceList);
-        m_config->txValidator()->web3NonceChecker()->batchRemoveMemoryNonce(
-            std::move(web3NonceMap));
+        task::wait(m_config->txValidator()->web3NonceChecker()->batchRemoveMemoryNonce(
+            std::move(web3NonceMap)));
 
         /*
         m_txsTable.batchRemove(txs2Remove | RANGES::views::keys,

bcos-txpool/bcos-txpool/txpool/validator/TxValidator.cpp

@@ -60,6 +60,8 @@ TransactionStatus TxValidator::verify(bcos::protocol::Transaction::ConstPtr _tx)
         _tx->setSystemTx(true);
     }
     m_txPoolNonceChecker->insert(_tx->nonce());
+    task::wait(
+        m_web3NonceChecker->insertMemoryNonce(std::string(_tx->sender()), u256(_tx->nonce())));
     return TransactionStatus::None;
 }
 

bcos-txpool/bcos-txpool/txpool/validator/Web3NonceChecker.cpp

@@ -19,8 +19,9 @@
  */
 
 #include "Web3NonceChecker.h"
+#include "../utilities/Common.h"
 #include <bcos-framework/storage2/Storage.h>
-#include <utilities/Common.h>
+#include <bcos-protocol/TransactionStatus.h>
 
 using namespace bcos;
 using namespace bcos::txpool;
@@ -31,6 +32,7 @@ task::Task<TransactionStatus> Web3NonceChecker::checkWeb3Nonce(
 {
     // sender is bytes view
     auto sender = std::string(_tx->sender());
+    auto const senderHex = toHex(sender);
     auto nonce = u256(_tx->nonce());
 
     // Note:
@@ -48,14 +50,14 @@ task::Task<TransactionStatus> Web3NonceChecker::checkWeb3Nonce(
                 nonce <= nonceInMemValue ||
                 nonce > nonceInMemValue + DEFAULT_WEB3_NONCE_CHECK_LIMIT)
             {
+                if (c_fileLogLevel == TRACE) [[unlikely]]
+                {
+                    TXPOOL_LOG(TRACE)
+                        << LOG_DESC("nonce check fail") << LOG_KV("sender", senderHex)
+                        << LOG_KV("nonce", nonce) << LOG_KV("nonceInMem", nonceInMemValue);
+                }
                 co_return TransactionStatus::NonceCheckFail;
             }
-            else
-            {
-                // update memory if nonce bigger than memory's
-                co_await storage2::writeOne(m_memoryNonces, sender, nonce);
-                co_return TransactionStatus::None;
-            }
         }
     }
     if (auto const nonceInLedger = co_await bcos::storage2::readOne(m_ledgerStateNonces, sender))
@@ -64,18 +66,17 @@ task::Task<TransactionStatus> Web3NonceChecker::checkWeb3Nonce(
             nonce < nonceInLedgerValue ||
             nonce > nonceInLedgerValue + DEFAULT_WEB3_NONCE_CHECK_LIMIT)
         {
+            if (c_fileLogLevel == TRACE) [[unlikely]]
+            {
+                TXPOOL_LOG(TRACE) << LOG_DESC("nonce check fail") << LOG_KV("sender", senderHex)
+                                  << LOG_KV("nonce", nonce)
+                                  << LOG_KV("nonceInLedger", nonceInLedgerValue);
+            }
             co_return TransactionStatus::NonceCheckFail;
         }
-        else
-        {
-            // update memory if nonce bigger than memory's
-            co_await storage2::writeOne(m_memoryNonces, sender, nonce);
-            co_return TransactionStatus::None;
-        }
     }
     // not in ledger memory, check from storage
     // TODO)): block number not use nowadays
-    auto const senderHex = toHex(sender);
     auto const storageState = co_await m_ledger->getStorageState(senderHex, 0);
     if (storageState.has_value())
     {
@@ -84,19 +85,25 @@ task::Task<TransactionStatus> Web3NonceChecker::checkWeb3Nonce(
         co_await storage2::writeOne(m_ledgerStateNonces, sender, nonceInStorage);
         if (nonce < nonceInStorage || nonce > nonceInStorage + DEFAULT_WEB3_NONCE_CHECK_LIMIT)
         {
+            if (c_fileLogLevel == TRACE) [[unlikely]]
+            {
+                TXPOOL_LOG(TRACE) << LOG_DESC("nonce check fail") << LOG_KV("sender", senderHex)
+                                  << LOG_KV("nonce", nonce)
+                                  << LOG_KV("nonceInStorage", nonceInStorage);
+            }
             co_return TransactionStatus::NonceCheckFail;
         }
     }
-    co_await storage2::writeOne(m_memoryNonces, sender, nonce);
     // TODO)): check balance？
     co_return TransactionStatus::None;
 }
 
-void Web3NonceChecker::batchRemoveMemoryNonce(std::unordered_map<std::string, u256> _nonceMap)
+task::Task<void> Web3NonceChecker::batchRemoveMemoryNonce(
+    std::unordered_map<std::string, u256> _nonceMap)
 {
     for (auto const& [sender, nonce] : _nonceMap)
     {
-        auto const nonceInMem = task::syncWait(storage2::readOne(m_memoryNonces, sender));
+        auto const nonceInMem = co_await storage2::readOne(m_memoryNonces, sender);
         if (nonceInMem.has_value())
         {
             // 假设交易池里有0xabcd的3笔交易，nonce分别是5，7，9。那么此时memory记录的nonce为9。
@@ -114,17 +121,44 @@ void Web3NonceChecker::batchRemoveMemoryNonce(std::unordered_map<std::string, u2
             // updated to 7, and the memory nonce will still be followed at this time.
             if (u256(nonceInMem.value()) > nonce)
             {
-                task::wait(storage2::removeOne(m_memoryNonces, sender));
+                co_await storage2::removeOne(m_memoryNonces, sender);
             }
         }
     }
+    co_return;
 }
 
 
-void Web3NonceChecker::batchInsert(
-    RANGES::input_range auto&& senders, RANGES::input_range auto&& nonces)
+task::Task<void> Web3NonceChecker::updateNonceCache(std::unordered_map<std::string, u256> map)
 {
-    task::wait(storage2::writeSome(m_ledgerStateNonces, senders, nonces));
+    co_await storage2::writeSome(
+        m_ledgerStateNonces, RANGES::views::keys(map), RANGES::views::values(map));
+    for (auto&& [sender, nonceInLedger] : map)
+    {
+        if (auto const nonceInMem = co_await storage2::readOne(m_memoryNonces, sender);
+            nonceInMem.has_value() && nonceInLedger >= nonceInMem.value())
+        {
+            co_await storage2::removeOne(m_memoryNonces, sender);
+        }
+    }
+}
+
+task::Task<void> Web3NonceChecker::insertMemoryNonce(std::string sender, u256 nonce)
+{
+    if (c_fileLogLevel == TRACE) [[unlikely]]
+    {
+        TXPOOL_LOG(TRACE) << LOG_DESC("write memory nonces") << LOG_KV("sender", sender)
+                          << LOG_KV("nonce", nonce);
+    }
+    // already checked in checkWeb3Nonce
+    auto const nonceInMem = co_await bcos::storage2::readOne(m_memoryNonces, sender);
+    if (nonceInMem.has_value() && nonceInMem.value() >= nonce)
+    {
+        // not update memory nonce if nonce is smaller than memory's
+        co_return;
+    }
+    co_await storage2::writeOne(m_memoryNonces, sender, nonce);
+    co_return;
 }
 
 void Web3NonceChecker::insert(std::string sender, u256 nonce)