<feat>(txpool): add txpool check web3 nonce logic.

bcos-boostssl/bcos-boostssl/websocket/WsMessage.h

@@ -31,7 +31,7 @@
 #include <string>
 #include <utility>
 
-inline void CHECK_OFFSET(uint16_t offset, uint16_t length)
+inline void CHECK_OFFSET(uint64_t offset, uint64_t length)
 {
     if (offset > length)
     {

bcos-framework/bcos-framework/testutils/faker/FakeLedger.h

@@ -317,6 +317,16 @@ class FakeLedger : public LedgerInterface, public std::enable_shared_from_this<F
     }
     void removeExpiredNonce(protocol::BlockNumber blockNumber, bool sync) override {}
 
+    task::Task<std::optional<ledger::StorageState>> getStorageState(
+        std::string_view _address, protocol::BlockNumber) override
+    {
+        if (eoaInLedger == _address)
+        {
+            ledger::StorageState state{.nonce = eoaInLedgerNonce, .balance = ""};
+            co_return state;
+        }
+        co_return std::nullopt;
+    }
     void setStatus(bool _normal) { m_statusNormal = _normal; }
     void setTotalTxCount(size_t _totalTxCount) { m_totalTxCount = _totalTxCount; }
     void setSystemConfig(std::string_view _key, std::string const& _value)
@@ -381,6 +391,12 @@ class FakeLedger : public LedgerInterface, public std::enable_shared_from_this<F
         });
     }
 
+    void initEoaContext(std::string eoaInLedger, std::string nonce)
+    {
+        this->eoaInLedger = std::move(eoaInLedger);
+        eoaInLedgerNonce = std::move(nonce);
+    }
+
 private:
     BlockFactory::Ptr m_blockFactory;
     std::vector<KeyPairInterface::Ptr> m_keyPairVec;
@@ -399,6 +415,8 @@ class FakeLedger : public LedgerInterface, public std::enable_shared_from_this<F
     std::map<std::string, std::string, std::less<>> m_systemConfig;
     std::vector<bytes> m_sealerList;
     std::shared_ptr<ThreadPool> m_worker = nullptr;
+    std::string eoaInLedger;
+    std::string eoaInLedgerNonce;
 };
 }  // namespace test
 }  // namespace bcos

bcos-framework/bcos-framework/testutils/faker/FakeTransaction.h

@@ -149,6 +149,34 @@ inline Transaction::Ptr fakeTransaction(CryptoSuite::Ptr _cryptoSuite, const std
     return fakeTransaction(_cryptoSuite, keyPair, std::string_view((char*)_to.data(), _to.size()),
         input, nonce, blockLimit, chainId, groupId);
 }
+
+inline Transaction::Ptr fakeWeb3Tx(CryptoSuite::Ptr _cryptoSuite, std::string nonce,
+    crypto::KeyPairInterface::UniquePtr const& key)
+{
+    bcostars::Transaction transaction;
+
+    transaction.data.to = key->address(_cryptoSuite->hashImpl()).hex();
+    std::string inputStr = "testTransaction";
+    transaction.data.input.assign(inputStr.begin(), inputStr.end());
+    transaction.data.nonce = std::move(nonce);
+    transaction.type = static_cast<tars::Char>(TransactionType::Web3Transacion);
+    std::mt19937 random(std::random_device{}());
+    std::string extraData = "extraData" + std::to_string(random());
+    auto hash = _cryptoSuite->hash(extraData);
+    transaction.extraTransactionBytes.assign(extraData.begin(), extraData.end());
+    transaction.extraTransactionHash.assign(hash.begin(), hash.end());
+    auto tx = std::make_shared<bcostars::protocol::TransactionImpl>(
+        [m_transaction = std::move(transaction)]() mutable { return &m_transaction; });
+    // set signature
+    tx->calculateHash(*_cryptoSuite->hashImpl());
+
+    auto signData = _cryptoSuite->signatureImpl()->sign(*key, tx->hash(), true);
+    tx->setSignatureData(*signData);
+    tx->forceSender(key->address(_cryptoSuite->hashImpl()).asBytes());
+    tx->calculateHash(*_cryptoSuite->hashImpl());
+    return tx;
+}
+
 inline TransactionsPtr fakeTransactions(int _size)
 {
     auto hashImpl = std::make_shared<Keccak256>();

bcos-rpc/bcos-rpc/tarsRPC/RPCServer.h

@@ -1,5 +1,7 @@
 #pragma once
-
+#ifdef _WIN32
+#include <tup/Tars.h>
+#endif
 #include "../groupmgr/NodeService.h"
 #include <bcos-tars-protocol/tars/RPC.h>
 #include <tbb/concurrent_hash_map.h>

bcos-rpc/bcos-rpc/web3jsonrpc/model/Web3Transaction.cpp

@@ -144,7 +144,7 @@ bcostars::Transaction Web3Transaction::takeToTarsTransaction()
     auto signRef = bcos::bytesConstRef(
         reinterpret_cast<const bcos::byte*>(tarsTx.signature.data()), tarsTx.signature.size());
     auto [_, sender] = signatureImpl.recoverAddress(hashImpl, hashForSign, signRef);
-    tarsTx.data.nonce = toHexStringWithPrefix(sender) + toQuantity(this->nonce);
+    tarsTx.data.nonce = toQuantity(this->nonce);
     tarsTx.data.chainID = std::to_string(this->chainId.value_or(0));
     tarsTx.dataHash.reserve(crypto::HashType::SIZE);
     RANGES::move(hashForSign, std::back_inserter(tarsTx.dataHash));

bcos-scheduler/src/SchedulerImpl.cpp

@@ -67,9 +67,7 @@ SchedulerImpl::SchedulerImpl(ExecutorManager::Ptr executorManager,
     m_txPool(txPool),
     m_transactionSubmitResultFactory(std::move(transactionSubmitResultFactory)),
     m_hashImpl(std::move(hashImpl)),
-    m_isAuthCheck(isAuthCheck),
     m_isWasm(isWasm),
-    m_isSerialExecute(isSerialExecute),
     m_schedulerTermId(schedulerTermId),
     m_preExeWorker("preExeScheduler", 2),  // assume that preExe is no slower than exe speed/2
     m_exeWorker("exeScheduler", 1)

bcos-scheduler/src/SchedulerImpl.h

@@ -218,9 +218,7 @@ class SchedulerImpl : public SchedulerInterface, public std::enable_shared_from_
     bcos::txpool::TxPoolInterface::Ptr m_txPool;
     bcos::protocol::TransactionSubmitResultFactory::Ptr m_transactionSubmitResultFactory;
     bcos::crypto::Hash::Ptr m_hashImpl;
-    bool m_isAuthCheck = false;
     bool m_isWasm = false;
-    bool m_isSerialExecute = false;
 
     std::function<void(protocol::BlockNumber blockNumber)> m_blockNumberReceiver;
     std::function<void(bcos::protocol::BlockNumber, bcos::protocol::TransactionSubmitResultsPtr,

bcos-sdk/bcos-cpp-sdk/tarsRPC/RPCClient.h

@@ -1,5 +1,7 @@
 #pragma once
-
+#ifdef _WIN32
+#include <tup/Tars.h>
+#endif
 #include "bcos-framework/protocol/Transaction.h"
 #include "bcos-framework/protocol/TransactionReceipt.h"
 #include "bcos-tars-protocol/tars/RPC.h"

bcos-sdk/bcos-cpp-sdk/tarsRPC/detail/TarsCallback.h

@@ -1,4 +1,7 @@
 #pragma once
+#ifdef _WIN32
+#include <tup/Tars.h>
+#endif
 #include "bcos-framework/protocol/TransactionReceipt.h"
 #include "bcos-tars-protocol/tars/RPC.h"
 #include <variant>

bcos-tars-protocol/bcos-tars-protocol/impl/TarsHashable.h

@@ -1,4 +1,7 @@
 #pragma once
+#ifdef _WIN32
+#include <tup/Tars.h>
+#endif
 #include "bcos-concepts/ByteBuffer.h"
 #include "bcos-concepts/Hash.h"
 #include "bcos-crypto/hasher/Hasher.h"

bcos-txpool/bcos-txpool/TxPoolFactory.cpp

@@ -22,6 +22,7 @@
 #include "bcos-txpool/sync/TransactionSync.h"
 #include "bcos-txpool/sync/protocol/PB/TxsSyncMsgFactoryImpl.h"
 #include "bcos-txpool/txpool/validator/TxValidator.h"
+#include "bcos-txpool/txpool/validator/Web3NonceChecker.h"
 #include "txpool/storage/MemoryStorage.h"
 #include "txpool/validator/TxPoolNonceChecker.h"
 
@@ -54,8 +55,9 @@ TxPool::Ptr TxPoolFactory::createTxPool(
 {
     TXPOOL_LOG(INFO) << LOG_DESC("create transaction validator");
     auto txpoolNonceChecker = std::make_shared<TxPoolNonceChecker>();
-    auto validator =
-        std::make_shared<TxValidator>(txpoolNonceChecker, m_cryptoSuite, m_groupId, m_chainId);
+    auto web3NonceChecker = std::make_shared<Web3NonceChecker>(m_ledger);
+    auto validator = std::make_shared<TxValidator>(
+        txpoolNonceChecker, std::move(web3NonceChecker), m_cryptoSuite, m_groupId, m_chainId);
 
     TXPOOL_LOG(INFO) << LOG_DESC("create transaction config");
     auto txpoolConfig = std::make_shared<TxPoolConfig>(validator, m_txResultFactory, m_blockFactory,

bcos-txpool/bcos-txpool/txpool/interfaces/NonceCheckerInterface.h

@@ -36,7 +36,7 @@ class NonceCheckerInterface
     virtual ~NonceCheckerInterface() = default;
 
     virtual bcos::protocol::TransactionStatus checkNonce(
-        bcos::protocol::Transaction::ConstPtr _tx, bool _shouldUpdate = false) = 0;
+        bcos::protocol::Transaction::ConstPtr _tx) = 0;
     virtual bool exists(bcos::protocol::NonceType const& _nonce) = 0;
     virtual void batchInsert(
         bcos::protocol::BlockNumber _batchId, bcos::protocol::NonceListPtr const& _nonceList) = 0;

bcos-txpool/bcos-txpool/txpool/interfaces/TxValidatorInterface.h

@@ -21,6 +21,7 @@
 #pragma once
 #include "bcos-txpool/txpool/interfaces/NonceCheckerInterface.h"
 #include "bcos-txpool/txpool/validator/LedgerNonceChecker.h"
+#include "bcos-txpool/txpool/validator/Web3NonceChecker.h"
 #include <bcos-framework/protocol/Transaction.h>
 #include <bcos-protocol/TransactionStatus.h>
 namespace bcos::txpool
@@ -33,11 +34,16 @@ class TxValidatorInterface
     virtual ~TxValidatorInterface() = default;
 
     virtual bcos::protocol::TransactionStatus verify(bcos::protocol::Transaction::ConstPtr _tx) = 0;
+    virtual bcos::protocol::TransactionStatus checkTransaction(
+        bcos::protocol::Transaction::ConstPtr _tx, bool onlyCheckLedgerNonce = false) = 0;
     virtual bcos::protocol::TransactionStatus checkLedgerNonceAndBlockLimit(
         bcos::protocol::Transaction::ConstPtr _tx) = 0;
     virtual bcos::protocol::TransactionStatus checkTxpoolNonce(
         bcos::protocol::Transaction::ConstPtr _tx) = 0;
+    virtual bcos::protocol::TransactionStatus checkWeb3Nonce(
+        bcos::protocol::Transaction::ConstPtr _tx) = 0;
     virtual LedgerNonceChecker::Ptr ledgerNonceChecker() = 0;
+    virtual Web3NonceChecker::Ptr web3NonceChecker() = 0;
     virtual void setLedgerNonceChecker(LedgerNonceChecker::Ptr _ledgerNonceChecker) = 0;
 };
 }  // namespace bcos::txpool

bcos-txpool/bcos-txpool/txpool/storage/MemoryStorage.cpp

@@ -287,11 +287,8 @@ TransactionStatus MemoryStorage::enforceSubmitTransaction(Transaction::Ptr _tx)
     {
         // check txpool nonce
         // check ledger tx
-        auto result = m_config->txValidator()->checkTxpoolNonce(_tx);
-        if (result == TransactionStatus::None)
-        {
-            result = m_config->txValidator()->checkLedgerNonceAndBlockLimit(_tx);
-        }
+        // check web3 tx
+        auto result = m_config->txValidator()->checkTransaction(_tx);
         Transaction::ConstPtr tx = nullptr;
         {
             TxsMap::ReadAccessor::Ptr accessor;
@@ -618,7 +615,6 @@ void MemoryStorage::batchRemove(BlockNumber batchId, TransactionSubmitResults co
     uint64_t lockT = 0;
     m_blockNumberUpdatedTime = recordT;
     size_t succCount = 0;
-    NonceList nonceList;
 
     auto range =
         txsResult | RANGES::views::transform([](TransactionSubmitResult::Ptr const& _txResult) {
@@ -669,20 +665,35 @@ void MemoryStorage::batchRemove(BlockNumber batchId, TransactionSubmitResults co
     notifyUnsealedTxsSize();
     // update the ledger nonce
 
-    auto nonceListRange = results | RANGES::views::filter([](auto const& _result) {
-        const auto& tx = _result.second.first;
-        const auto& txResult = _result.second.second;
-        return tx == nullptr ? !txResult->nonce().empty() : true;
-    }) | RANGES::views::transform([](auto const& _result) {
-        const auto& tx = _result.second.first;
-        const auto& txResult = _result.second.second;
-        return tx != nullptr ? tx->nonce() : txResult->nonce();
-    });
-
-    auto nonceListPtr = std::make_shared<NonceList>(nonceListRange.begin(), nonceListRange.end());
+    auto nonceListPtr = std::make_shared<NonceList>();
+    std::unordered_map<std::string, std::string> web3NonceMap;
+    for (auto&& [_, txPair] : results)
+    {
+        auto const& [tx, txResult] = txPair;
+        if (tx)
+        {
+            if (tx->type() == TransactionType::Web3Transacion) [[unlikely]]
+            {
+                web3NonceMap[std::string(tx->sender())] = tx->nonce();
+            }
+            else
+            {
+                nonceListPtr->emplace_back(tx->nonce());
+            }
+        }
+        else if (!txResult->nonce().empty())
+        {
+            nonceListPtr->emplace_back(txResult->nonce());
+        }
+    }
     m_config->txValidator()->ledgerNonceChecker()->batchInsert(batchId, nonceListPtr);
     auto updateLedgerNonceT = utcTime() - startT;
 
+    startT = utcTime();
+    m_config->txValidator()->web3NonceChecker()->batchInsert(
+        RANGES::views::keys(web3NonceMap), RANGES::views::values(web3NonceMap));
+    auto updateWeb3NonceT = utcTime() - startT;
+
     startT = utcTime();
     // update the txpool nonce
     m_config->txPoolNonceChecker()->batchRemove(*nonceListPtr);
@@ -705,6 +716,7 @@ void MemoryStorage::batchRemove(BlockNumber batchId, TransactionSubmitResults co
                      << LOG_KV("batchId", batchId) << LOG_KV("timecost", (utcTime() - recordT))
                      << LOG_KV("lockT", lockT) << LOG_KV("removeT", removeT)
                      << LOG_KV("updateLedgerNonceT", updateLedgerNonceT)
+                     << LOG_KV("updateWeb3NonceT", updateLedgerNonceT)
                      << LOG_KV("updateTxPoolNonceT", updateTxPoolNonceT);
 }
 
@@ -808,7 +820,7 @@ void MemoryStorage::batchFetchTxs(Block::Ptr _txsList, Block::Ptr _sysTxsList, s
         // txPool, the txs with duplicated nonce here are already-committed, but have not been
         // dropped
         // check txpool txs, no need to check txpool nonce
-        auto result = m_config->txValidator()->checkLedgerNonceAndBlockLimit(tx);
+        auto result = m_config->txValidator()->checkTransaction(tx, true);
         if (result == TransactionStatus::NonceCheckFail)
         {
             // in case of the same tx notified more than once
@@ -1265,7 +1277,7 @@ HashListPtr MemoryStorage::getTxsHash(int _limit)
                 return true;
             }
             // check txpool txs, no need to check txpool nonce
-            auto result = m_config->txValidator()->checkLedgerNonceAndBlockLimit(tx);
+            auto result = m_config->txValidator()->checkTransaction(tx, true);
             if (result != TransactionStatus::None)
             {
                 TxsMap::WriteAccessor::Ptr writeAccessor;
@@ -1346,7 +1358,8 @@ void MemoryStorage::cleanUpExpiredTransactions()
             }
         }
         // check txpool txs, no need to check txpool nonce
-        auto result = m_config->txValidator()->checkLedgerNonceAndBlockLimit(tx);
+        auto validator = m_config->txValidator();
+        auto result = validator->checkTransaction(tx, true);
         // blockLimit expired
         if (result != TransactionStatus::None)
         {

bcos-txpool/bcos-txpool/txpool/storage/MemoryStorage.h

@@ -23,6 +23,7 @@
 #include "bcos-task/Task.h"
 #include "bcos-txpool/TxPoolConfig.h"
 #include "bcos-txpool/txpool/utilities/Common.h"
+#include "bcos-txpool/txpool/validator/Web3NonceChecker.h"
 #include <bcos-txpool/txpool/utilities/TransactionBucket.h>
 #include <bcos-utilities/BucketMap.h>
 #include <bcos-utilities/FixedBytes.h>

bcos-txpool/bcos-txpool/txpool/validator/LedgerNonceChecker.cpp

@@ -33,10 +33,10 @@ void LedgerNonceChecker::initNonceCache(
     }
 }
 
-TransactionStatus LedgerNonceChecker::checkNonce(Transaction::ConstPtr _tx, bool _shouldUpdate)
+TransactionStatus LedgerNonceChecker::checkNonce(Transaction::ConstPtr _tx)
 {
     // check nonce
-    auto status = TxPoolNonceChecker::checkNonce(_tx, _shouldUpdate);
+    auto status = TxPoolNonceChecker::checkNonce(_tx);
     if (status != TransactionStatus::None)
     {
         return status;

bcos-txpool/bcos-txpool/txpool/validator/LedgerNonceChecker.h

@@ -41,7 +41,7 @@ class LedgerNonceChecker : public TxPoolNonceChecker
         }
     }
     bcos::protocol::TransactionStatus checkNonce(
-        bcos::protocol::Transaction::ConstPtr _tx, bool _shouldUpdate = false) override;
+        bcos::protocol::Transaction::ConstPtr _tx) override;
 
     void batchInsert(bcos::protocol::BlockNumber _batchId,
         bcos::protocol::NonceListPtr const& _nonceList) override;

bcos-txpool/bcos-txpool/txpool/validator/TxPoolNonceChecker.cpp

@@ -30,20 +30,14 @@ bool TxPoolNonceChecker::exists(NonceType const& _nonce)
     return m_nonces.contains(_nonce);
 }
 
-TransactionStatus TxPoolNonceChecker::checkNonce(Transaction::ConstPtr _tx, bool _shouldUpdate)
+TransactionStatus TxPoolNonceChecker::checkNonce(Transaction::ConstPtr _tx)
 {
     auto nonce = _tx->nonce();
 
     if (m_nonces.contains(nonce))
     {
         return TransactionStatus::NonceCheckFail;
     }
-
-    if (_shouldUpdate)
-    {
-        NonceSet::WriteAccessor::Ptr accessor;
-        m_nonces.insert(accessor, std::move(nonce));
-    }
     return TransactionStatus::None;
 }
 

bcos-txpool/bcos-txpool/txpool/validator/TxPoolNonceChecker.h

@@ -31,7 +31,7 @@ class TxPoolNonceChecker : public NonceCheckerInterface
 public:
     TxPoolNonceChecker() : m_nonces(256){};
     bcos::protocol::TransactionStatus checkNonce(
-        bcos::protocol::Transaction::ConstPtr _tx, bool _shouldUpdate = false) override;
+        bcos::protocol::Transaction::ConstPtr _tx) override;
     void batchInsert(bcos::protocol::BlockNumber _batchId,
         bcos::protocol::NonceListPtr const& _nonceList) override;
     void batchRemove(bcos::protocol::NonceList const& _nonceList) override;