Merge branch 'main' into na/add-timeout-cli

cmd/solver/main.go

@@ -64,7 +64,6 @@ func main() {
 	if err != nil {
 		lmt.Logger(ctx).Fatal("Unable to load config", zap.Error(err))
 	}
-
 	redactedConfig := redactConfig(&cfg)
 
 	lmt.Logger(ctx).Info("starting skip go fast solver",

config/local/config.yml

@@ -74,9 +74,9 @@ chains:
     chain_id: "1"
     type: "evm"
     environment: "mainnet"
-    gas_token_symbol: "ethereum"
+    gas_token_symbol: "ETH"
     gas_token_decimals: 18
-    gas_token_coingecko_id: "ETH"
+    gas_token_coingecko_id: "ethereum"
     hyperlane_domain: "1"
     fast_transfer_contract_address: 0xe7935104c9670015b21c6300e5b95d2f75474cda
     quick_start_num_blocks_back: 300000

config/sample/config.yml

@@ -74,9 +74,9 @@ chains:
     chain_id: "1"
     type: "evm"
     environment: "mainnet"
-    gas_token_symbol: "ethereum"
+    gas_token_symbol: "ETH"
     gas_token_decimals: 18
-    gas_token_coingecko_id: "ETH"
+    gas_token_coingecko_id: "ethereum"
     hyperlane_domain: "1"
     fast_transfer_contract_address: 0xe7935104c9670015b21c6300e5b95d2f75474cda
     quick_start_num_blocks_back: 300000

hyperlane/relayer.go

@@ -4,9 +4,10 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"math/big"
+
 	dbtypes "github.com/skip-mev/go-fast-solver/db"
 	"github.com/skip-mev/go-fast-solver/shared/metrics"
-	"math/big"
 
 	"strings"
 
@@ -253,6 +254,10 @@ func (r *relayer) checkpointAtIndex(
 	return multiSigCheckpoint, nil
 }
 
+var (
+	ErrCouldNotDetermineRelayFee = fmt.Errorf("could not determine relay fee")
+)
+
 // isRelayFeeLessThanMax simulates a relay of a message and checks that the fee
 // to relay the message is less than the users specified max relay fee in uusdc
 func (r *relayer) isRelayFeeLessThanMax(
@@ -264,6 +269,11 @@ func (r *relayer) isRelayFeeLessThanMax(
 ) (bool, error) {
 	txFeeUUSDC, err := r.hyperlane.QuoteProcessUUSDC(ctx, domain, message, metadata)
 	if err != nil {
+		if strings.Contains(err.Error(), "execution reverted") {
+			// if the quote process call has reverted, we return a sentinel
+			// error so that callers can specifically handle this case
+			return false, ErrCouldNotDetermineRelayFee
+		}
 		return false, fmt.Errorf("quoting process call in uusdc: %w", err)
 	}
 

hyperlane/relayer_runner.go

@@ -83,6 +83,15 @@ func (r *RelayerRunner) Run(ctx context.Context) error {
 					case errors.Is(err, ErrRelayTooExpensive):
 						lmt.Logger(ctx).Warn(
 							"relaying transfer is too expensive, waiting for better conditions",
+							zap.Int64("transferId", transfer.ID),
+							zap.String("sourceChainID", transfer.SourceChainID),
+							zap.String("destChainID", transfer.DestinationChainID),
+							zap.String("txHash", transfer.MessageSentTx),
+						)
+					case errors.Is(err, ErrCouldNotDetermineRelayFee):
+						lmt.Logger(ctx).Warn(
+							"could not determine relay fee, retrying",
+							zap.Int64("transferId", transfer.ID),
 							zap.String("sourceChainID", transfer.SourceChainID),
 							zap.String("destChainID", transfer.DestinationChainID),
 							zap.String("txHash", transfer.MessageSentTx),
@@ -116,7 +125,9 @@ func (r *RelayerRunner) Run(ctx context.Context) error {
 						lmt.Logger(ctx).Error(
 							"error relaying pending hyperlane transfer",
 							zap.Error(err),
+							zap.Int64("transferId", transfer.ID),
 							zap.String("sourceChainID", transfer.SourceChainID),
+							zap.String("destChainID", transfer.DestinationChainID),
 							zap.String("txHash", transfer.MessageSentTx),
 						)
 					}
@@ -134,7 +145,9 @@ func (r *RelayerRunner) Run(ctx context.Context) error {
 					lmt.Logger(ctx).Error(
 						"error inserting submitted tx for hyperlane transfer",
 						zap.Error(err),
+						zap.Int64("transferId", transfer.ID),
 						zap.String("sourceChainID", transfer.SourceChainID),
+						zap.String("destChainID", transfer.DestinationChainID),
 						zap.String("txHash", transfer.MessageSentTx),
 					)
 				}

ordersettler/ordersettler.go

@@ -5,6 +5,7 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
+	bridgeclient "github.com/skip-mev/go-fast-solver/shared/bridges/cctp"
 	"math"
 	"math/big"
 	"strconv"
@@ -34,6 +35,10 @@ var params = Config{
 	Delay: 20 * time.Second,
 }
 
+const (
+	excessiveSettlementLatency = 1 * time.Hour
+)
+
 type Database interface {
 	GetAllOrderSettlementsWithSettlementStatus(ctx context.Context, settlementStatus string) ([]db.OrderSettlement, error)
 
@@ -186,6 +191,16 @@ func (r *OrderSettler) findNewSettlements(ctx context.Context) error {
 
 			orderFillEvent, _, err := bridgeClient.QueryOrderFillEvent(ctx, chain.FastTransferContractAddress, fill.OrderID)
 			if err != nil {
+				if _, ok := err.(bridgeclient.ErrOrderFillEventNotFound); ok {
+					lmt.Logger(ctx).Warn(
+						"failed to find order fill event",
+						zap.String("fastTransferGatewayAddress", chain.FastTransferContractAddress),
+						zap.String("orderID", fill.OrderID),
+						zap.String("chainID", chain.ChainID),
+						zap.Error(err),
+					)
+					continue
+				}
 				return fmt.Errorf("querying for order fill event on destination chain at address %s for order id %s: %w", chain.FastTransferContractAddress, fill.OrderID, err)
 			}
 			profit := new(big.Int).Sub(amount, orderFillEvent.FillAmount)
@@ -552,6 +567,9 @@ func (r *OrderSettler) verifyOrderSettlement(ctx context.Context, settlement db.
 	if !settlement.InitiateSettlementTx.Valid {
 		return errors.New("message received txHash is null")
 	}
+	if settlement.CreatedAt.Add(excessiveSettlementLatency).Before(time.Now()) {
+		metrics.FromContext(ctx).IncExcessiveOrderSettlementLatency(settlement.SourceChainID, settlement.DestinationChainID, settlement.SettlementStatus)
+	}
 
 	if settlement.SettlementStatus == dbtypes.SettlementStatusPending {
 		gasCost, failure, err := destinationBridgeClient.GetTxResult(ctx, settlement.InitiateSettlementTx.String)

shared/bridges/cctp/bridge_client.go

@@ -58,6 +58,14 @@ func (e ErrTxResultNotFound) Error() string {
 	return fmt.Sprintf("tx result not found for tx: %s", e.TxHash)
 }
 
+type ErrOrderFillEventNotFound struct {
+	OrderID string
+}
+
+func (e ErrOrderFillEventNotFound) Error() string {
+	return fmt.Sprintf("order fill event not found for order: %s", e.OrderID)
+}
+
 type BridgeClient interface {
 	BlockHeight(ctx context.Context) (uint64, error)
 	SignerGasTokenBalance(ctx context.Context) (*big.Int, error)

shared/bridges/cctp/cosmos_bridge_client.go

@@ -466,6 +466,9 @@ func (c *CosmosBridgeClient) QueryOrderFillEvent(ctx context.Context, gatewayCon
 	if err != nil {
 		return nil, time.Time{}, fmt.Errorf("searching for order fill tx for order %s at gateway %s: %w", orderID, gatewayContractAddress, err)
 	}
+	if searchResult.TotalCount == 0 {
+		return nil, time.Time{}, ErrOrderFillEventNotFound{OrderID: orderID}
+	}
 	if searchResult.TotalCount != 1 {
 		return nil, time.Time{}, fmt.Errorf("expected only 1 tx to be returned from search for order filled events with order id %s at gateway %s, but instead got %d", orderID, gatewayContractAddress, searchResult.TotalCount)
 	}

shared/metrics/metrics.go

@@ -50,6 +50,7 @@ type Metrics interface {
 	SetGasBalance(chainID, chainName, gasTokenSymbol string, gasBalance, warningThreshold, criticalThreshold big.Int, gasTokenDecimals uint8)
 
 	IncExcessiveOrderFulfillmentLatency(sourceChainID, destinationChainID, orderStatus string)
+	IncExcessiveOrderSettlementLatency(sourceChainID, destinationChainID, settlementStatus string)
 	IncExcessiveHyperlaneRelayLatency(sourceChainID, destinationChainID string)
 }
 
@@ -79,6 +80,7 @@ type PromMetrics struct {
 
 	orderSettlementStatusChange      metrics.Counter
 	settlementLatency                metrics.Histogram
+	excessiveOrderSettlementLatency  metrics.Counter
 	excessiveOrderFulfillmentLatency metrics.Counter
 
 	fundRebalanceTransferStatusChange metrics.Counter
@@ -109,6 +111,11 @@ func NewPromMetrics() Metrics {
 			Name:      "excessive_order_fulfillment_latency_counter",
 			Help:      "number of observations of excessive order fulfillment latency, paginated by source and destination chain and status",
 		}, []string{sourceChainIDLabel, destinationChainIDLabel, orderStatusLabel}),
+		excessiveOrderSettlementLatency: prom.NewCounterFrom(stdprom.CounterOpts{
+			Namespace: "solver",
+			Name:      "excessive_order_settlement_latency_counter",
+			Help:      "number of observations of excessive order settlement latency, paginated by source and destination chain and status",
+		}, []string{sourceChainIDLabel, destinationChainIDLabel, settlementStatusLabel}),
 		orderSettlementStatusChange: prom.NewCounterFrom(stdprom.CounterOpts{
 			Namespace: "solver",
 			Name:      "order_settlement_status_change_counter",
@@ -131,15 +138,15 @@ func NewPromMetrics() Metrics {
 		}, []string{successLabel, chainIDLabel}),
 		fillLatency: prom.NewHistogramFrom(stdprom.HistogramOpts{
 			Namespace: "solver",
-			Name:      "latency_per_fill_minutes",
-			Help:      "latency from source transaction to fill completion, paginated by source and destination chain id (in minutes)",
-			Buckets:   []float64{5, 15, 30, 60, 120, 180},
+			Name:      "latency_per_fill_seconds",
+			Help:      "latency from source transaction to fill completion, paginated by source and destination chain id (in seconds)",
+			Buckets:   []float64{1, 5, 10, 15, 20, 30, 40, 50, 60, 120, 300, 600},
 		}, []string{sourceChainIDLabel, destinationChainIDLabel, orderStatusLabel}),
 		settlementLatency: prom.NewHistogramFrom(stdprom.HistogramOpts{
 			Namespace: "solver",
 			Name:      "latency_per_settlement_minutes",
 			Help:      "latency from source transaction to fill completion, paginated by source and destination chain id (in minutes)",
-			Buckets:   []float64{5, 15, 30, 60, 120, 180},
+			Buckets:   []float64{1, 5, 15, 30, 60, 120, 180, 240, 300},
 		}, []string{sourceChainIDLabel, destinationChainIDLabel, settlementStatusLabel}),
 		hplMessageStatusChange: prom.NewCounterFrom(stdprom.CounterOpts{
 			Namespace: "solver",
@@ -156,7 +163,7 @@ func NewPromMetrics() Metrics {
 			Namespace: "solver",
 			Name:      "latency_per_hyperlane_message_seconds",
 			Help:      "latency for hyperlane message relaying, paginated by status, source and destination chain id (in seconds)",
-			Buckets:   []float64{30, 60, 300, 600, 900, 1200, 1500, 1800, 2400, 3000, 3600},
+			Buckets:   []float64{1, 5, 10, 15, 20, 30, 40, 50, 60, 120, 300, 600},
 		}, []string{sourceChainIDLabel, destinationChainIDLabel, transferStatusLabel}),
 		hplRelayTooExpensive: prom.NewCounterFrom(stdprom.CounterOpts{
 			Namespace: "solver",
@@ -224,7 +231,7 @@ func (m *PromMetrics) IncTransactionVerified(success bool, chainID string) {
 }
 
 func (m *PromMetrics) ObserveFillLatency(sourceChainID, destinationChainID, orderStatus string, latency time.Duration) {
-	m.fillLatency.With(sourceChainIDLabel, sourceChainID, destinationChainIDLabel, destinationChainID, orderStatusLabel, orderStatus).Observe(latency.Minutes())
+	m.fillLatency.With(sourceChainIDLabel, sourceChainID, destinationChainIDLabel, destinationChainID, orderStatusLabel, orderStatus).Observe(latency.Seconds())
 }
 
 func (m *PromMetrics) ObserveSettlementLatency(sourceChainID, destinationChainID, settlementStatus string, latency time.Duration) {
@@ -305,6 +312,14 @@ func (m *PromMetrics) IncExcessiveOrderFulfillmentLatency(sourceChainID, destina
 	).Add(1)
 }
 
+func (m *PromMetrics) IncExcessiveOrderSettlementLatency(sourceChainID, destinationChainID, settlementStatus string) {
+	m.excessiveOrderSettlementLatency.With(
+		sourceChainIDLabel, sourceChainID,
+		destinationChainIDLabel, destinationChainID,
+		settlementStatusLabel, settlementStatus,
+	).Add(1)
+}
+
 func (m *PromMetrics) IncExcessiveHyperlaneRelayLatency(sourceChainID, destinationChainID string) {
 	m.excessiveHyperlaneRelayLatency.With(
 		sourceChainIDLabel, sourceChainID,
@@ -316,6 +331,8 @@ type NoOpMetrics struct{}
 
 func (n NoOpMetrics) IncExcessiveOrderFulfillmentLatency(sourceChainID, destinationChainID, orderStatus string) {
 }
+func (n NoOpMetrics) IncExcessiveOrderSettlementLatency(sourceChainID, destinationChainID, settlementStatus string) {
+}
 func (n NoOpMetrics) IncExcessiveHyperlaneRelayLatency(sourceChainID, destinationChainID string) {
 }
 func (n NoOpMetrics) IncHyperlaneRelayTooExpensive(sourceChainID, destinationChainID string) {

shared/signing/evm/evm_transaction.go

@@ -3,9 +3,11 @@ package evm
 import (
 	"context"
 	"fmt"
-	"github.com/skip-mev/go-fast-solver/shared/lmt"
+	"math"
 	"math/big"
 
+	"github.com/skip-mev/go-fast-solver/shared/lmt"
+
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -150,7 +152,7 @@ func WithEstimatedGasLimit(from, to, value string, data []byte) TxBuildOption {
 			return fmt.Errorf("estimating gas limit: %w", err)
 		}
 
-		tx.Gas = gasLimit
+		tx.Gas = uint64(math.Ceil(float64(gasLimit) * 1.2))
 		return nil
 	}
 }

shared/signing/evm/evm_transaction_test.go

@@ -0,0 +1,29 @@
+package evm_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/skip-mev/go-fast-solver/mocks/shared/evmrpc"
+	"github.com/skip-mev/go-fast-solver/shared/signing/evm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+)
+
+func TestWithEstimatedGasLimit(t *testing.T) {
+	t.Run("gas limit is multiplied by 1.2 from what the node returns", func(t *testing.T) {
+		mockRPC := evmrpc.NewMockEVMChainRPC(t)
+		mockRPC.EXPECT().EstimateGas(mock.Anything, mock.Anything).Return(100_000, nil)
+
+		builder := evm.NewTxBuilder(mockRPC)
+		opt := evm.WithEstimatedGasLimit("0xfrom", "0xto", "0", []byte("0xdeadbeef"))
+
+		tx := &types.DynamicFeeTx{}
+		err := opt(context.Background(), builder, tx)
+		assert.NoError(t, err)
+
+		gasLimit := types.NewTx(tx).Gas()
+		assert.Equal(t, uint64(120_000), gasLimit)
+	})
+}