Merge pull request #400 from PeggyJV/bolten/2.0.0-rc3

Exponential backoff in logic call skips, frequent Ethereum height updates

module/proto/gravity/v1/msgs.proto

@@ -33,6 +33,10 @@ service Msg {
   rpc SetDelegateKeys(MsgDelegateKeys) returns (MsgDelegateKeysResponse) {
     // option (google.api.http).post = "/gravity/v1/delegate_keys";
   }
+  rpc SubmitEthereumHeightVote(MsgEthereumHeightVote)
+      returns (MsgEthereumHeightVoteResponse) {
+    // option (google.api.http).post = "/gravity/v1/ethereum_height_vote";
+  }
 }
 
 // MsgSendToEthereum submits a SendToEthereum attempt to bridge an asset over to
@@ -138,6 +142,15 @@ message DelegateKeysSignMsg {
   uint64 nonce = 2;
 }
 
+// Periodic update of latest observed Ethereum and Cosmos heights from the
+// orchestrator
+message MsgEthereumHeightVote {
+  uint64 ethereum_height = 1;
+  string signer = 2;
+}
+
+message MsgEthereumHeightVoteResponse {}
+
 ////////////
 // Events //
 ////////////

module/proto/gravity/v1/query.proto

@@ -148,6 +148,12 @@ service Query {
     // option (google.api.http).get =
     // "/gravity/v1/delegate_keys";
   }
+
+  rpc LastObservedEthereumHeight(LastObservedEthereumHeightRequest)
+      returns (LastObservedEthereumHeightResponse) {
+    // option (google.api.http).get =
+    // "/gravity/v1/last_observed_ethereum_height"
+  }
 }
 
 //  rpc Params
@@ -315,7 +321,13 @@ message UnbatchedSendToEthereumsRequest {
   string sender_address = 1;
   cosmos.base.query.v1beta1.PageRequest pagination = 2;
 }
+
 message UnbatchedSendToEthereumsResponse {
   repeated SendToEthereum send_to_ethereums = 1;
   cosmos.base.query.v1beta1.PageResponse pagination = 2;
 }
+
+message LastObservedEthereumHeightRequest {}
+message LastObservedEthereumHeightResponse {
+  LatestEthereumBlockHeight last_observed_ethereum_height = 1;
+}

module/x/gravity/abci.go

@@ -28,6 +28,7 @@ func BeginBlocker(ctx sdk.Context, k keeper.Keeper) {
 func EndBlocker(ctx sdk.Context, k keeper.Keeper) {
 	outgoingTxSlashing(ctx, k)
 	eventVoteRecordTally(ctx, k)
+	updateObservedEthereumHeight(ctx, k)
 }
 
 func createBatchTxs(ctx sdk.Context, k keeper.Keeper) {
@@ -150,6 +151,80 @@ func eventVoteRecordTally(ctx sdk.Context, k keeper.Keeper) {
 	}
 }
 
+// Periodically, every orchestrator will submit their latest observed Ethereum and Cosmos heights in
+// order to keep this information current regardless of the level of bridge activity.
+//
+// We determine if we should update the latest heights based on the following criteria:
+// 1. A consensus of validators agrees that the proposed height is equal to or less than their
+//    last observed height, in order to reconcile the many different heights that will be submitted.
+//    The highest height that meets this criteria will be the proposed height.
+// 2. The proposed consensus heights from this process are greater than the values stored from the last time
+//    we observed an Ethereum event from the bridge
+func updateObservedEthereumHeight(ctx sdk.Context, k keeper.Keeper) {
+	// wait some minutes before checking the height votes
+	if ctx.BlockHeight()%50 != 0 {
+		return
+	}
+
+	ethereumHeightPowers := make(map[uint64]sdk.Int)
+	cosmosHeightPowers := make(map[uint64]sdk.Int)
+	// we can use the same value as event vote records for this threshold
+	requiredPower := types.EventVoteRecordPowerThreshold(k.StakingKeeper.GetLastTotalPower(ctx))
+
+	// populate the list
+	k.IterateEthereumHeightVotes(ctx, func(valAddres sdk.ValAddress, height types.LatestEthereumBlockHeight) bool {
+		if _, ok := ethereumHeightPowers[height.EthereumHeight]; !ok {
+			ethereumHeightPowers[height.EthereumHeight] = sdk.NewInt(0)
+		}
+
+		if _, ok := cosmosHeightPowers[height.CosmosHeight]; !ok {
+			cosmosHeightPowers[height.CosmosHeight] = sdk.NewInt(0)
+		}
+
+		return false
+	})
+
+	// vote on acceptable height values (less than or equal to the validator's observed value)
+	k.IterateEthereumHeightVotes(ctx, func(valAddress sdk.ValAddress, height types.LatestEthereumBlockHeight) bool {
+		validatorPower := sdk.NewInt(k.StakingKeeper.GetLastValidatorPower(ctx, valAddress))
+
+		for ethereumVoteHeight, ethereumPower := range ethereumHeightPowers {
+			if ethereumVoteHeight <= height.EthereumHeight {
+				ethereumHeightPowers[ethereumVoteHeight] = ethereumPower.Add(validatorPower)
+			}
+		}
+
+		for cosmosVoteHeight, cosmosPower := range cosmosHeightPowers {
+			if cosmosVoteHeight <= height.CosmosHeight {
+				cosmosHeightPowers[cosmosVoteHeight] = cosmosPower.Add(validatorPower)
+			}
+		}
+
+		return false
+	})
+
+	// find the highest height submitted that a consensus of validators agreed was acceptable
+	ethereumHeight := uint64(0)
+	cosmosHeight := uint64(0)
+
+	for ethereumVoteHeight, ethereumPower := range ethereumHeightPowers {
+		if ethereumVoteHeight > ethereumHeight && ethereumPower.GTE(requiredPower) {
+			ethereumHeight = ethereumVoteHeight
+		}
+	}
+
+	for cosmosVoteHeight, cosmosPower := range cosmosHeightPowers {
+		if cosmosVoteHeight > cosmosHeight && cosmosPower.GTE(requiredPower) {
+			cosmosHeight = cosmosVoteHeight
+		}
+	}
+
+	lastObservedHeights := k.GetLastObservedEthereumBlockHeight(ctx)
+	if ethereumHeight > lastObservedHeights.EthereumHeight && cosmosHeight > lastObservedHeights.CosmosHeight {
+		k.SetLastObservedEthereumBlockHeightWithCosmos(ctx, ethereumHeight, cosmosHeight)
+	}
+}
+
 // cleanupTimedOutBatchTxs deletes batches that have passed their expiration on Ethereum
 // keep in mind several things when modifying this function
 // A) unlike nonces timeouts are not monotonically increasing, meaning batch 5 can have a later timeout than batch 6

module/x/gravity/abci_test.go

@@ -307,6 +307,57 @@ func TestBatchTxTimeout(t *testing.T) {
 	require.NotNil(t, gotThirdBatch)
 }
 
+func TestUpdateObservedEthereumHeight(t *testing.T) {
+	input, ctx := keeper.SetupFiveValChain(t)
+	gravityKeeper := input.GravityKeeper
+
+	gravityKeeper.SetLastObservedEthereumBlockHeightWithCosmos(ctx, 2, 5)
+
+	// update runs on mod 50 block heights, no votes have been sent so it
+	// shoudl leave the set values alone
+	ctx = ctx.WithBlockHeight(50)
+	gravity.EndBlocker(ctx, gravityKeeper)
+
+	lastHeight := gravityKeeper.GetLastObservedEthereumBlockHeight(ctx)
+	require.Equal(t, lastHeight.EthereumHeight, uint64(2))
+	require.Equal(t, lastHeight.CosmosHeight, uint64(5))
+
+	ctx = ctx.WithBlockHeight(3)
+	input.GravityKeeper.SetEthereumHeightVote(ctx, keeper.ValAddrs[0], 10)
+
+	ctx = ctx.WithBlockHeight(33)
+	input.GravityKeeper.SetEthereumHeightVote(ctx, keeper.ValAddrs[1], 20)
+
+	ctx = ctx.WithBlockHeight(63)
+	input.GravityKeeper.SetEthereumHeightVote(ctx, keeper.ValAddrs[2], 30)
+
+	ctx = ctx.WithBlockHeight(93)
+	input.GravityKeeper.SetEthereumHeightVote(ctx, keeper.ValAddrs[3], 40)
+
+	ctx = ctx.WithBlockHeight(123)
+	input.GravityKeeper.SetEthereumHeightVote(ctx, keeper.ValAddrs[4], 50)
+
+	// run endblocker on a non-mod 50 block to ensure the update isn't being
+	// called and changing the set values
+	gravity.EndBlocker(ctx, gravityKeeper)
+
+	lastHeight = gravityKeeper.GetLastObservedEthereumBlockHeight(ctx)
+	require.Equal(t, lastHeight.EthereumHeight, uint64(2))
+	require.Equal(t, lastHeight.CosmosHeight, uint64(5))
+
+	// run update in endblocker and verify that 4/5 validators agree that
+	// block height 33 for cosmos and 20 for ethereum are possible, since they
+	// are equal to or less than their own observed block height, and since
+	// those are the highest heights with a consensus of validator power, they
+	// should be set
+	ctx = ctx.WithBlockHeight(150)
+	gravity.EndBlocker(ctx, gravityKeeper)
+
+	lastHeight = gravityKeeper.GetLastObservedEthereumBlockHeight(ctx)
+	require.Equal(t, lastHeight.EthereumHeight, uint64(20))
+	require.Equal(t, lastHeight.CosmosHeight, uint64(33))
+}
+
 func fundAccount(ctx sdk.Context, bankKeeper types.BankKeeper, addr sdk.AccAddress, amounts sdk.Coins) error {
 	if err := bankKeeper.MintCoins(ctx, types.ModuleName, amounts); err != nil {
 		return err

module/x/gravity/client/cli/query.go

@@ -45,6 +45,7 @@ func GetQueryCmd() *cobra.Command {
 		CmdDelegateKeysByEthereumSigner(),
 		CmdDelegateKeysByOrchestrator(),
 		CmdDelegateKeys(),
+		CmdLastObservedEthereumHeight(),
 	)
 
 	return gravityQueryCmd
@@ -805,6 +806,30 @@ func CmdDelegateKeys() *cobra.Command {
 	return cmd
 }
 
+func CmdLastObservedEthereumHeight() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "last-observed-ethereum-height",
+		Args:  cobra.NoArgs,
+		Short: "query the last observed ethereum and cosmos heights",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			clientCtx, queryClient, err := newContextAndQueryClient(cmd)
+			if err != nil {
+				return err
+			}
+
+			res, err := queryClient.LastObservedEthereumHeight(cmd.Context(), &types.LastObservedEthereumHeightRequest{})
+			if err != nil {
+				return err
+			}
+
+			return clientCtx.PrintProto(res)
+		},
+	}
+
+	flags.AddQueryFlagsToCmd(cmd)
+	return cmd
+}
+
 func newContextAndQueryClient(cmd *cobra.Command) (client.Context, types.QueryClient, error) {
 	clientCtx, err := client.GetClientQueryContext(cmd)
 	if err != nil {

module/x/gravity/handler.go

@@ -41,6 +41,10 @@ func NewHandler(k keeper.Keeper) sdk.Handler {
 			res, err := msgServer.SetDelegateKeys(sdk.WrapSDKContext(ctx), msg)
 			return sdk.WrapServiceResult(ctx, res, err)
 
+		case *types.MsgEthereumHeightVote:
+			res, err := msgServer.SubmitEthereumHeightVote(sdk.WrapSDKContext(ctx), msg)
+			return sdk.WrapServiceResult(ctx, res, err)
+
 		default:
 			return nil, sdkerrors.Wrapf(sdkerrors.ErrUnknownRequest, "unrecognized %s message type: %T", types.ModuleName, msg)
 		}

module/x/gravity/keeper/ethereum_event_vote.go

@@ -214,10 +214,15 @@ func (k Keeper) GetLastObservedEthereumBlockHeight(ctx sdk.Context) types.Latest
 
 // SetLastObservedEthereumBlockHeight sets the block height in the store.
 func (k Keeper) SetLastObservedEthereumBlockHeight(ctx sdk.Context, ethereumHeight uint64) {
+	k.SetLastObservedEthereumBlockHeightWithCosmos(ctx, ethereumHeight, uint64(ctx.BlockHeight()))
+}
+
+// SetLastObservedEthereumBlockHeight sets the block height in the store, specifying the cosmos height
+func (k Keeper) SetLastObservedEthereumBlockHeightWithCosmos(ctx sdk.Context, ethereumHeight uint64, cosmosHeight uint64) {
 	store := ctx.KVStore(k.storeKey)
 	height := types.LatestEthereumBlockHeight{
 		EthereumHeight: ethereumHeight,
-		CosmosHeight:   uint64(ctx.BlockHeight()),
+		CosmosHeight:   cosmosHeight,
 	}
 	store.Set([]byte{types.LastEthereumBlockHeightKey}, k.cdc.MustMarshal(&height))
 }

module/x/gravity/keeper/grpc_query.go

@@ -460,3 +460,14 @@ func (k Keeper) DelegateKeys(c context.Context, req *types.DelegateKeysRequest)
 	}
 	return res, nil
 }
+
+func (k Keeper) LastObservedEthereumHeight(c context.Context, req *types.LastObservedEthereumHeightRequest) (*types.LastObservedEthereumHeightResponse, error) {
+	ctx := sdk.UnwrapSDKContext(c)
+	lastObservedEthereumHeight := k.GetLastObservedEthereumBlockHeight(ctx)
+
+	res := &types.LastObservedEthereumHeightResponse{
+		LastObservedEthereumHeight: &lastObservedEthereumHeight,
+	}
+
+	return res, nil
+}

module/x/gravity/keeper/keeper.go

@@ -592,6 +592,56 @@ func (k Keeper) CreateContractCallTx(ctx sdk.Context, invalidationNonce uint64,
 	return newContractCallTx
 }
 
+//////////////////////////////////////
+// Observed Ethereum/Cosmos heights //
+//////////////////////////////////////
+
+// GetEthereumHeightVoteRecord gets the latest observed heights per validator
+func (k Keeper) GetEthereumHeightVote(ctx sdk.Context, valAddress sdk.ValAddress) types.LatestEthereumBlockHeight {
+	store := ctx.KVStore(k.storeKey)
+	key := types.MakeEthereumHeightVoteKey(valAddress)
+	bytes := store.Get(key)
+
+	if len(bytes) == 0 {
+		return types.LatestEthereumBlockHeight{
+			CosmosHeight:   0,
+			EthereumHeight: 0,
+		}
+	}
+
+	height := types.LatestEthereumBlockHeight{}
+	k.cdc.MustUnmarshal(bytes, &height)
+	return height
+}
+
+// SetEthereumHeightVoteRecord sets the latest observed heights per validator
+func (k Keeper) SetEthereumHeightVote(ctx sdk.Context, valAddress sdk.ValAddress, ethereumHeight uint64) {
+	store := ctx.KVStore(k.storeKey)
+	height := types.LatestEthereumBlockHeight{
+		EthereumHeight: ethereumHeight,
+		CosmosHeight:   uint64(ctx.BlockHeight()),
+	}
+	key := types.MakeEthereumHeightVoteKey(valAddress)
+	store.Set(key, k.cdc.MustMarshal(&height))
+}
+
+func (k Keeper) IterateEthereumHeightVotes(ctx sdk.Context, cb func(val sdk.ValAddress, height types.LatestEthereumBlockHeight) (stop bool)) {
+	store := ctx.KVStore(k.storeKey)
+	iter := sdk.KVStorePrefixIterator(store, []byte{types.EthereumHeightVoteKey})
+	defer iter.Close()
+
+	for ; iter.Valid(); iter.Next() {
+		var height types.LatestEthereumBlockHeight
+		key := bytes.NewBuffer(bytes.TrimPrefix(iter.Key(), []byte{types.EthereumHeightVoteKey}))
+		val := sdk.ValAddress(key.Next(20))
+
+		k.cdc.MustUnmarshal(iter.Value(), &height)
+		if cb(val, height) {
+			break
+		}
+	}
+}
+
 /////////////////
 // MIGRATE     //
 /////////////////

module/x/gravity/keeper/msg_server.go

@@ -295,6 +295,19 @@ func (k msgServer) CancelSendToEthereum(c context.Context, msg *types.MsgCancelS
 	return &types.MsgCancelSendToEthereumResponse{}, nil
 }
 
+func (k msgServer) SubmitEthereumHeightVote(c context.Context, msg *types.MsgEthereumHeightVote) (*types.MsgEthereumHeightVoteResponse, error) {
+	ctx := sdk.UnwrapSDKContext(c)
+
+	val, err := k.getSignerValidator(ctx, msg.Signer)
+	if err != nil {
+		return nil, err
+	}
+
+	k.Keeper.SetEthereumHeightVote(ctx, val, msg.EthereumHeight)
+
+	return &types.MsgEthereumHeightVoteResponse{}, nil
+}
+
 // getSignerValidator takes an sdk.AccAddress that represents either a validator or orchestrator address and returns
 // the assoicated validator address
 func (k Keeper) getSignerValidator(ctx sdk.Context, signerString string) (sdk.ValAddress, error) {