Integration test for max block size (#419)

* base for new integration test

* fix split shares edge case

* add vs code to gitignore

* get rid of prepadding messages and use code from core to help generate share commitments

* cleanup

* remove padding to fix test

* nextlowest power of two

* moar bugs

* update go mod

* fix validate basic test for padding

* patch test to create exact number of shares correctly

* minor cleanup

* fix integration test that requires multiple signers

* fix cli integration test

* format gitignore

* only use one delimLen function

* use consistent function to calculate msg shares used

* formatting

* use correct estimation technique for message shares taken

* remove commented code

* use release version of core

* estimate the square size correctly for small square sizes

* patch encoding config to register pfd types for lagacy amino

* use the short test flag when using the race detector

* clean up comment

* use test.short

* fix remainging denom in test

* add comment explaining stuct init

.github/workflows/test.yml

@@ -182,7 +182,7 @@ jobs:
         if: env.GIT_DIFF
       - name: test & coverage report creation
         run: |
-          cat pkgs.txt.part.${{ matrix.part }} | xargs go test -mod=readonly -json -timeout 30m -race -tags='cgo ledger test_ledger_mock'  > ${{ matrix.part }}-race-output.txt
+          cat pkgs.txt.part.${{ matrix.part }} | xargs go test -mod=readonly -json -timeout 30m -race -test.short -tags='cgo ledger test_ledger_mock'  > ${{ matrix.part }}-race-output.txt
         if: env.GIT_DIFF
       - uses: actions/upload-artifact@v3
         with:

.gitignore

@@ -5,3 +5,4 @@ secret.yml
 build
 coverage.txt
 tools-stamp
+.vscode

Makefile

@@ -77,7 +77,7 @@ test-unit:
 	@VERSION=$(VERSION) go test -mod=readonly -tags='ledger test_ledger_mock' ./...
 
 test-race:
-	@VERSION=$(VERSION) go test -mod=readonly -race -tags='ledger test_ledger_mock' ./...
+	@VERSION=$(VERSION) go test -mod=readonly -race -test.short -tags='ledger test_ledger_mock' ./...
 
 benchmark:
 	@go test -mod=readonly -bench=. ./...

app/app.go

@@ -145,6 +145,10 @@ var (
 		qgbmodule.AppModuleBasic{},
 	)
 
+	// ModuleEncodingRegisters keeps track of all the module methods needed to
+	// register interfaces and specific type to encoding config
+	ModuleEncodingRegisters = moduleMapToSlice(ModuleBasics)
+
 	// module account permissions
 	maccPerms = map[string][]string{
 		authtypes.FeeCollectorName:     nil,
@@ -683,3 +687,14 @@ func (app *App) setTxHandler(txConfig client.TxConfig, indexEventsStr []string)
 
 	app.SetTxHandler(txHandler)
 }
+
+func moduleMapToSlice(m module.BasicManager) []encoding.ModuleRegister {
+	// TODO: might be able to use some standard generics in go 1.18
+	s := make([]encoding.ModuleRegister, len(m))
+	i := 0
+	for _, v := range m {
+		s[i] = v
+		i++
+	}
+	return s
+}

app/encoding/encoding.go

@@ -8,7 +8,10 @@ import (
 	"github.com/cosmos/cosmos-sdk/x/auth/tx"
 )
 
-type InterfaceRegister func(codectypes.InterfaceRegistry)
+type ModuleRegister interface {
+	RegisterLegacyAminoCodec(*codec.LegacyAmino)
+	RegisterInterfaces(codectypes.InterfaceRegistry)
+}
 
 // EncodingConfig specifies the concrete encoding types to use for a given app.
 // This is provided for compatibility between protobuf and amino implementations.
@@ -20,14 +23,22 @@ type EncodingConfig struct {
 }
 
 // MakeEncodingConfig creates an encoding config for the app.
-func MakeEncodingConfig(regs ...InterfaceRegister) EncodingConfig {
+func MakeEncodingConfig(regs ...ModuleRegister) EncodingConfig {
+	// create the codec
 	amino := codec.NewLegacyAmino()
-	std.RegisterLegacyAminoCodec(amino)
 	interfaceRegistry := codectypes.NewInterfaceRegistry()
+
+	// register the standard types from the sdk
+	std.RegisterLegacyAminoCodec(amino)
 	std.RegisterInterfaces(interfaceRegistry)
+
+	// register specific modules
 	for _, reg := range regs {
-		reg(interfaceRegistry)
+		reg.RegisterInterfaces(interfaceRegistry)
+		reg.RegisterLegacyAminoCodec(amino)
 	}
+
+	// create the final configuration
 	marshaler := codec.NewProtoCodec(interfaceRegistry)
 	txCfg := tx.NewTxConfig(marshaler, tx.DefaultSignModes)
 

app/prepare_proposal.go

@@ -49,7 +49,7 @@ func (app *App) PrepareProposal(req abci.RequestPrepareProposal) abci.ResponsePr
 func (app *App) estimateSquareSize(data *core.Data) uint64 {
 	txBytes := 0
 	for _, tx := range data.Txs {
-		txBytes += len(tx) + delimLen(uint64(len(tx)))
+		txBytes += len(tx) + types.DelimLen(uint64(len(tx)))
 	}
 	txShareEstimate := txBytes / consts.TxShareSize
 	if txBytes > 0 {
@@ -58,7 +58,7 @@ func (app *App) estimateSquareSize(data *core.Data) uint64 {
 
 	evdBytes := 0
 	for _, evd := range data.Evidence.Evidence {
-		evdBytes += evd.Size() + delimLen(uint64(evd.Size()))
+		evdBytes += evd.Size() + types.DelimLen(uint64(evd.Size()))
 	}
 	evdShareEstimate := evdBytes / consts.TxShareSize
 	if evdBytes > 0 {
@@ -68,9 +68,8 @@ func (app *App) estimateSquareSize(data *core.Data) uint64 {
 	msgShareEstimate := estimateMsgShares(app.txConfig, data.Txs)
 
 	totalShareEstimate := txShareEstimate + evdShareEstimate + msgShareEstimate
-
-	estimatedSize := types.NextPowerOf2(uint64(math.Sqrt(float64(totalShareEstimate))))
-
+	sr := math.Sqrt(float64(totalShareEstimate))
+	estimatedSize := types.NextHighestPowerOf2(uint64(sr))
 	switch {
 	case estimatedSize > consts.MaxSquareSize:
 		return consts.MaxSquareSize
@@ -111,9 +110,7 @@ func estimateMsgShares(txConf client.TxConfig, txs [][]byte) int {
 			continue
 		}
 
-		msgShares += (wireMsg.MessageSize / consts.MsgShareSize) + 1 // plus one to round up
-
+		msgShares += uint64(MsgSharesUsed(int(wireMsg.MessageSize)))
 	}
-
 	return int(msgShares)
 }

app/split_shares.go

@@ -3,7 +3,6 @@ package app
 import (
 	"bytes"
 	"crypto/sha256"
-	"math/bits"
 	"sort"
 
 	"github.com/celestiaorg/celestia-app/x/payment/types"
@@ -23,10 +22,11 @@ import (
 // block data are only used to avoid dereferening, not because we need the block
 // data to be mutable.
 func SplitShares(txConf client.TxConfig, squareSize uint64, data *core.Data) ([][]byte, *core.Data) {
-	var (
-		processedTxs [][]byte
-		messages     core.Messages
-	)
+	processedTxs := make([][]byte, 0)
+	// we initiate this struct here so that the empty output is identiacal in
+	// tests
+	messages := core.Messages{}
+
 	sqwr := newShareSplitter(txConf, squareSize, data)
 
 	for _, rawTx := range data.Txs {
@@ -209,19 +209,19 @@ func (sqwr *shareSplitter) writeMalleatedTx(
 func (sqwr *shareSplitter) hasRoomForBoth(tx, msg []byte) bool {
 	currentShareCount, availableBytes := sqwr.shareCount()
 
-	txBytesTaken := delimLen(uint64(len(tx))) + len(tx)
+	txBytesTaken := types.DelimLen(uint64(len(tx))) + len(tx)
 
 	maxTxSharesTaken := ((txBytesTaken - availableBytes) / consts.TxShareSize) + 1 // plus one becuase we have to add at least one share
 
-	maxMsgSharesTaken := len(msg) / consts.MsgShareSize
+	maxMsgSharesTaken := MsgSharesUsed(len(msg))
 
 	return currentShareCount+maxTxSharesTaken+maxMsgSharesTaken <= sqwr.maxShareCount
 }
 
 func (sqwr *shareSplitter) hasRoomForTx(tx []byte) bool {
 	currentShareCount, availableBytes := sqwr.shareCount()
 
-	bytesTaken := delimLen(uint64(len(tx))) + len(tx)
+	bytesTaken := types.DelimLen(uint64(len(tx))) + len(tx)
 	if bytesTaken <= availableBytes {
 		return true
 	}
@@ -238,9 +238,9 @@ func (sqwr *shareSplitter) shareCount() (count, availableTxBytes int) {
 }
 
 func (sqwr *shareSplitter) export() [][]byte {
-	count, pendingTxBytes := sqwr.shareCount()
+	count, availableBytes := sqwr.shareCount()
 	// increment the count if there are any pending tx bytes
-	if pendingTxBytes > 0 {
+	if availableBytes < consts.TxShareSize {
 		count++
 	}
 	shares := make([][]byte, sqwr.maxShareCount)
@@ -274,6 +274,19 @@ func (sqwr *shareSplitter) export() [][]byte {
 	return shares
 }
 
+// MsgSharesUsed calculates the minimum number of shares a message will take up.
+// It accounts for the necessary delimiter and potential padding.
+func MsgSharesUsed(msgSize int) int {
+	// add the delimiter to the message size
+	msgSize = types.DelimLen(uint64(msgSize)) + msgSize
+	shareCount := msgSize / consts.MsgShareSize
+	// increment the share count if the message overflows the last counted share
+	if msgSize%consts.MsgShareSize != 0 {
+		shareCount++
+	}
+	return shareCount
+}
+
 func hasWirePayForData(tx sdk.Tx) bool {
 	for _, msg := range tx.GetMsgs() {
 		msgName := sdk.MsgTypeURL(msg)
@@ -283,7 +296,3 @@ func hasWirePayForData(tx sdk.Tx) bool {
 	}
 	return false
 }
-
-func delimLen(x uint64) int {
-	return 8 - bits.LeadingZeros64(x)%8
-}