Skip to content

Commit

Permalink
feat!: increase share size to 512 bytes (#850)
Browse files Browse the repository at this point in the history
Closes #825

This is ready for review but we may not merge it b/c we haven't made a
final decision on what the share size should be

Co-authored-by: evan-forbes <[email protected]>
  • Loading branch information
rootulp and evan-forbes authored Oct 18, 2022
1 parent ded7346 commit ecc9423
Show file tree
Hide file tree
Showing 14 changed files with 261 additions and 122 deletions.
4 changes: 2 additions & 2 deletions app/estimate_square_size_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@ func Test_estimateSquareSize(t *testing.T) {
tests := []test{
{"empty block minimum square size", 0, 0, 0, appconsts.MinSquareSize},
{"full block with only txs", 10000, 0, 0, appconsts.MaxSquareSize},
{"random small block square size 4", 0, 1, appconsts.SparseShareContentSize, 4},
{"random small block square size 2", 0, 1, appconsts.SparseShareContentSize, 2},
{"random small block square size 4", 0, 1, appconsts.SparseShareContentSize * 10, 4},
{"random small block w/ 10 normal txs square size 4", 10, 1, appconsts.SparseShareContentSize, 8},
{"random small block w/ 10 normal txs square size 4", 10, 1, appconsts.SparseShareContentSize, 4},
{"random small block square size 16", 0, 4, appconsts.SparseShareContentSize * 8, 16},
{"random medium block square size 32", 0, 50, appconsts.SparseShareContentSize * 4, 32},
{"full block max square size", 0, 8000, appconsts.SparseShareContentSize, appconsts.MaxSquareSize},
Expand Down
2 changes: 1 addition & 1 deletion app/test/block_size_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ func TestIntegrationTestSuite(t *testing.T) {
cfg.EnableTMLogging = false
cfg.MinGasPrices = "0ucls"
cfg.NumValidators = 1
cfg.TimeoutCommit = time.Millisecond * 200
cfg.TimeoutCommit = time.Millisecond * 400
suite.Run(t, NewIntegrationTestSuite(cfg))
}

Expand Down
4 changes: 2 additions & 2 deletions pkg/appconsts/appconsts.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import (
// https://github.com/celestiaorg/celestia-specs/blob/master/src/specs/consensus.md#constants
const (
// ShareSize is the size of a share in bytes.
ShareSize = 256
ShareSize = 512

// NamespaceSize is the namespace size in bytes.
NamespaceSize = 8
Expand All @@ -27,7 +27,7 @@ const (

// CompactShareReservedBytes is the number of bytes reserved for the location of
// the first unit (transaction, ISR, evidence) in a compact share.
CompactShareReservedBytes = 1
CompactShareReservedBytes = 2

// ContinuationCompactShareContentSize is the number of bytes usable for
// data in a continuation compact share. A continuation share is any
Expand Down
27 changes: 14 additions & 13 deletions pkg/da/data_availability_header_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ func TestNilDataAvailabilityHeaderHashDoesntCrash(t *testing.T) {
func TestMinDataAvailabilityHeader(t *testing.T) {
dah := MinDataAvailabilityHeader()
expectedHash := []byte{
0x7b, 0x57, 0x8b, 0x35, 0x1b, 0x1b, 0xb, 0xbd, 0x70, 0xbb, 0x35, 0x0, 0x19, 0xeb, 0xc9, 0x64,
0xc4, 0x4a, 0x14, 0xa, 0x37, 0xef, 0x71, 0x5b, 0x55, 0x2a, 0x7f, 0x8f, 0x31, 0x5a, 0xcd, 0x19,
0x6f, 0x52, 0xda, 0xc1, 0x65, 0x45, 0xe4, 0x57, 0x25, 0xbe, 0x6e, 0xa3, 0x2a, 0xed, 0x55, 0x26,
0x6e, 0x45, 0x3, 0x48, 0x0, 0xee, 0xe1, 0xd8, 0x7c, 0x94, 0x28, 0xf4, 0x84, 0x4e, 0xa4, 0x7a,
}
require.Equal(t, expectedHash, dah.hash)
require.NoError(t, dah.ValidateBasic())
Expand All @@ -47,31 +47,32 @@ func TestNewDataAvailabilityHeader(t *testing.T) {
{
name: "typical",
expectedHash: []byte{
0xfe, 0x9c, 0x6b, 0xd8, 0xe5, 0x7c, 0xd1, 0x5d, 0x1f, 0xd6, 0x55, 0x7e, 0x87, 0x7d, 0xd9, 0x7d,
0xdb, 0xf2, 0x66, 0xfa, 0x60, 0x24, 0x2d, 0xb3, 0xa0, 0x9c, 0x4f, 0x4e, 0x5b, 0x2a, 0x2c, 0x2a,
0x57, 0x71, 0xc6, 0x77, 0x2f, 0x32, 0x95, 0x73, 0xaa, 0xb8, 0x20, 0xd1, 0xbe, 0x4c, 0xc2, 0x21,
0x7d, 0x54, 0xb6, 0x7e, 0xf2, 0x4f, 0xbc, 0xd3, 0x9a, 0x95, 0x15, 0xd0, 0x92, 0x63, 0xc1, 0xf9,
},
squareSize: 2,
shares: generateShares(4, 1),
},
{
name: "max square size",
expectedHash: []byte{
0xe2, 0x87, 0x23, 0xd0, 0x2d, 0x54, 0x25, 0x5f, 0x79, 0x43, 0x8e, 0xfb, 0xb7, 0xe8, 0xfa, 0xf5,
0xbf, 0x93, 0x50, 0xb3, 0x64, 0xd0, 0x4f, 0xa7, 0x7b, 0xb1, 0x83, 0x3b, 0x8, 0xba, 0xd3, 0xa4,
0xbf, 0xe5, 0x8f, 0x4b, 0xae, 0x2b, 0x65, 0x8b, 0xa8, 0xcb, 0xf9, 0xee, 0x8c, 0x6a, 0x1f, 0x72,
0xa9, 0x58, 0xc4, 0xcc, 0xca, 0x41, 0x4c, 0xbf, 0x8b, 0x18, 0xf9, 0x53, 0xe, 0xb1, 0x40, 0x54,
},
squareSize: appconsts.MaxSquareSize,
shares: generateShares(appconsts.MaxSquareSize*appconsts.MaxSquareSize, 99),
},
}

for _, tt := range tests {
tt := tt
eds, err := ExtendShares(tt.squareSize, tt.shares)
require.NoError(t, err)
resdah := NewDataAvailabilityHeader(eds)
require.Equal(t, tt.squareSize*2, uint64(len(resdah.ColumnRoots)), tt.name)
require.Equal(t, tt.squareSize*2, uint64(len(resdah.RowsRoots)), tt.name)
require.Equal(t, tt.expectedHash, resdah.hash, tt.name)
t.Run(tt.name, func(t *testing.T) {
eds, err := ExtendShares(tt.squareSize, tt.shares)
require.NoError(t, err)
resdah := NewDataAvailabilityHeader(eds)
require.Equal(t, tt.squareSize*2, uint64(len(resdah.ColumnRoots)), tt.name)
require.Equal(t, tt.squareSize*2, uint64(len(resdah.RowsRoots)), tt.name)
require.Equal(t, tt.expectedHash, resdah.hash, tt.name)
})
}
}

Expand Down
14 changes: 7 additions & 7 deletions pkg/prove/proof_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -152,14 +152,14 @@ func TestTxShareIndex(t *testing.T) {
{appconsts.FirstCompactShareContentSize + appconsts.ContinuationCompactShareContentSize + 1, 2},
{appconsts.FirstCompactShareContentSize + (appconsts.ContinuationCompactShareContentSize * 2), 2},
{appconsts.FirstCompactShareContentSize + (appconsts.ContinuationCompactShareContentSize * 2) + 1, 3},
// 81 compact shares + partially filled out last share
// 81 full compact shares then a partially filled out 82nd share (which is index 81 because 0-indexed)
{appconsts.FirstCompactShareContentSize + (appconsts.ContinuationCompactShareContentSize * 80) + 160, 81},
// 81 compact shares + full last share
{appconsts.FirstCompactShareContentSize + (appconsts.ContinuationCompactShareContentSize * 80) + 246, 81},
// 82 compact shares + one byte in last share
{appconsts.FirstCompactShareContentSize + (appconsts.ContinuationCompactShareContentSize * 80) + 247, 82},
// 82 compact shares + two bytes in last share
{appconsts.FirstCompactShareContentSize + (appconsts.ContinuationCompactShareContentSize * 80) + 248, 82},
// 81 full compact shares then a full 82nd share
{appconsts.FirstCompactShareContentSize + (appconsts.ContinuationCompactShareContentSize * 80) + 501, 81},
// 82 full compact shares then one byte in 83rd share
{appconsts.FirstCompactShareContentSize + (appconsts.ContinuationCompactShareContentSize * 80) + 502, 82},
// 82 compact shares then two bytes in 83rd share
{appconsts.FirstCompactShareContentSize + (appconsts.ContinuationCompactShareContentSize * 80) + 503, 82},
}

for _, tt := range tests {
Expand Down
39 changes: 39 additions & 0 deletions pkg/shares/reserved_bytes.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package shares

import (
"bytes"
"encoding/binary"
"fmt"

"github.com/celestiaorg/celestia-app/pkg/appconsts"
)

// NewReservedBytes returns a byte slice of length
// appconsts.CompactShareReservedBytes that contains a varint of the byteIndex
// of the first unit that starts in a compact share. If no unit starts in the
// compact share, ReservedBytes is [0, 0].
func NewReservedBytes(byteIndex uint64) ([]byte, error) {
if byteIndex >= appconsts.ShareSize {
return []byte{}, fmt.Errorf("byte index %d must be less than share size %d", byteIndex, appconsts.ShareSize)
}
reservedBytes := make([]byte, appconsts.CompactShareReservedBytes)
binary.PutUvarint(reservedBytes, byteIndex)
return reservedBytes, nil
}

// ParseReservedBytes parses a byte slice of length
// appconsts.CompactShareReservedBytes into a byteIndex.
func ParseReservedBytes(reservedBytes []byte) (uint64, error) {
if len(reservedBytes) != appconsts.CompactShareReservedBytes {
return 0, fmt.Errorf("reserved bytes must be of length %d", appconsts.CompactShareReservedBytes)
}
reader := bytes.NewReader(reservedBytes)
byteIndex, err := binary.ReadUvarint(reader)
if err != nil {
return 0, err
}
if byteIndex >= appconsts.ShareSize {
return 0, fmt.Errorf("reserved bytes varint %d must be less than share size %d", byteIndex, appconsts.ShareSize)
}
return byteIndex, nil
}
84 changes: 84 additions & 0 deletions pkg/shares/reserved_bytes_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
package shares

import (
"testing"

"github.com/stretchr/testify/assert"
)

func TestParseReservedBytes(t *testing.T) {
type testCase struct {
name string
input []byte
want uint64
expectErr bool
}
testCases := []testCase{
{"byte index of 0", []byte{0, 0}, 0, false},
{"byte index of 2", []byte{2, 0}, 2, false},
{"byte index of 4", []byte{4, 0}, 4, false},
{"byte index of 8", []byte{8, 0}, 8, false},
{"byte index of 16", []byte{16, 0}, 16, false},
{"byte index of 32", []byte{32, 0}, 32, false},
{"byte index of 64", []byte{64, 0}, 64, false},
{"byte index of 128", []byte{128, 1}, 128, false},
{"byte index of 256", []byte{128, 2}, 256, false},
{"byte index of 511", []byte{255, 3}, 511, false},

// error cases
{"empty", []byte{}, 0, true},
{"too few reserved bytes", []byte{1}, 0, true},
{"too many reserved bytes", []byte{3, 3, 3}, 0, true},
{"byte index of 512 is equal to share size", []byte{128, 4}, 0, true},
{"byte index of 1000 is greater than share size", []byte{232, 7}, 0, true},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
got, err := ParseReservedBytes(tc.input)
if tc.expectErr {
assert.Error(t, err)
return
}
assert.NoError(t, err)
assert.Equal(t, tc.want, got)
})
}
}

func TestNewReservedBytes(t *testing.T) {
type testCase struct {
name string
input uint64
want []byte
expectErr bool
}
testCases := []testCase{
{"byte index of 0", 0, []byte{0, 0}, false},
{"byte index of 2", 2, []byte{2, 0}, false},
{"byte index of 4", 4, []byte{4, 0}, false},
{"byte index of 8", 8, []byte{8, 0}, false},
{"byte index of 16", 16, []byte{16, 0}, false},
{"byte index of 32", 32, []byte{32, 0}, false},
{"byte index of 64", 64, []byte{64, 0}, false},
{"byte index of 128", 128, []byte{128, 1}, false},
{"byte index of 256", 256, []byte{128, 2}, false},
{"byte index of 511", 511, []byte{255, 3}, false},

// error cases
{"byte index of 512 is equal to share size", 512, []byte{}, true},
{"byte index of 1000 is greater than share size", 1000, []byte{}, true},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
got, err := NewReservedBytes(tc.input)
if tc.expectErr {
assert.Error(t, err)
return
}
assert.NoError(t, err)
assert.Equal(t, tc.want, got)
})
}
}
Loading

0 comments on commit ecc9423

Please sign in to comment.