diff --git a/api/clients/codecs/blob.go b/api/clients/codecs/blob.go new file mode 100644 index 0000000000..64cb3678a5 --- /dev/null +++ b/api/clients/codecs/blob.go @@ -0,0 +1,85 @@ +package codecs + +import ( + "fmt" +) + +// Blob is data that is dispersed on eigenDA. +// +// A Blob will contain either an encodedPayload, or a coeffPoly. Whether the Blob contains the former or the latter +// is determined by how the dispersing client has been configured. +type Blob struct { + encodedPayload *encodedPayload + coeffPoly *coeffPoly +} + +// BlobFromEncodedPayload creates a Blob containing an encodedPayload +func blobFromEncodedPayload(encodedPayload *encodedPayload) *Blob { + return &Blob{encodedPayload: encodedPayload} +} + +// blobFromCoeffPoly creates a Blob containing a coeffPoly +func blobFromCoeffPoly(poly *coeffPoly) *Blob { + return &Blob{coeffPoly: poly} +} + +// NewBlob initializes a Blob from raw bytes, and the expected BlobForm +// +// This function will return an error if the input bytes cannot be successfully interpreted as the claimed BlobForm +func NewBlob(bytes []byte, blobForm BlobForm) (*Blob, error) { + switch blobForm { + case Eval: + encodedPayload, err := newEncodedPayload(bytes) + if err != nil { + return nil, fmt.Errorf("new encoded payload: %v", err) + } + + return blobFromEncodedPayload(encodedPayload), nil + case Coeff: + coeffPoly, err := coeffPolyFromBytes(bytes) + if err != nil { + return nil, fmt.Errorf("new coeff poly: %v", err) + } + + return blobFromCoeffPoly(coeffPoly), nil + default: + return nil, fmt.Errorf("unsupported blob form type: %v", blobForm) + } +} + +// GetBytes gets the raw bytes of the Blob +func (b *Blob) GetBytes() []byte { + if b.encodedPayload == nil { + return b.encodedPayload.getBytes() + } else { + return b.coeffPoly.getBytes() + } +} + +// ToPayload converts the Blob into a Payload +func (b *Blob) ToPayload() (*Payload, error) { + var encodedPayload *encodedPayload + var err error + if b.encodedPayload != nil { + encodedPayload = b.encodedPayload + } else if b.coeffPoly != nil { + evalPoly, err := b.coeffPoly.toEvalPoly() + if err != nil { + return nil, fmt.Errorf("coeff poly to eval poly: %v", err) + } + + encodedPayload, err = evalPoly.toEncodedPayload() + if err != nil { + return nil, fmt.Errorf("eval poly to encoded payload: %v", err) + } + } else { + return nil, fmt.Errorf("blob has no contents") + } + + payload, err := encodedPayload.decode() + if err != nil { + return nil, fmt.Errorf("decode encoded payload: %v", err) + } + + return payload, nil +} diff --git a/api/clients/codecs/blob_codec.go b/api/clients/codecs/blob_codec.go index 5be5190261..53a3ce87ff 100644 --- a/api/clients/codecs/blob_codec.go +++ b/api/clients/codecs/blob_codec.go @@ -7,9 +7,10 @@ import ( type BlobEncodingVersion byte const ( - // This minimal blob encoding contains a 32 byte header = [0x00, version byte, uint32 len of data, 0x00, 0x00,...] + // PayloadEncodingVersion0 entails a 32 byte header = [0x00, version byte, big-endian uint32 len of payload, 0x00, 0x00,...] // followed by the encoded data [0x00, 31 bytes of data, 0x00, 31 bytes of data,...] - DefaultBlobEncoding BlobEncodingVersion = 0x0 + // NOTE: this encoding will soon be updated, such that the result will be padded to align to 32 bytes + PayloadEncodingVersion0 BlobEncodingVersion = 0x0 ) type BlobCodec interface { @@ -19,7 +20,7 @@ type BlobCodec interface { func BlobEncodingVersionToCodec(version BlobEncodingVersion) (BlobCodec, error) { switch version { - case DefaultBlobEncoding: + case PayloadEncodingVersion0: return DefaultBlobCodec{}, nil default: return nil, fmt.Errorf("unsupported blob encoding version: %x", version) diff --git a/api/clients/codecs/blob_form.go b/api/clients/codecs/blob_form.go new file mode 100644 index 0000000000..b40a3097c1 --- /dev/null +++ b/api/clients/codecs/blob_form.go @@ -0,0 +1,12 @@ +package codecs + +// BlobForm is an enum that represents the different ways that a blob may be represented +type BlobForm uint + +const ( + // Eval is short for "evaluation form". The field elements represent the evaluation at the polynomial's expanded + // roots of unity + Eval BlobForm = iota + // Coeff is short for "coefficient form". The field elements represent the coefficients of the polynomial + Coeff +) diff --git a/api/clients/codecs/coeff_poly.go b/api/clients/codecs/coeff_poly.go new file mode 100644 index 0000000000..ea5e67c37f --- /dev/null +++ b/api/clients/codecs/coeff_poly.go @@ -0,0 +1,57 @@ +package codecs + +import ( + "fmt" + "math" + + "github.com/Layr-Labs/eigenda/encoding" + "github.com/Layr-Labs/eigenda/encoding/fft" + "github.com/Layr-Labs/eigenda/encoding/rs" + "github.com/consensys/gnark-crypto/ecc/bn254/fr" +) + +// coeffPoly is a polynomial in coefficient form. +// +// The underlying bytes represent 32 byte field elements, and each field element represents a coefficient +type coeffPoly struct { + fieldElements []fr.Element +} + +// coeffPolyFromBytes creates a new coeffPoly from bytes. This function performs the necessary checks to guarantee that the +// bytes are well-formed, and returns a new object if they are +func coeffPolyFromBytes(bytes []byte) (*coeffPoly, error) { + paddedBytes := encoding.PadToPowerOfTwo(bytes) + + fieldElements, err := rs.BytesToFieldElements(paddedBytes) + if err != nil { + return nil, fmt.Errorf("deserialize field elements: %w", err) + } + + return &coeffPoly{fieldElements: fieldElements}, nil +} + +// coeffPolyFromElements creates a new coeffPoly from field elements. +func coeffPolyFromElements(elements []fr.Element) (*coeffPoly, error) { + return &coeffPoly{fieldElements: elements}, nil +} + +// toEvalPoly converts a coeffPoly to an evalPoly, using the FFT operation +func (cp *coeffPoly) toEvalPoly() (*evalPoly, error) { + maxScale := uint8(math.Log2(float64(len(cp.fieldElements)))) + fftedElements, err := fft.NewFFTSettings(maxScale).FFT(cp.fieldElements, false) + if err != nil { + return nil, fmt.Errorf("perform FFT: %w", err) + } + + evalPoly, err := evalPolyFromElements(fftedElements) + if err != nil { + return nil, fmt.Errorf("construct eval poly: %w", err) + } + + return evalPoly, nil +} + +// GetBytes returns the bytes that underlie the polynomial +func (cp *coeffPoly) getBytes() []byte { + return rs.FieldElementsToBytes(cp.fieldElements) +} diff --git a/api/clients/codecs/default_blob_codec.go b/api/clients/codecs/default_blob_codec.go index 6d3ec29944..4b6fc590c7 100644 --- a/api/clients/codecs/default_blob_codec.go +++ b/api/clients/codecs/default_blob_codec.go @@ -22,7 +22,7 @@ func (v DefaultBlobCodec) EncodeBlob(rawData []byte) ([]byte, error) { codecBlobHeader := make([]byte, 32) // first byte is always 0 to ensure the codecBlobHeader is a valid bn254 element // encode version byte - codecBlobHeader[1] = byte(DefaultBlobEncoding) + codecBlobHeader[1] = byte(PayloadEncodingVersion0) // encode length as uint32 binary.BigEndian.PutUint32(codecBlobHeader[2:6], uint32(len(rawData))) // uint32 should be more than enough to store the length (approx 4gb) diff --git a/api/clients/codecs/encoded_payload.go b/api/clients/codecs/encoded_payload.go new file mode 100644 index 0000000000..36f0e21dc8 --- /dev/null +++ b/api/clients/codecs/encoded_payload.go @@ -0,0 +1,64 @@ +package codecs + +import ( + "encoding/binary" + "fmt" + + "github.com/Layr-Labs/eigenda/encoding/utils/codec" +) + +// encodedPayload represents a payload that has had an encoding applied to it +type encodedPayload struct { + bytes []byte +} + +// newEncodedPayload accepts an array of bytes which represent an encodedPayload. It performs the checks necessary +// to guarantee that the bytes are well-formed, and returns a newly constructed object if they are. +// +// Note that this function does not decode the input bytes to perform additional checks, so it is possible to construct +// an encodedPayload, where an attempt to decode will fail. +func newEncodedPayload(encodedPayloadBytes []byte) (*encodedPayload, error) { + inputLen := len(encodedPayloadBytes) + if inputLen < 32 { + return nil, fmt.Errorf( + "input bytes have length %d, which is smaller than the required 32 header bytes", inputLen) + } + + return &encodedPayload{ + bytes: encodedPayloadBytes, + }, nil +} + +// decode applies the inverse of DefaultBlobEncoding to an encodedPayload, and returns the decoded Payload +func (ep *encodedPayload) decode() (*Payload, error) { + claimedLength := binary.BigEndian.Uint32(ep.bytes[2:6]) + + // decode raw data modulo bn254 + nonPaddedData, err := codec.RemoveInternalPadding(ep.bytes[32:]) + if err != nil { + return nil, fmt.Errorf("remove internal padding: %w", err) + } + + if uint32(len(nonPaddedData)) < claimedLength { + return nil, fmt.Errorf( + "data length %d is less than length claimed in encoded payload header %d", + len(nonPaddedData), claimedLength) + } + + return NewPayload(nonPaddedData[0:claimedLength]), nil +} + +// toEvalPoly converts an encodedPayload into an evalPoly +func (ep *encodedPayload) toEvalPoly() (*evalPoly, error) { + evalPoly, err := evalPolyFromBytes(ep.bytes) + if err != nil { + return nil, fmt.Errorf("new eval poly: %w", err) + } + + return evalPoly, nil +} + +// getBytes returns the raw bytes that underlie the encodedPayload +func (ep *encodedPayload) getBytes() []byte { + return ep.bytes +} diff --git a/api/clients/codecs/eval_poly.go b/api/clients/codecs/eval_poly.go new file mode 100644 index 0000000000..8fa7bef3f2 --- /dev/null +++ b/api/clients/codecs/eval_poly.go @@ -0,0 +1,84 @@ +package codecs + +import ( + "encoding/binary" + "fmt" + "math" + + "github.com/Layr-Labs/eigenda/encoding" + "github.com/Layr-Labs/eigenda/encoding/fft" + "github.com/Layr-Labs/eigenda/encoding/rs" + "github.com/Layr-Labs/eigenda/encoding/utils/codec" + "github.com/consensys/gnark-crypto/ecc/bn254/fr" +) + +// evalPoly is a polynomial in evaluation form. +// +// The underlying bytes represent 32 byte field elements, and the field elements represent the evaluation at the +// polynomial's roots of unity +type evalPoly struct { + fieldElements []fr.Element +} + +// evalPolyFromBytes creates a new evalPoly from bytes. This function performs the necessary checks to guarantee that the +// bytes are well-formed, and returns a new object if they are +func evalPolyFromBytes(bytes []byte) (*evalPoly, error) { + paddedBytes := encoding.PadToPowerOfTwo(bytes) + + fieldElements, err := rs.BytesToFieldElements(paddedBytes) + if err != nil { + return nil, fmt.Errorf("deserialize field elements: %w", err) + } + + return &evalPoly{fieldElements: fieldElements}, nil +} + +// evalPolyFromElements creates a new evalPoly from field elements. +func evalPolyFromElements(fieldElements []fr.Element) (*evalPoly, error) { + return &evalPoly{fieldElements: fieldElements}, nil +} + +// toCoeffPoly converts an evalPoly to a coeffPoly, using the IFFT operation +func (ep *evalPoly) toCoeffPoly() (*coeffPoly, error) { + maxScale := uint8(math.Log2(float64(len(ep.fieldElements)))) + ifftedElements, err := fft.NewFFTSettings(maxScale).FFT(ep.fieldElements, true) + if err != nil { + return nil, fmt.Errorf("perform IFFT: %w", err) + } + + coeffPoly, err := coeffPolyFromElements(ifftedElements) + if err != nil { + return nil, fmt.Errorf("construct coeff poly: %w", err) + } + + return coeffPoly, nil +} + +// toEncodedPayload converts an evalPoly into an encoded payload +// +// This conversion entails removing the power-of-2 padding which is added to an encodedPayload when originally creating +// an evalPoly. +func (ep *evalPoly) toEncodedPayload() (*encodedPayload, error) { + polynomialBytes := rs.FieldElementsToBytes(ep.fieldElements) + + payloadLength := binary.BigEndian.Uint32(polynomialBytes[2:6]) + + // add 32 to the padded data length, since the encoded payload includes an encoded payload header + encodedPayloadLength := codec.GetPaddedDataLength(payloadLength) + 32 + + if uint32(len(polynomialBytes)) < payloadLength { + return nil, fmt.Errorf( + "polynomial contains fewer bytes (%d) than expected encoded payload (%d), as determined by claimed length in encoded payload header (%d)", + len(polynomialBytes), encodedPayloadLength, payloadLength) + } + + encodedPayloadBytes := make([]byte, encodedPayloadLength) + copy(encodedPayloadBytes, polynomialBytes[:encodedPayloadLength]) + + encodedPayload, err := newEncodedPayload(encodedPayloadBytes) + if err != nil { + return nil, fmt.Errorf("construct encoded payload: %w", err) + } + + return encodedPayload, nil +} diff --git a/api/clients/codecs/payload.go b/api/clients/codecs/payload.go new file mode 100644 index 0000000000..fe10e24ebc --- /dev/null +++ b/api/clients/codecs/payload.go @@ -0,0 +1,79 @@ +package codecs + +import ( + "encoding/binary" + "fmt" + + "github.com/Layr-Labs/eigenda/encoding/utils/codec" +) + +// Payload represents arbitrary user data, without any processing. +type Payload struct { + bytes []byte +} + +// NewPayload wraps an arbitrary array of bytes into a Payload type. +func NewPayload(payloadBytes []byte) *Payload { + return &Payload{ + bytes: payloadBytes, + } +} + +// Encode applies the PayloadEncodingVersion0 to the original payload bytes +// +// Example encoding: +// +// Encoded Payload header (32 bytes total) Encoded Payload Data +// [0x00, version byte, big-endian uint32 len of payload, 0x00, ...] + [0x00, 31 bytes of data, 0x00, 31 bytes of data,...] +func (p *Payload) encode() (*encodedPayload, error) { + payloadHeader := make([]byte, 32) + // first byte is always 0 to ensure the payloadHeader is a valid bn254 element + payloadHeader[1] = byte(PayloadEncodingVersion0) // encode version byte + + // encode payload length as uint32 + binary.BigEndian.PutUint32( + payloadHeader[2:6], + uint32(len(p.bytes))) // uint32 should be more than enough to store the length (approx 4gb) + + // encode payload modulo bn254, and align to 32 bytes + encodedData := codec.PadPayload(p.bytes) + + encodedPayload, err := newEncodedPayload(append(payloadHeader, encodedData...)) + if err != nil { + return nil, fmt.Errorf("encoding payload: %w", err) + } + + return encodedPayload, nil +} + +// ToBlob converts the Payload bytes into a Blob +func (p *Payload) ToBlob(form BlobForm) (*Blob, error) { + encodedPayload, err := p.encode() + if err != nil { + return nil, fmt.Errorf("encoding payload: %w", err) + } + + switch form { + case Eval: + return blobFromEncodedPayload(encodedPayload), nil + case Coeff: + evalPolynomial, err := encodedPayload.toEvalPoly() + if err != nil { + return nil, fmt.Errorf("encoded payload to eval poly: %w", err) + } + + coeffPoly, err := evalPolynomial.toCoeffPoly() + if err != nil { + return nil, fmt.Errorf("eval poly to coeff poly: %w", err) + } + + return blobFromCoeffPoly(coeffPoly), nil + default: + return nil, fmt.Errorf("unknown polynomial form: %v", form) + } +} + +// GetBytes returns the bytes that underlie the payload, i.e. the unprocessed user data +func (p *Payload) GetBytes() []byte { + return p.bytes +} diff --git a/api/clients/codecs/payload_test.go b/api/clients/codecs/payload_test.go new file mode 100644 index 0000000000..7e2a3e579c --- /dev/null +++ b/api/clients/codecs/payload_test.go @@ -0,0 +1,40 @@ +package codecs + +import ( + "bytes" + "testing" + + "github.com/Layr-Labs/eigenda/common/testutils/random" + "github.com/stretchr/testify/require" +) + +// TestCodec tests the encoding and decoding of random byte streams +func TestPayloadEncoding(t *testing.T) { + testRandom := random.NewTestRandom(t) + + // Number of test iterations + const iterations = 100 + + for i := 0; i < iterations; i++ { + payload := NewPayload(testRandom.Bytes(testRandom.Intn(1024) + 1)) + encodedPayload, err := payload.encode() + require.NoError(t, err) + + // Decode the encoded data + decodedPayload, err := encodedPayload.decode() + require.NoError(t, err) + + if err != nil { + t.Fatalf("Iteration %d: failed to decode blob: %v", i, err) + } + + // Compare the original data with the decoded data + if !bytes.Equal(payload.GetBytes(), decodedPayload.GetBytes()) { + t.Fatalf( + "Iteration %d: original and decoded data do not match\nOriginal: %v\nDecoded: %v", + i, + payload.GetBytes(), + decodedPayload.GetBytes()) + } + } +} diff --git a/api/clients/codecs/poly_test.go b/api/clients/codecs/poly_test.go new file mode 100644 index 0000000000..8665724e5e --- /dev/null +++ b/api/clients/codecs/poly_test.go @@ -0,0 +1,50 @@ +package codecs + +import ( + "bytes" + "testing" + + "github.com/Layr-Labs/eigenda/common/testutils/random" + "github.com/stretchr/testify/require" +) + +// TestFftEncode checks that data can be IfftEncoded and FftEncoded repeatedly, always getting back the original data +// TODO: we should probably be using fuzzing instead of this kind of ad-hoc random search testing +func TestFftEncode(t *testing.T) { + testRandom := random.NewTestRandom(t) + + // Number of test iterations + iterations := 100 + + for i := 0; i < iterations; i++ { + originalData := testRandom.Bytes(testRandom.Intn(1024) + 1) // ensure it's not length 0 + + payload := NewPayload(originalData) + encodedPayload, err := payload.encode() + require.NoError(t, err) + + evalPoly, err := encodedPayload.toEvalPoly() + require.NoError(t, err) + + coeffPoly, err := evalPoly.toCoeffPoly() + require.NoError(t, err) + + convertedEvalPoly, err := coeffPoly.toEvalPoly() + require.NoError(t, err) + + convertedEncodedPayload, err := convertedEvalPoly.toEncodedPayload() + require.NoError(t, err) + + decodedPayload, err := convertedEncodedPayload.decode() + require.NoError(t, err) + + // Compare the original data with the decoded data + if !bytes.Equal(originalData, decodedPayload.GetBytes()) { + t.Fatalf( + "Iteration %d: original and decoded data do not match\nOriginal: %v\nDecoded: %v", + i, + originalData, + decodedPayload.GetBytes()) + } + } +} diff --git a/api/clients/eigenda_client_test.go b/api/clients/eigenda_client_test.go index 29435472be..87f7926b8f 100644 --- a/api/clients/eigenda_client_test.go +++ b/api/clients/eigenda_client_test.go @@ -66,7 +66,7 @@ func TestPutRetrieveBlobIFFTSuccess(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, DisablePointVerificationMode: false, WaitForFinalization: true, }, @@ -133,7 +133,7 @@ func TestPutRetrieveBlobIFFTNoDecodeSuccess(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, DisablePointVerificationMode: false, WaitForFinalization: true, }, @@ -204,7 +204,7 @@ func TestPutRetrieveBlobNoIFFTSuccess(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, DisablePointVerificationMode: true, WaitForFinalization: true, }, @@ -237,7 +237,7 @@ func TestPutBlobFailDispersal(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, @@ -270,7 +270,7 @@ func TestPutBlobFailureInsufficentSignatures(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, @@ -303,7 +303,7 @@ func TestPutBlobFailureGeneral(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, @@ -336,7 +336,7 @@ func TestPutBlobFailureUnknown(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, @@ -371,7 +371,7 @@ func TestPutBlobFinalizationTimeout(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, @@ -431,7 +431,7 @@ func TestPutBlobIndividualRequestTimeout(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, @@ -494,7 +494,7 @@ func TestPutBlobTotalTimeout(t *testing.T) { CustomQuorumIDs: []uint{}, SignerPrivateKeyHex: "75f9e29cac7f5774d106adb355ef294987ce39b7863b75bb3f2ea42ca160926d", DisableTLS: false, - PutBlobEncodingVersion: codecs.DefaultBlobEncoding, + PutBlobEncodingVersion: codecs.PayloadEncodingVersion0, WaitForFinalization: true, }, Client: disperserClient, diff --git a/encoding/rs/utils.go b/encoding/rs/utils.go index d1959380b3..4af149c62a 100644 --- a/encoding/rs/utils.go +++ b/encoding/rs/utils.go @@ -2,6 +2,7 @@ package rs import ( "errors" + "fmt" "math" "github.com/Layr-Labs/eigenda/encoding" @@ -10,6 +11,7 @@ import ( "github.com/consensys/gnark-crypto/ecc/bn254/fr" ) +// ToFrArray TODO: This function will be removed in favor ToFieldElementArray func ToFrArray(data []byte) ([]fr.Element, error) { numEle := GetNumElement(uint64(len(data)), encoding.BYTES_PER_SYMBOL) eles := make([]fr.Element, numEle) @@ -35,6 +37,48 @@ func ToFrArray(data []byte) ([]fr.Element, error) { return eles, nil } +// ToFieldElementArray accept a byte array as an input, and converts it to an array of field elements +// +// This function expects the input array to be a multiple of the size of a field element. +// TODO: test +func BytesToFieldElements(inputData []byte) ([]fr.Element, error) { + if len(inputData)%encoding.BYTES_PER_SYMBOL != 0 { + return nil, fmt.Errorf( + "input array length %d isn't a multiple of encoding.BYTES_PER_SYMBOL %d", + len(inputData), encoding.BYTES_PER_SYMBOL) + } + + elementCount := len(inputData) / encoding.BYTES_PER_SYMBOL + outputElements := make([]fr.Element, elementCount) + for i := 0; i < elementCount; i++ { + destinationStartIndex := i * encoding.BYTES_PER_SYMBOL + destinationEndIndex := destinationStartIndex + encoding.BYTES_PER_SYMBOL + + err := outputElements[i].SetBytesCanonical(inputData[destinationStartIndex:destinationEndIndex]) + if err != nil { + return nil, fmt.Errorf("fr set bytes canonical: %w", err) + } + } + + return outputElements, nil +} + +// FieldElementsToBytes accepts an array of field elements, and converts it to an array of bytes +func FieldElementsToBytes(fieldElements []fr.Element) []byte { + outputBytes := make([]byte, len(fieldElements)*encoding.BYTES_PER_SYMBOL) + + for i := 0; i < len(fieldElements); i++ { + destinationStartIndex := i * encoding.BYTES_PER_SYMBOL + destinationEndIndex := destinationStartIndex + encoding.BYTES_PER_SYMBOL + + fieldElementBytes := fieldElements[i].Bytes() + + copy(outputBytes[destinationStartIndex:destinationEndIndex], fieldElementBytes[:]) + } + + return outputBytes +} + // ToByteArray converts a list of Fr to a byte array func ToByteArray(dataFr []fr.Element, maxDataSize uint64) []byte { n := len(dataFr) diff --git a/encoding/utils.go b/encoding/utils.go index fe665fe7d5..2e7d2179aa 100644 --- a/encoding/utils.go +++ b/encoding/utils.go @@ -34,3 +34,11 @@ func NextPowerOf2[T constraints.Integer](d T) T { nextPower := math.Ceil(math.Log2(float64(d))) return T(math.Pow(2.0, nextPower)) } + +// PadToPowerOfTwo pads a byte slice to the next power of 2 +// TODO: test to make sure this doesn't increase size if already a power of 2 +func PadToPowerOfTwo(bytes []byte) []byte { + paddedLength := NextPowerOf2(len(bytes)) + padding := make([]byte, paddedLength-len(bytes)) + return append(bytes, padding...) +} diff --git a/encoding/utils/codec/codec.go b/encoding/utils/codec/codec.go index 09659d4332..e0ba4bb54b 100644 --- a/encoding/utils/codec/codec.go +++ b/encoding/utils/codec/codec.go @@ -1,6 +1,8 @@ package codec import ( + "fmt" + "github.com/Layr-Labs/eigenda/encoding" ) @@ -65,3 +67,77 @@ func RemoveEmptyByteFromPaddedBytes(data []byte) []byte { } return validData[:validLen] } + +// PadPayload internally pads the input data by prepending a 0x00 to each chunk of 31 bytes. This guarantees that +// the data will be a valid field element for the bn254 curve +// +// Additionally, this function will add necessary padding to align the output to 32 bytes +func PadPayload(inputData []byte) []byte { + // 31 bytes, for the bn254 curve + bytesPerChunk := uint32(encoding.BYTES_PER_SYMBOL - 1) + + // this is the length of the output, which is aligned to 32 bytes + outputLength := GetPaddedDataLength(uint32(len(inputData))) + paddedOutput := make([]byte, outputLength) + + // pre-pad the input, so that it aligns to 31 bytes. This means that the internally padded result will automatically + // align to 32 bytes. Doing this padding in advance simplifies the for loop. + requiredPad := (bytesPerChunk - uint32(len(inputData))%bytesPerChunk) % bytesPerChunk + prePaddedPayload := append(inputData, make([]byte, requiredPad)...) + + for element := uint32(0); element < outputLength/encoding.BYTES_PER_SYMBOL; element++ { + // add the 0x00 internal padding to guarantee that the data is in the valid range + zeroByteIndex := element * encoding.BYTES_PER_SYMBOL + paddedOutput[zeroByteIndex] = 0x00 + + destIndex := zeroByteIndex + 1 + srcIndex := element * bytesPerChunk + + // copy 31 bytes of data from the payload to the padded output + copy(paddedOutput[destIndex:destIndex+bytesPerChunk], prePaddedPayload[srcIndex:srcIndex+bytesPerChunk]) + } + + return paddedOutput +} + +// GetPaddedDataLength accepts the length of a byte array, and returns the length that the array would be after +// adding internal byte padding +func GetPaddedDataLength(inputLen uint32) uint32 { + bytesPerChunk := uint32(encoding.BYTES_PER_SYMBOL - 1) + chunkCount := inputLen / bytesPerChunk + + if inputLen%bytesPerChunk != 0 { + chunkCount++ + } + + return chunkCount * encoding.BYTES_PER_SYMBOL +} + +// RemoveInternalPadding accepts an array of padded data, and removes the internal padding that was added in PadPayload +// +// This function assumes that the input aligns to 32 bytes. Since it is removing 1 byte for every 31 bytes kept, the output +// from this function is not guaranteed to align to 32 bytes. +func RemoveInternalPadding(paddedData []byte) ([]byte, error) { + if len(paddedData)%encoding.BYTES_PER_SYMBOL != 0 { + return nil, fmt.Errorf( + "padded data (length %d) must be multiple of encoding.BYTES_PER_SYMBOL %d", + len(paddedData), + encoding.BYTES_PER_SYMBOL) + } + + bytesPerChunk := encoding.BYTES_PER_SYMBOL - 1 + + symbolCount := len(paddedData) / encoding.BYTES_PER_SYMBOL + outputLength := symbolCount * bytesPerChunk + + outputData := make([]byte, outputLength) + + for i := 0; i < symbolCount; i++ { + dstIndex := i * bytesPerChunk + srcIndex := i*encoding.BYTES_PER_SYMBOL + 1 + + copy(outputData[dstIndex:dstIndex+bytesPerChunk], paddedData[srcIndex:srcIndex+bytesPerChunk]) + } + + return outputData, nil +} diff --git a/encoding/utils/codec/codec_test.go b/encoding/utils/codec/codec_test.go index 3137d7fe7b..a51b9ea06a 100644 --- a/encoding/utils/codec/codec_test.go +++ b/encoding/utils/codec/codec_test.go @@ -49,3 +49,13 @@ func TestSimplePaddingCodec_Fuzz(t *testing.T) { } } } + +// TestGetPaddedDataLength tests that GetPaddedDataLength is behaving as expected +func TestGetPaddedDataLength(t *testing.T) { + startLengths := []uint32{30, 31, 32, 33, 68} + expectedResults := []uint32{32, 32, 64, 64, 96} + + for i := range startLengths { + require.Equal(t, codec.GetPaddedDataLength(startLengths[i]), expectedResults[i]) + } +}