diff --git a/.gitignore b/.gitignore index 2115642..5ca0594 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ # Output of the go coverage tool, specifically when used with LiteIDE *.out vendor -.vscode \ No newline at end of file +.vscode +coverage.txt \ No newline at end of file diff --git a/inflight.go b/inflight.go index cd67e5e..85491ef 100644 --- a/inflight.go +++ b/inflight.go @@ -1,6 +1,9 @@ package inflight import ( + "bytes" + "crypto/md5" + "fmt" "io" "io/ioutil" "path/filepath" @@ -9,7 +12,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/s3iface" "github.com/cenkalti/backoff" - uuid "github.com/satori/go.uuid" ) type ( @@ -18,20 +20,24 @@ type ( // KeyPath is a string value which represents the s3 name space objects will be written to KeyPath string + + // ObjectKeyFunc is a function which generates the string to be used as the object key. + ObjectKeyFunc func([]byte) (string, error) ) -// ObjectKeyFunc is a function which generates the string to be used as the object key. -type ObjectKeyFunc func() (string, error) +func defaultObjectKeyFunc(b []byte) (string, error) { + return fmt.Sprintf("%x", md5.Sum(b)), nil +} -func defaultObjectKeyFunc() (string, error) { - var ( - u uuid.UUID - err error - ) - if u, err = uuid.NewV4(); err != nil { - return "", err - } - return u.String(), nil +// Ref represents the path to an object in S3 broken down by bucket, Path, and Object +// Bucket = "my-s3-bucket" +// Path = "some/path/within" +// Object = "an-object-in-s3.json" +// s3://my-s3-bucket/some/path/within/an-object-in-s3.json +type Ref struct { + Bucket string `json:"bucket"` + Path string `json:"path"` + Object string `json:"object"` } // Inflight is a structure which provides an interface to retrieving and writing data to s3, @@ -58,8 +64,8 @@ func NewInflight(bucket Bucket, keypath KeyPath, s3 s3iface.S3API) *Inflight { // Write will take the data given and attempt to put it in S3 // It then will return the S3 URI back to the caller so that the data may be passed between callers -func (i *Inflight) Write(data io.ReadSeeker) (ref *Ref, err error) { - objID, err := i.ObjectKeyFunc() +func (i *Inflight) Write(data []byte) (ref *Ref, err error) { + objID, err := i.ObjectKeyFunc(data) if err != nil { return nil, backoff.Permanent(err) } @@ -71,7 +77,7 @@ func (i *Inflight) Write(data io.ReadSeeker) (ref *Ref, err error) { } err = backoff.Retry( - i.tryWriteToS3(data, ref.Object), + i.tryWriteToS3(bytes.NewReader(data), ref.Object), backoff.NewExponentialBackOff(), ) diff --git a/inflight_test.go b/inflight_test.go index 0939446..e8ff2e6 100644 --- a/inflight_test.go +++ b/inflight_test.go @@ -49,7 +49,11 @@ func (m *mocks3PutObjectRequestRetryableErrorExpectSuccessAfterSecondAttempt) Pu } return s3.PutObjectRequest{ - Request: &aws.Request{Data: &s3.PutObjectOutput{}, Error: nil}} + Request: &aws.Request{ + Data: &s3.PutObjectOutput{}, + Error: nil, + }, + } } func TestNewInflightGivenBucketAndKeyExpectCorrectValues(t *testing.T) { @@ -89,8 +93,7 @@ func TestGetGivenObjectNotExistExpectError(t *testing.T) { func (m mocks3GetObjectRequestReturnBytes) GetObjectRequest(*s3.GetObjectInput) s3.GetObjectRequest { return s3.GetObjectRequest{ Request: &aws.Request{Data: &s3.GetObjectOutput{ - Body: ioutil.NopCloser(bytes.NewReader(m.bytesToReturn)), - }, Error: nil}, + Body: ioutil.NopCloser(bytes.NewReader(m.bytesToReturn))}, Error: nil}, } } func TestGetGivenObjectExistExpectCorrectBytes(t *testing.T) { @@ -115,15 +118,14 @@ func (m *mocks3GetObjectRequestRetryableErrorReturnBytesAfterSecondAttempt) GetO m.times++ return s3.GetObjectRequest{ Request: &aws.Request{Data: &s3.GetObjectOutput{ - Body: ioutil.NopCloser(bytes.NewReader([]byte{})), - }, Error: awserr.New("RequestTimeout", "", errors.New(""))}, + Body: ioutil.NopCloser(bytes.NewReader([]byte{}))}, Error: awserr.New("RequestTimeout", "", errors.New(""))}, } } return s3.GetObjectRequest{ Request: &aws.Request{Data: &s3.GetObjectOutput{ - Body: ioutil.NopCloser(bytes.NewReader(m.bytesToReturn)), - }, Error: nil}, + Body: ioutil.NopCloser(bytes.NewReader(m.bytesToReturn))}, + Error: nil}, } } func TestGetGivenObjectExistExpectRetryableErrorThenBytesReturned(t *testing.T) { @@ -153,7 +155,7 @@ func TestWriteGivenSomeBytesExpectRetryableErrorThenIdentifierReturned(t *testin } inflight := NewInflight(givenBucket, givenKeyPath, s3) - actualRef, err := inflight.Write(bytes.NewReader(givenBytes)) + actualRef, err := inflight.Write(givenBytes) if err != nil { t.Fail() } @@ -181,11 +183,11 @@ func TestWriteGivenSomeBytesButUUIDReturnsErrorExpectPermanentError(t *testing.T } inflight := NewInflight(givenBucket, givenKeyPath, s3) - inflight.ObjectKeyFunc = func() (string, error) { + inflight.ObjectKeyFunc = func(b []byte) (string, error) { return "", errors.New("") } - _, err := inflight.Write(bytes.NewReader(givenBytes)) + _, err := inflight.Write(givenBytes) if err == nil { t.Fail() } @@ -201,11 +203,11 @@ func TestWriteGivenSomeBytesButUUIDReturnsErrorExpectStringFromGenerator(t *test } inflight := NewInflight(givenBucket, givenKeyPath, s3) - inflight.ObjectKeyFunc = func() (string, error) { + inflight.ObjectKeyFunc = func(b []byte) (string, error) { return "from_the_func", nil } - ref, err := inflight.Write(bytes.NewReader(givenBytes)) + ref, err := inflight.Write(givenBytes) if err != nil && ref.Object != "from_the_func" { t.Fail() } diff --git a/ref.go b/ref.go deleted file mode 100644 index 4c4bb39..0000000 --- a/ref.go +++ /dev/null @@ -1,12 +0,0 @@ -package inflight - -// Ref represents the path to an object in S3 broken down by bucket, Path, and Object -// Bucket = "my-s3-bucket" -// Path = "some/path/within" -// Object = "an-object-in-s3.json" -// s3://my-s3-bucket/some/path/within/an-object-in-s3.json -type Ref struct { - Bucket string `json:"bucket"` - Path string `json:"path"` - Object string `json:"object"` -}