diff --git a/.travis.yml b/.travis.yml index 91179e9..7f2aca0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,6 @@ language: go -env: - - DEP_VERSION=0.5.0" -before_install: - - curl -L -s https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 -o $GOPATH/bin/dep - - chmod +x $GOPATH/bin/dep install: - - dep ensure -v + - go get ./... go: - '1.x' os: @@ -13,7 +8,8 @@ os: matrix: fast_finish: true script: - - go test ./... -race -coverprofile=coverage.txt -covermode=atomic + - make get-tools + - make test after_success: - bash <(curl -s https://codecov.io/bash) diff --git a/Gopkg.lock b/Gopkg.lock deleted file mode 100644 index 7317a12..0000000 --- a/Gopkg.lock +++ /dev/null @@ -1,61 +0,0 @@ -# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. - - -[[projects]] - digest = "1:90fc5a41688f499df4279dca3329c183b0a9e03d8ee16e3d7f09a6cfa9669372" - name = "github.com/aws/aws-sdk-go-v2" - packages = [ - "aws", - "aws/awserr", - "aws/signer/v4", - "internal/awsutil", - "internal/sdk", - "private/protocol", - "private/protocol/query", - "private/protocol/query/queryutil", - "private/protocol/rest", - "private/protocol/restxml", - "private/protocol/xml", - "private/protocol/xml/xmlutil", - "service/s3", - "service/s3/s3iface", - ] - pruneopts = "UT" - revision = "c2b476bac53067cebc587747e866881b2e383625" - version = "v0.7.0" - -[[projects]] - digest = "1:553f73a4171c265045ae4f5d3122429ecf2c9c0c232c91f336127fe45480104a" - name = "github.com/cenkalti/backoff" - packages = ["."] - pruneopts = "UT" - revision = "62661b46c4093e2c1f38d943e663db1a29873e80" - version = "v2.1.0" - -[[projects]] - digest = "1:bb81097a5b62634f3e9fec1014657855610c82d19b9a40c17612e32651e35dca" - name = "github.com/jmespath/go-jmespath" - packages = ["."] - pruneopts = "UT" - revision = "c2b33e84" - -[[projects]] - digest = "1:274f67cb6fed9588ea2521ecdac05a6d62a8c51c074c1fccc6a49a40ba80e925" - name = "github.com/satori/go.uuid" - packages = ["."] - pruneopts = "UT" - revision = "f58768cc1a7a7e77a3bd49e98cdd21419399b6a3" - version = "v1.2.0" - -[solve-meta] - analyzer-name = "dep" - analyzer-version = 1 - input-imports = [ - "github.com/aws/aws-sdk-go-v2/aws", - "github.com/aws/aws-sdk-go-v2/service/s3", - "github.com/aws/aws-sdk-go-v2/service/s3/s3iface", - "github.com/cenkalti/backoff", - "github.com/satori/go.uuid", - ] - solver-name = "gps-cdcl" - solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml deleted file mode 100644 index 1f84ca3..0000000 --- a/Gopkg.toml +++ /dev/null @@ -1,42 +0,0 @@ -# Gopkg.toml example -# -# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html -# for detailed Gopkg.toml documentation. -# -# required = ["github.com/user/thing/cmd/thing"] -# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] -# -# [[constraint]] -# name = "github.com/user/project" -# version = "1.0.0" -# -# [[constraint]] -# name = "github.com/user/project2" -# branch = "dev" -# source = "github.com/myfork/project2" -# -# [[override]] -# name = "github.com/x/y" -# version = "2.4.0" -# -# [prune] -# non-go = false -# go-tests = true -# unused-packages = true - - -[[constraint]] - name = "github.com/aws/aws-sdk-go-v2" - version = "0.7.0" - -[[constraint]] - name = "github.com/cenkalti/backoff" - version = "2.1.0" - -[[constraint]] - name = "github.com/satori/go.uuid" - version = "1.2.0" - -[prune] - go-tests = true - unused-packages = true diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..4159135 --- /dev/null +++ b/Makefile @@ -0,0 +1,8 @@ +.PHONY: get-tools +get-tools: + @echo "+ $@: retrieving build/test dependencies " + @go get -u -v github.com/google/go-cmp/... + +test: + @echo 'run `make get-tools` if you are unable to run this test`' + @go test ./... -race -coverprofile=coverage.txt -covermode=atomic \ No newline at end of file diff --git a/inflight.go b/inflight.go index da02202..0fdcc98 100644 --- a/inflight.go +++ b/inflight.go @@ -20,33 +20,57 @@ type ( KeyPath string ) +// ObjectKeyFunc is a function which generates the string to be used as the object key. +type ObjectKeyFunc func() (string, error) + +func defaultObjectKeyFunc() (string, error) { + var ( + u uuid.UUID + err error + ) + if u, err = uuid.NewV4(); err != nil { + return "", err + } + return u.String(), nil +} + // Inflight is a structure which provides an interface to retrieving and writing data to s3, // it doesn't care about what data you're writing, just provides an easy way to get to it type Inflight struct { s3iface.S3API Bucket Bucket KeyPath KeyPath + + // ObjectKeyFunc will be called when Inflight#Write(io.ReadSeeker) is invoked. + // The data will be given the name that this function generates. + ObjectKeyFunc ObjectKeyFunc } // NewInflight Creates a reference to an Inflight struct func NewInflight(bucket Bucket, keypath KeyPath, s3 s3iface.S3API) *Inflight { return &Inflight{ - Bucket: bucket, - KeyPath: keypath, - S3API: s3, + Bucket: bucket, + KeyPath: keypath, + S3API: s3, + ObjectKeyFunc: ObjectKeyFunc(defaultObjectKeyFunc), } } // Write will take the data given and attempt to put it in S3 // It then will return the S3 URI back to the caller so that the data may be passed between callers -func (i *Inflight) Write(data io.ReadSeeker) (*Ref, error) { - ref := &Ref{ +func (i *Inflight) Write(data io.ReadSeeker) (ref *Ref, err error) { + objID, err := i.ObjectKeyFunc() + if err != nil { + return nil, backoff.Permanent(err) + } + + ref = &Ref{ Bucket: string(i.Bucket), Path: string(i.KeyPath), - Object: uuid.NewV4().String(), + Object: objID, } - err := backoff.Retry( + err = backoff.Retry( i.tryWriteToS3(data, ref.Object), backoff.NewExponentialBackOff(), ) diff --git a/inflight_test.go b/inflight_test.go index 89b03d4..0939446 100644 --- a/inflight_test.go +++ b/inflight_test.go @@ -4,9 +4,10 @@ import ( "bytes" "errors" "io/ioutil" - "reflect" "testing" + "github.com/google/go-cmp/cmp" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/awserr" @@ -57,14 +58,15 @@ func TestNewInflightGivenBucketAndKeyExpectCorrectValues(t *testing.T) { givenKeyPath := KeyPath("key/path") expected := &Inflight{ - Bucket: Bucket("bucket"), - KeyPath: KeyPath("key/path"), - S3API: s3, + Bucket: Bucket("bucket"), + KeyPath: KeyPath("key/path"), + S3API: s3, + ObjectKeyFunc: ObjectKeyFunc(defaultObjectKeyFunc), } actual := NewInflight(givenBucket, givenKeyPath, s3) - if !reflect.DeepEqual(expected, actual) { + if cmp.Equal(expected, actual) { t.Fail() } } @@ -168,3 +170,43 @@ func TestWriteGivenSomeBytesExpectRetryableErrorThenIdentifierReturned(t *testin t.Fail() } } + +func TestWriteGivenSomeBytesButUUIDReturnsErrorExpectPermanentError(t *testing.T) { + givenBytes := []byte("hi") + givenBucket := Bucket("a_bucket") + + givenKeyPath := KeyPath("a/key/path") + s3 := &mocks3PutObjectRequestRetryableErrorExpectSuccessAfterSecondAttempt{ + givenBytes: givenBytes, + } + + inflight := NewInflight(givenBucket, givenKeyPath, s3) + inflight.ObjectKeyFunc = func() (string, error) { + return "", errors.New("") + } + + _, err := inflight.Write(bytes.NewReader(givenBytes)) + if err == nil { + t.Fail() + } +} + +func TestWriteGivenSomeBytesButUUIDReturnsErrorExpectStringFromGenerator(t *testing.T) { + givenBytes := []byte("hi") + givenBucket := Bucket("a_bucket") + + givenKeyPath := KeyPath("a/key/path") + s3 := &mocks3PutObjectRequestRetryableErrorExpectSuccessAfterSecondAttempt{ + givenBytes: givenBytes, + } + + inflight := NewInflight(givenBucket, givenKeyPath, s3) + inflight.ObjectKeyFunc = func() (string, error) { + return "from_the_func", nil + } + + ref, err := inflight.Write(bytes.NewReader(givenBytes)) + if err != nil && ref.Object != "from_the_func" { + t.Fail() + } +}