diff --git a/go.mod b/go.mod index 978a1a2..a5ce713 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,10 @@ module github.com/scylladb/gocqlx/v2 go 1.16 require ( - github.com/davecgh/go-spew v1.1.1 // indirect github.com/gocql/gocql v0.0.0-20200131111108-92af2e088537 github.com/golang/snappy v0.0.1 // indirect - github.com/google/go-cmp v0.2.0 + github.com/google/go-cmp v0.5.4 + github.com/psanford/memfs v0.0.0-20210214183328-a001468d78ef github.com/scylladb/go-reflectx v1.0.1 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a gopkg.in/inf.v0 v0.9.1 diff --git a/go.sum b/go.sum index e0c1551..e92d4c1 100644 --- a/go.sum +++ b/go.sum @@ -2,18 +2,14 @@ github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYE github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/gocql/gocql v0.0.0-20200131111108-92af2e088537 h1:NaMut1fdw76YYX/TPinSAbai4DShF5tPort3bHpET6g= github.com/gocql/gocql v0.0.0-20200131111108-92af2e088537/go.mod h1:DL0ekTmBSTdlNF25Orwt/JMzqIq3EJ4MVa/J/uK64OY= -github.com/golang/snappy v0.0.0-20170215233205-553a64147049 h1:K9KHZbXKpGydfDN0aZrsoHpLJlZsBrGMFWbgLDGnPZk= github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -21,14 +17,16 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/psanford/memfs v0.0.0-20210214183328-a001468d78ef h1:NKxTG6GVGbfMXc2mIk+KphcH6hagbVXhcFkbTgYleTI= +github.com/psanford/memfs v0.0.0-20210214183328-a001468d78ef/go.mod h1:tcaRap0jS3eifrEEllL6ZMd9dg8IlDpi2S1oARrQ+NI= github.com/scylladb/go-reflectx v1.0.1 h1:b917wZM7189pZdlND9PbIJ6NQxfDPfBvUaQ7cjj1iZQ= github.com/scylladb/go-reflectx v1.0.1/go.mod h1:rWnOfDIRWBGN0miMLIcoPt/Dhi2doCMZqwMCJ3KupFc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= diff --git a/migrate/checksum.go b/migrate/checksum.go index 681045d..f302f50 100644 --- a/migrate/checksum.go +++ b/migrate/checksum.go @@ -8,7 +8,7 @@ import ( "crypto/md5" "encoding/hex" "io" - "os" + "io/fs" ) var encode = hex.EncodeToString @@ -18,15 +18,15 @@ func checksum(b []byte) string { return encode(v[:]) } -func fileChecksum(path string) (string, error) { - f, err := os.Open(path) +func fileChecksum(f fs.FS, path string) (string, error) { + file, err := f.Open(path) if err != nil { return "", nil } - defer f.Close() + defer file.Close() h := md5.New() - if _, err := io.Copy(h, f); err != nil { + if _, err := io.Copy(h, file); err != nil { return "", err } v := h.Sum(nil) diff --git a/migrate/checksum_test.go b/migrate/checksum_test.go index ee1636d..cc9dbd6 100644 --- a/migrate/checksum_test.go +++ b/migrate/checksum_test.go @@ -4,10 +4,13 @@ package migrate -import "testing" +import ( + "os" + "testing" +) func TestFileChecksum(t *testing.T) { - c, err := fileChecksum("testdata/file") + c, err := fileChecksum(os.DirFS("testdata"), "file") if err != nil { t.Fatal(err) } diff --git a/migrate/migrate.go b/migrate/migrate.go index d913109..077ffff 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "io" + "io/fs" "io/ioutil" "os" "path/filepath" @@ -90,10 +91,20 @@ func ensureInfoTable(ctx context.Context, session gocqlx.Session) error { return session.ContextQuery(ctx, infoSchema, nil).ExecRelease() } -// Migrate reads the cql files from a directory and applies required migrations. -// It also supports code based migrations, see Callback and CallbackFunc. -// Any comment in form `-- CALL ;` will trigger an CallComment callback. +// Migrate is a wrapper around FromFS. +// It executes migrations from a directory on disk. +// +// Deprecated: use FromFS instead func Migrate(ctx context.Context, session gocqlx.Session, dir string) error { + return FromFS(ctx, session, os.DirFS(dir)) +} + +// FromFS executes new CQL files from a file system abstraction (io/fs.FS). +// The provided FS has to be a flat directory containing *.cql files. +// +// It supports code based migrations, see Callback and CallbackFunc. +// Any comment in form `-- CALL ;` will trigger an CallComment callback. +func FromFS(ctx context.Context, session gocqlx.Session, f fs.FS) error { // get database migrations dbm, err := List(ctx, session) if err != nil { @@ -101,26 +112,26 @@ func Migrate(ctx context.Context, session gocqlx.Session, dir string) error { } // get file migrations - fm, err := filepath.Glob(filepath.Join(dir, "*.cql")) + fm, err := fs.Glob(f, "*.cql") if err != nil { - return fmt.Errorf("list migrations in %q: %s", dir, err) + return fmt.Errorf("list migrations: %w", err) } if len(fm) == 0 { - return fmt.Errorf("no migration files found in %q", dir) + return fmt.Errorf("no migration files found") } sort.Strings(fm) // verify migrations if len(dbm) > len(fm) { - return fmt.Errorf("database is ahead of %q", dir) + return fmt.Errorf("database is ahead") } for i := 0; i < len(dbm); i++ { - if dbm[i].Name != filepath.Base(fm[i]) { - fmt.Println(dbm[i].Name, filepath.Base(fm[i]), i) + if dbm[i].Name != fm[i] { + fmt.Println(dbm[i].Name, fm[i], i) return errors.New("inconsistent migrations") } - c, err := fileChecksum(fm[i]) + c, err := fileChecksum(f, fm[i]) if err != nil { return fmt.Errorf("calculate checksum for %q: %s", fm[i], err) } @@ -132,13 +143,13 @@ func Migrate(ctx context.Context, session gocqlx.Session, dir string) error { // apply migrations if len(dbm) > 0 { last := len(dbm) - 1 - if err := applyMigration(ctx, session, fm[last], dbm[last].Done); err != nil { + if err := applyMigration(ctx, session, f, fm[last], dbm[last].Done); err != nil { return fmt.Errorf("apply migration %q: %s", fm[last], err) } } for i := len(dbm); i < len(fm); i++ { - if err := applyMigration(ctx, session, fm[i], 0); err != nil { + if err := applyMigration(ctx, session, f, fm[i], 0); err != nil { return fmt.Errorf("apply migration %q: %s", fm[i], err) } } @@ -150,14 +161,14 @@ func Migrate(ctx context.Context, session gocqlx.Session, dir string) error { return nil } -func applyMigration(ctx context.Context, session gocqlx.Session, path string, done int) error { - f, err := os.Open(path) +func applyMigration(ctx context.Context, session gocqlx.Session, f fs.FS, path string, done int) error { + file, err := f.Open(path) if err != nil { return err } - b, err := ioutil.ReadAll(f) - f.Close() + b, err := ioutil.ReadAll(file) + file.Close() if err != nil { return err } diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index 9c46ce8..35986b6 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -9,12 +9,11 @@ package migrate_test import ( "context" "fmt" - "io/ioutil" - "os" - "path/filepath" + "io/fs" "strings" "testing" + "github.com/psanford/memfs" "github.com/scylladb/gocqlx/v2" . "github.com/scylladb/gocqlx/v2/gocqlxtest" "github.com/scylladb/gocqlx/v2/migrate" @@ -52,10 +51,7 @@ func TestMigration(t *testing.T) { ctx := context.Background() t.Run("init", func(t *testing.T) { - dir := makeMigrationDir(t, 2) - defer os.Remove(dir) - - if err := migrate.Migrate(ctx, session, dir); err != nil { + if err := migrate.FromFS(ctx, session, makeTestFS(2)); err != nil { t.Fatal(err) } if c := countMigrations(t, session); c != 2 { @@ -64,10 +60,7 @@ func TestMigration(t *testing.T) { }) t.Run("update", func(t *testing.T) { - dir := makeMigrationDir(t, 4) - defer os.Remove(dir) - - if err := migrate.Migrate(ctx, session, dir); err != nil { + if err := migrate.FromFS(ctx, session, makeTestFS(4)); err != nil { t.Fatal(err) } if c := countMigrations(t, session); c != 4 { @@ -76,10 +69,8 @@ func TestMigration(t *testing.T) { }) t.Run("ahead", func(t *testing.T) { - dir := makeMigrationDir(t, 2) - defer os.Remove(dir) - - if err := migrate.Migrate(ctx, session, dir); err == nil || !strings.Contains(err.Error(), "ahead") { + err := migrate.FromFS(ctx, session, makeTestFS(2)) + if err == nil || !strings.Contains(err.Error(), "ahead") { t.Fatal("expected error") } else { t.Log(err) @@ -87,12 +78,10 @@ func TestMigration(t *testing.T) { }) t.Run("tempered with file", func(t *testing.T) { - dir := makeMigrationDir(t, 4) - defer os.Remove(dir) - - appendMigrationFile(t, dir, 3, "\nSELECT * FROM bla;\n") + f := makeTestFS(4) + writeFile(f, 3, "SELECT * FROM bla;") - if err := migrate.Migrate(ctx, session, dir); err == nil || !strings.Contains(err.Error(), "tempered") { + if err := migrate.FromFS(ctx, session, f); err == nil || !strings.Contains(err.Error(), "tempered") { t.Fatal("expected error") } else { t.Log(err) @@ -109,19 +98,11 @@ func TestMigrationNoSemicolon(t *testing.T) { t.Fatal(err) } - ctx := context.Background() - - dir := makeMigrationDir(t, 1) - defer os.Remove(dir) + f := makeTestFS(0) + f.WriteFile("0.cql", []byte(fmt.Sprintf(insertMigrate, 0)+";"+fmt.Sprintf(insertMigrate, 1)), fs.ModePerm) - f, err := os.OpenFile(filepath.Join(dir, "0.cql"), os.O_WRONLY|os.O_APPEND, 0) - if err != nil { - t.Fatal(err) - } - fmt.Fprintf(f, insertMigrate, 0) // note no ; at the end - f.Close() - - if err := migrate.Migrate(ctx, session, dir); err != nil { + ctx := context.Background() + if err := migrate.FromFS(ctx, session, f); err != nil { t.Fatal(err) } if c := countMigrations(t, session); c != 2 { @@ -231,62 +212,38 @@ func TestMigrationCallback(t *testing.T) { ctx := context.Background() t.Run("init", func(t *testing.T) { - dir := makeMigrationDir(t, 2) - defer os.Remove(dir) + f := makeTestFS(2) reset() - if err := migrate.Migrate(ctx, session, dir); err != nil { + if err := migrate.FromFS(ctx, session, f); err != nil { t.Fatal(err) } assertCallbacks(t, 2, 2, 0) }) t.Run("no duplicate calls", func(t *testing.T) { - dir := makeMigrationDir(t, 4) - defer os.Remove(dir) + f := makeTestFS(4) reset() - if err := migrate.Migrate(ctx, session, dir); err != nil { + if err := migrate.FromFS(ctx, session, f); err != nil { t.Fatal(err) } assertCallbacks(t, 2, 2, 0) }) t.Run("in calls", func(t *testing.T) { - dir := makeMigrationDir(t, 4) - defer os.Remove(dir) + f := makeTestFS(4) + writeFile(f, 4, "\n-- CALL Foo;\n") + writeFile(f, 5, "\n-- CALL Bar;\n") reset() - appendMigrationFile(t, dir, 4, "\n-- CALL Foo;\n") - appendMigrationFile(t, dir, 5, "\n-- CALL Bar;\n") - - if err := migrate.Migrate(ctx, session, dir); err != nil { + if err := migrate.FromFS(ctx, session, f); err != nil { t.Fatal(err) } assertCallbacks(t, 2, 2, 2) }) } -func makeMigrationDir(tb testing.TB, n int) (dir string) { - tb.Helper() - - dir, err := ioutil.TempDir("", "gocqlx_migrate") - if err != nil { - tb.Fatal(err) - } - - for i := 0; i < n; i++ { - path := migrateFilePath(dir, i) - cql := []byte(fmt.Sprintf(insertMigrate, i) + ";") - if err := ioutil.WriteFile(path, cql, os.ModePerm); err != nil { - os.Remove(dir) - tb.Fatal(err) - } - } - - return dir -} - func countMigrations(tb testing.TB, session gocqlx.Session) int { tb.Helper() @@ -297,17 +254,14 @@ func countMigrations(tb testing.TB, session gocqlx.Session) int { return v } -func appendMigrationFile(tb testing.TB, dir string, i int, text string) { - path := migrateFilePath(dir, i) - f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, os.ModePerm) - if err != nil { - tb.Fatal(err) - } - if _, err := f.WriteString(text); err != nil { - tb.Fatal(err) +func makeTestFS(n int) *memfs.FS { + f := memfs.New() + for i := 0; i < n; i++ { + writeFile(f, i, fmt.Sprintf(insertMigrate, i)+";") } + return f } -func migrateFilePath(dir string, i int) string { - return filepath.Join(dir, fmt.Sprint(i, ".cql")) +func writeFile(f *memfs.FS, i int, text string) { + f.WriteFile(fmt.Sprint(i, ".cql"), []byte(text), fs.ModePerm) }