diff --git a/Earthfile b/Earthfile index 504e8862..ffdcf495 100644 --- a/Earthfile +++ b/Earthfile @@ -66,7 +66,7 @@ code: COPY --dir buildkitd/buildkitd.go buildkitd/settings.go buildkitd/certificates.go buildkitd/ COPY --dir earthfile2llb/*.go earthfile2llb/ COPY --dir ast/antlrhandler ast/spec ast/hint ast/command ast/commandflag ast/*.go ast/ - COPY --dir inputgraph/*.go inputgraph/ + COPY --dir inputgraph/*.go inputgraph/testdata inputgraph/ # update-buildkit updates earthly's buildkit dependency. update-buildkit: diff --git a/earthfile2llb/interpreter.go b/earthfile2llb/interpreter.go index e3517a0a..4b5801b6 100644 --- a/earthfile2llb/interpreter.go +++ b/earthfile2llb/interpreter.go @@ -1720,7 +1720,7 @@ func (i *Interpreter) handleWithDocker(ctx context.Context, cmd spec.Command) er }) } for _, loadStr := range opts.Loads { - loadImg, loadTarget, flagArgs, err := parseLoad(loadStr) + loadImg, loadTarget, flagArgs, err := ParseLoad(loadStr) if err != nil { return i.wrapError(err, cmd.SourceLocation, "parse load") } @@ -2109,7 +2109,9 @@ func unescapeSlashPlus(str string) string { return strings.ReplaceAll(str, "\\+", "+") } -func parseLoad(loadStr string) (image string, target string, extraArgs []string, err error) { +// ParseLoad splits a --load value into the image, target, & extra args. +// Example: --load my-image=(+target --arg1 foo --arg2=bar) +func ParseLoad(loadStr string) (image string, target string, extraArgs []string, err error) { words := strings.SplitN(loadStr, " ", 2) if len(words) == 0 { return "", "", nil, nil diff --git a/inputgraph/inputgraph.go b/inputgraph/inputgraph.go index 6db7d562..791a467f 100644 --- a/inputgraph/inputgraph.go +++ b/inputgraph/inputgraph.go @@ -14,10 +14,10 @@ import ( "github.com/earthly/earthly/buildcontext" "github.com/earthly/earthly/conslogging" "github.com/earthly/earthly/domain" + "github.com/earthly/earthly/earthfile2llb" "github.com/earthly/earthly/util/buildkitskipper/hasher" "github.com/earthly/earthly/util/flagutil" "github.com/earthly/earthly/util/stringutil" - "github.com/earthly/earthly/variables" "github.com/pkg/errors" ) @@ -249,11 +249,39 @@ func (l *loader) handleCommand(ctx context.Context, cmd spec.Command) error { return l.handleCopy(ctx, cmd) case command.Pipeline: return l.handlePipeline(ctx, cmd) + case command.SaveImage: + return l.handleSaveImage(ctx, cmd) + case command.Run: + return l.handleRun(ctx, cmd) + case command.Arg: + return l.handleArg(ctx, cmd) + case command.SaveArtifact: + return l.handleSaveArtifact(ctx, cmd) default: - return nil + return errors.Errorf("unhandled command: %s", cmd.Name) } } +func (l *loader) handleSaveImage(ctx context.Context, cmd spec.Command) error { + l.hashCommand(cmd) + return nil +} + +func (l *loader) handleSaveArtifact(ctx context.Context, cmd spec.Command) error { + l.hashCommand(cmd) + return nil +} + +func (l *loader) handleRun(ctx context.Context, cmd spec.Command) error { + l.hashCommand(cmd) + return nil +} + +func (l *loader) handleArg(ctx context.Context, cmd spec.Command) error { + l.hashCommand(cmd) + return nil +} + func (l *loader) handleWith(ctx context.Context, with spec.WithStatement) error { if with.Command.Name != command.Docker { return errors.Wrap(ErrUnableToDetermineHash, "expected WITH DOCKER") @@ -276,11 +304,14 @@ func (l *loader) handleWithDocker(ctx context.Context, cmd spec.Command) error { if strings.Contains(load, "$") { return errors.Wrap(ErrUnableToDetermineHash, "unable to handle arg in WITH DOCKER --load") } - _, v, _ := variables.ParseKeyValue(load) - if v == "" { - return errors.Wrap(ErrUnableToDetermineHash, "unable to handle WITH DOCKER --load with implicit image name (hint: specify the image name rather than relying on the target's SAVE IMAGE command)") + _, target, extraArgs, err := earthfile2llb.ParseLoad(load) + if err != nil { + return errors.Wrap(err, "failed to parse --load value") + } + if len(extraArgs) > 0 { + return errors.Wrap(ErrUnableToDetermineHash, "--load args are not yet supported") } - err := l.loadTargetFromString(ctx, v) + err = l.loadTargetFromString(ctx, target) if err != nil { return err } diff --git a/inputgraph/inputgraph_test.go b/inputgraph/inputgraph_test.go new file mode 100644 index 00000000..59daa317 --- /dev/null +++ b/inputgraph/inputgraph_test.go @@ -0,0 +1,141 @@ +package inputgraph + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/earthly/earthly/conslogging" + "github.com/earthly/earthly/domain" + "github.com/stretchr/testify/require" +) + +func TestHashTargetWithDocker(t *testing.T) { + r := require.New(t) + target := domain.Target{ + LocalPath: "./testdata/with-docker", + Target: "with-docker-load", + } + + ctx := context.Background() + cons := conslogging.New(os.Stderr, &sync.Mutex{}, conslogging.NoColor, 0, conslogging.Info) + + org, project, hash, err := HashTarget(ctx, target, cons) + r.NoError(err) + r.Equal("earthly-technologies", org) + r.Equal("core", project) + + hex := fmt.Sprintf("%x", hash) + r.Equal("9d2903bc18c99831f4a299090abaf94d25d89321", hex) + + path := "./testdata/with-docker/Earthfile" + + tmpDir, err := os.MkdirTemp(os.TempDir(), "with-docker") + r.NoError(err) + + tmpFile := filepath.Join(tmpDir, "Earthfile") + defer func() { + err = os.RemoveAll(tmpDir) + r.NoError(err) + }() + + err = copyFile(path, tmpFile) + r.NoError(err) + + err = replaceInFile(tmpFile, "saved:latest", "other:latest") + r.NoError(err) + + target = domain.Target{ + LocalPath: tmpDir, + Target: "with-docker-load", + } + + _, _, hash, err = HashTarget(ctx, target, cons) + r.NoError(err) + + hex = fmt.Sprintf("%x", hash) + r.Equal("84b6f722421695a7ded144c1b72efb3b8f3339c6", hex) +} + +func copyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.Create(dst) + if err != nil { + return err + } + defer out.Close() + _, err = io.Copy(out, in) + if err != nil { + return err + } + return nil +} + +func replaceInFile(path, find, replace string) error { + f, err := os.OpenFile(path, os.O_RDWR, 0) + if err != nil { + return err + } + defer f.Close() + + dataBytes, err := io.ReadAll(f) + if err != nil { + return err + } + + data := string(dataBytes) + data = strings.ReplaceAll(data, find, replace) + _, err = f.Seek(0, 0) + if err != nil { + return err + } + + _, err = f.WriteString(data) + if err != nil { + return err + } + + return nil +} + +func TestHashTargetWithDockerNoAlias(t *testing.T) { + r := require.New(t) + target := domain.Target{ + LocalPath: "./testdata/with-docker", + Target: "with-docker-load-no-alias", + } + + ctx := context.Background() + cons := conslogging.New(os.Stderr, &sync.Mutex{}, conslogging.NoColor, 0, conslogging.Info) + + org, project, hash, err := HashTarget(ctx, target, cons) + r.NoError(err) + r.Equal("earthly-technologies", org) + r.Equal("core", project) + + hex := fmt.Sprintf("%x", hash) + r.Equal("d73e37689c7780cbff2cba2de1a23141618b7b14", hex) +} + +func TestHashTargetWithDockerArgs(t *testing.T) { + r := require.New(t) + target := domain.Target{ + LocalPath: "./testdata/with-docker", + Target: "with-docker-load-args", + } + + ctx := context.Background() + cons := conslogging.New(os.Stderr, &sync.Mutex{}, conslogging.NoColor, 0, conslogging.Info) + + _, _, _, err := HashTarget(ctx, target, cons) + r.Error(err) +} diff --git a/inputgraph/testdata/with-docker/Earthfile b/inputgraph/testdata/with-docker/Earthfile new file mode 100644 index 00000000..9ca57c79 --- /dev/null +++ b/inputgraph/testdata/with-docker/Earthfile @@ -0,0 +1,29 @@ +VERSION 0.7 + +PROJECT earthly-technologies/core + +load-target: + ARG foo=1 + FROM alpine + RUN echo "hi" > /tmp/x + SAVE IMAGE saved:latest + +with-docker-load: + FROM earthly/dind:alpine + WITH DOCKER --load saved:latest=+load-target + RUN echo "loaded" + END + +with-docker-load-no-alias: + BUILD +load-target + FROM earthly/dind:alpine + WITH DOCKER --load +load-target + RUN echo "loaded" + END + +with-docker-load-args: + BUILD +load-target + FROM earthly/dind:alpine + WITH DOCKER --load foo=(+load-target --foo=2) + RUN echo "loaded" + END diff --git a/tests/autoskip/Earthfile b/tests/autoskip/Earthfile index 7a99004b..ac3e1243 100644 --- a/tests/autoskip/Earthfile +++ b/tests/autoskip/Earthfile @@ -47,15 +47,15 @@ test-auto-skip-requires-project: test-auto-skip-wait: DO --pass-args +RUN_EARTHLY_ARGS --earthfile=wait.earth --target=+test --output_contains="not skipped" - DO --pass-args +RUN_EARTHLY_ARGS --earthfile=wait.earth --target=+test --output_contains="8be037baa7b4d09a8e2a37f74154d319d64c996a" + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=wait.earth --target=+test --output_contains="3deeaceb7e263a72114ebc2da62d9fecc87506f3" test-auto-skip-if-else: DO --pass-args +RUN_EARTHLY_ARGS --earthfile=if-else.earth --target=+test --output_contains="condition ok" - DO --pass-args +RUN_EARTHLY_ARGS --earthfile=if-else.earth --target=+test --output_contains="40f57fc7955914f3a954c6bd9a2ebe48d0b14f40" + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=if-else.earth --target=+test --output_contains="26ac9a5c6e4893049c218da18decfba3caff1746" test-auto-skip-for-in: DO --pass-args +RUN_EARTHLY_ARGS --earthfile=for.earth --target=+test --output_contains="hello 3" - DO --pass-args +RUN_EARTHLY_ARGS --earthfile=for.earth --target=+test --output_contains="3d7d7fec7efc5a746e9ac658160427decbf58a0b" + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=for.earth --target=+test --output_contains="ac33a31a36f496ed219293301bb44513a3a52d28" test-auto-skip-copy-glob: DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --should_fail=true --target=+globstar --output_contains="globstar (\*\*) not supported" @@ -63,24 +63,20 @@ test-auto-skip-copy-glob: COPY --dir glob . DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+dir --output_contains="glob/subdir/hello.txt" - DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+dir --output_contains="2d1b86207f72caedd143863a859b788b11278a06" + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+dir --output_contains="7dc88536e40a00684aeeb5ef3d6621d1e867aab2" DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob --output_contains="glob/subdir/hello.txt" - DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob --output_contains="17446ba4f7a3388ce0c8d57ea44089758c15602a" + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob --output_contains="9a45ff884f09e576caf91fed3d3a209b109c68e4" DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob-mid-path --output_contains="glob/subdir/hello.txt" - DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob-mid-path --output_contains="2e539ff29b24e54a3fe2c38534da68071b86c84b" + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob-mid-path --output_contains="cec86ff354a3064adda819695eb91db965bf0b3e" DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob-dir --output_contains="glob/subdir/hello.txt" - DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob-dir --output_contains="62b16b26d69ca13260f1f62883df8ff30169f4c4" + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob-dir --output_contains="111c0b03e1c36e9e55740cade50719e4c6e4e0ec" RUN echo data > glob/subdir/new.txt DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob-dir --output_contains="glob/subdir/new.txt" -test-invalid: - COPY glob/*/hello.txt / - RUN --no-cache ls -l / - RUN_EARTHLY_ARGS: COMMAND ARG earthfile