Skip to content

Commit

Permalink
Auto-skip: finish support for WITH DOCKER --load (#3388)
Browse files Browse the repository at this point in the history
Extends auto-skip to handle SAVE IMAGE, and removes invalid "unable to
handle WITH DOCKER --load with implicit image name" error.
  • Loading branch information
mikejholly authored Oct 17, 2023
1 parent d919ffa commit 17ed280
Show file tree
Hide file tree
Showing 6 changed files with 219 additions and 20 deletions.
2 changes: 1 addition & 1 deletion Earthfile
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ code:
COPY --dir buildkitd/buildkitd.go buildkitd/settings.go buildkitd/certificates.go buildkitd/
COPY --dir earthfile2llb/*.go earthfile2llb/
COPY --dir ast/antlrhandler ast/spec ast/hint ast/command ast/commandflag ast/*.go ast/
COPY --dir inputgraph/*.go inputgraph/
COPY --dir inputgraph/*.go inputgraph/testdata inputgraph/

# update-buildkit updates earthly's buildkit dependency.
update-buildkit:
Expand Down
6 changes: 4 additions & 2 deletions earthfile2llb/interpreter.go
Original file line number Diff line number Diff line change
Expand Up @@ -1720,7 +1720,7 @@ func (i *Interpreter) handleWithDocker(ctx context.Context, cmd spec.Command) er
})
}
for _, loadStr := range opts.Loads {
loadImg, loadTarget, flagArgs, err := parseLoad(loadStr)
loadImg, loadTarget, flagArgs, err := ParseLoad(loadStr)
if err != nil {
return i.wrapError(err, cmd.SourceLocation, "parse load")
}
Expand Down Expand Up @@ -2109,7 +2109,9 @@ func unescapeSlashPlus(str string) string {
return strings.ReplaceAll(str, "\\+", "+")
}

func parseLoad(loadStr string) (image string, target string, extraArgs []string, err error) {
// ParseLoad splits a --load value into the image, target, & extra args.
// Example: --load my-image=(+target --arg1 foo --arg2=bar)
func ParseLoad(loadStr string) (image string, target string, extraArgs []string, err error) {
words := strings.SplitN(loadStr, " ", 2)
if len(words) == 0 {
return "", "", nil, nil
Expand Down
43 changes: 37 additions & 6 deletions inputgraph/inputgraph.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@ import (
"github.com/earthly/earthly/buildcontext"
"github.com/earthly/earthly/conslogging"
"github.com/earthly/earthly/domain"
"github.com/earthly/earthly/earthfile2llb"
"github.com/earthly/earthly/util/buildkitskipper/hasher"
"github.com/earthly/earthly/util/flagutil"
"github.com/earthly/earthly/util/stringutil"
"github.com/earthly/earthly/variables"
"github.com/pkg/errors"
)

Expand Down Expand Up @@ -249,11 +249,39 @@ func (l *loader) handleCommand(ctx context.Context, cmd spec.Command) error {
return l.handleCopy(ctx, cmd)
case command.Pipeline:
return l.handlePipeline(ctx, cmd)
case command.SaveImage:
return l.handleSaveImage(ctx, cmd)
case command.Run:
return l.handleRun(ctx, cmd)
case command.Arg:
return l.handleArg(ctx, cmd)
case command.SaveArtifact:
return l.handleSaveArtifact(ctx, cmd)
default:
return nil
return errors.Errorf("unhandled command: %s", cmd.Name)
}
}

func (l *loader) handleSaveImage(ctx context.Context, cmd spec.Command) error {
l.hashCommand(cmd)
return nil
}

func (l *loader) handleSaveArtifact(ctx context.Context, cmd spec.Command) error {
l.hashCommand(cmd)
return nil
}

func (l *loader) handleRun(ctx context.Context, cmd spec.Command) error {
l.hashCommand(cmd)
return nil
}

func (l *loader) handleArg(ctx context.Context, cmd spec.Command) error {
l.hashCommand(cmd)
return nil
}

func (l *loader) handleWith(ctx context.Context, with spec.WithStatement) error {
if with.Command.Name != command.Docker {
return errors.Wrap(ErrUnableToDetermineHash, "expected WITH DOCKER")
Expand All @@ -276,11 +304,14 @@ func (l *loader) handleWithDocker(ctx context.Context, cmd spec.Command) error {
if strings.Contains(load, "$") {
return errors.Wrap(ErrUnableToDetermineHash, "unable to handle arg in WITH DOCKER --load")
}
_, v, _ := variables.ParseKeyValue(load)
if v == "" {
return errors.Wrap(ErrUnableToDetermineHash, "unable to handle WITH DOCKER --load with implicit image name (hint: specify the image name rather than relying on the target's SAVE IMAGE command)")
_, target, extraArgs, err := earthfile2llb.ParseLoad(load)
if err != nil {
return errors.Wrap(err, "failed to parse --load value")
}
if len(extraArgs) > 0 {
return errors.Wrap(ErrUnableToDetermineHash, "--load args are not yet supported")
}
err := l.loadTargetFromString(ctx, v)
err = l.loadTargetFromString(ctx, target)
if err != nil {
return err
}
Expand Down
141 changes: 141 additions & 0 deletions inputgraph/inputgraph_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
package inputgraph

import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
"testing"

"github.com/earthly/earthly/conslogging"
"github.com/earthly/earthly/domain"
"github.com/stretchr/testify/require"
)

func TestHashTargetWithDocker(t *testing.T) {
r := require.New(t)
target := domain.Target{
LocalPath: "./testdata/with-docker",
Target: "with-docker-load",
}

ctx := context.Background()
cons := conslogging.New(os.Stderr, &sync.Mutex{}, conslogging.NoColor, 0, conslogging.Info)

org, project, hash, err := HashTarget(ctx, target, cons)
r.NoError(err)
r.Equal("earthly-technologies", org)
r.Equal("core", project)

hex := fmt.Sprintf("%x", hash)
r.Equal("9d2903bc18c99831f4a299090abaf94d25d89321", hex)

path := "./testdata/with-docker/Earthfile"

tmpDir, err := os.MkdirTemp(os.TempDir(), "with-docker")
r.NoError(err)

tmpFile := filepath.Join(tmpDir, "Earthfile")
defer func() {
err = os.RemoveAll(tmpDir)
r.NoError(err)
}()

err = copyFile(path, tmpFile)
r.NoError(err)

err = replaceInFile(tmpFile, "saved:latest", "other:latest")
r.NoError(err)

target = domain.Target{
LocalPath: tmpDir,
Target: "with-docker-load",
}

_, _, hash, err = HashTarget(ctx, target, cons)
r.NoError(err)

hex = fmt.Sprintf("%x", hash)
r.Equal("84b6f722421695a7ded144c1b72efb3b8f3339c6", hex)
}

func copyFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
out, err := os.Create(dst)
if err != nil {
return err
}
defer out.Close()
_, err = io.Copy(out, in)
if err != nil {
return err
}
return nil
}

func replaceInFile(path, find, replace string) error {
f, err := os.OpenFile(path, os.O_RDWR, 0)
if err != nil {
return err
}
defer f.Close()

dataBytes, err := io.ReadAll(f)
if err != nil {
return err
}

data := string(dataBytes)
data = strings.ReplaceAll(data, find, replace)
_, err = f.Seek(0, 0)
if err != nil {
return err
}

_, err = f.WriteString(data)
if err != nil {
return err
}

return nil
}

func TestHashTargetWithDockerNoAlias(t *testing.T) {
r := require.New(t)
target := domain.Target{
LocalPath: "./testdata/with-docker",
Target: "with-docker-load-no-alias",
}

ctx := context.Background()
cons := conslogging.New(os.Stderr, &sync.Mutex{}, conslogging.NoColor, 0, conslogging.Info)

org, project, hash, err := HashTarget(ctx, target, cons)
r.NoError(err)
r.Equal("earthly-technologies", org)
r.Equal("core", project)

hex := fmt.Sprintf("%x", hash)
r.Equal("d73e37689c7780cbff2cba2de1a23141618b7b14", hex)
}

func TestHashTargetWithDockerArgs(t *testing.T) {
r := require.New(t)
target := domain.Target{
LocalPath: "./testdata/with-docker",
Target: "with-docker-load-args",
}

ctx := context.Background()
cons := conslogging.New(os.Stderr, &sync.Mutex{}, conslogging.NoColor, 0, conslogging.Info)

_, _, _, err := HashTarget(ctx, target, cons)
r.Error(err)
}
29 changes: 29 additions & 0 deletions inputgraph/testdata/with-docker/Earthfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
VERSION 0.7

PROJECT earthly-technologies/core

load-target:
ARG foo=1
FROM alpine
RUN echo "hi" > /tmp/x
SAVE IMAGE saved:latest

with-docker-load:
FROM earthly/dind:alpine
WITH DOCKER --load saved:latest=+load-target
RUN echo "loaded"
END

with-docker-load-no-alias:
BUILD +load-target
FROM earthly/dind:alpine
WITH DOCKER --load +load-target
RUN echo "loaded"
END

with-docker-load-args:
BUILD +load-target
FROM earthly/dind:alpine
WITH DOCKER --load foo=(+load-target --foo=2)
RUN echo "loaded"
END
18 changes: 7 additions & 11 deletions tests/autoskip/Earthfile
Original file line number Diff line number Diff line change
Expand Up @@ -47,40 +47,36 @@ test-auto-skip-requires-project:

test-auto-skip-wait:
DO --pass-args +RUN_EARTHLY_ARGS --earthfile=wait.earth --target=+test --output_contains="not skipped"
DO --pass-args +RUN_EARTHLY_ARGS --earthfile=wait.earth --target=+test --output_contains="8be037baa7b4d09a8e2a37f74154d319d64c996a"
DO --pass-args +RUN_EARTHLY_ARGS --earthfile=wait.earth --target=+test --output_contains="3deeaceb7e263a72114ebc2da62d9fecc87506f3"

test-auto-skip-if-else:
DO --pass-args +RUN_EARTHLY_ARGS --earthfile=if-else.earth --target=+test --output_contains="condition ok"
DO --pass-args +RUN_EARTHLY_ARGS --earthfile=if-else.earth --target=+test --output_contains="40f57fc7955914f3a954c6bd9a2ebe48d0b14f40"
DO --pass-args +RUN_EARTHLY_ARGS --earthfile=if-else.earth --target=+test --output_contains="26ac9a5c6e4893049c218da18decfba3caff1746"

test-auto-skip-for-in:
DO --pass-args +RUN_EARTHLY_ARGS --earthfile=for.earth --target=+test --output_contains="hello 3"
DO --pass-args +RUN_EARTHLY_ARGS --earthfile=for.earth --target=+test --output_contains="3d7d7fec7efc5a746e9ac658160427decbf58a0b"
DO --pass-args +RUN_EARTHLY_ARGS --earthfile=for.earth --target=+test --output_contains="ac33a31a36f496ed219293301bb44513a3a52d28"

test-auto-skip-copy-glob:
DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --should_fail=true --target=+globstar --output_contains="globstar (\*\*) not supported"

COPY --dir glob .

DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+dir --output_contains="glob/subdir/hello.txt"
DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+dir --output_contains="2d1b86207f72caedd143863a859b788b11278a06"
DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+dir --output_contains="7dc88536e40a00684aeeb5ef3d6621d1e867aab2"

DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob --output_contains="glob/subdir/hello.txt"
DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob --output_contains="17446ba4f7a3388ce0c8d57ea44089758c15602a"
DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob --output_contains="9a45ff884f09e576caf91fed3d3a209b109c68e4"

DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob-mid-path --output_contains="glob/subdir/hello.txt"
DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob-mid-path --output_contains="2e539ff29b24e54a3fe2c38534da68071b86c84b"
DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob-mid-path --output_contains="cec86ff354a3064adda819695eb91db965bf0b3e"

DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob-dir --output_contains="glob/subdir/hello.txt"
DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob-dir --output_contains="62b16b26d69ca13260f1f62883df8ff30169f4c4"
DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob-dir --output_contains="111c0b03e1c36e9e55740cade50719e4c6e4e0ec"

RUN echo data > glob/subdir/new.txt
DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob-dir --output_contains="glob/subdir/new.txt"

test-invalid:
COPY glob/*/hello.txt /
RUN --no-cache ls -l /

RUN_EARTHLY_ARGS:
COMMAND
ARG earthfile
Expand Down

0 comments on commit 17ed280

Please sign in to comment.