Auto-skip: Basic support for WAIT (#3358)

This PR adds AutoSkip support for `WAIT...END` blocks.

inputgraph/inputgraph.go

@@ -137,7 +137,7 @@ func (l *loader) handlePipeline(ctx context.Context, cmd spec.Command) error {
 }
 
 func (l *loader) handleCommand(ctx context.Context, cmd spec.Command) error {
-	l.hasher.HashCommand(cmd)
+	l.hashCommand(cmd)
 	switch cmd.Name {
 	case command.From:
 		return l.handleFrom(ctx, cmd)
@@ -164,7 +164,7 @@ func (l *loader) handleWith(ctx context.Context, with spec.WithStatement) error
 }
 
 func (l *loader) handleWithDocker(ctx context.Context, cmd spec.Command) error {
-	l.hasher.HashCommand(cmd) // special case since handleWithDocker doesn't get called from handleCommand
+	l.hashCommand(cmd) // special case since handleWithDocker doesn't get called from handleCommand
 	opts := commandflag.WithDockerOpts{}
 	_, err := parseArgs("WITH DOCKER", &opts, getArgsCopy(cmd))
 	if err != nil {
@@ -194,8 +194,21 @@ func (l *loader) handleFor(ctx context.Context, forStmt spec.ForStatement) error
 	return errors.Wrap(ErrUnableToDetermineHash, "for not supported")
 }
 
+func (l *loader) hashWaitStatement(w spec.WaitStatement) {
+	w.SourceLocation = nil
+	l.hasher.HashString("WAIT")
+	l.hasher.HashInt(len(w.Body))
+	l.hasher.HashJSONMarshalled(w.Args)
+}
+
 func (l *loader) handleWait(ctx context.Context, waitStmt spec.WaitStatement) error {
-	return errors.Wrap(ErrUnableToDetermineHash, "wait not supported")
+	l.hashWaitStatement(waitStmt)
+	for _, stmt := range waitStmt.Body {
+		if err := l.handleStatement(ctx, stmt); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 func (l *loader) handleTry(ctx context.Context, tryStmt spec.TryStatement) error {
@@ -271,6 +284,16 @@ func (l *loader) loadTargetFromString(ctx context.Context, targetName string) er
 	return loaderInst.load(ctx)
 }
 
+func (l *loader) hashVersion(v spec.Version) {
+	v.SourceLocation = nil
+	l.hasher.HashJSONMarshalled(v)
+}
+
+func (l *loader) hashCommand(cmd spec.Command) {
+	cmd.SourceLocation = nil
+	l.hasher.HashJSONMarshalled(cmd)
+}
+
 func (l *loader) findProject(ctx context.Context) (org, project string, err error) {
 	if l.target.IsRemote() {
 		return "", "", ErrRemoteNotSupported
@@ -283,7 +306,7 @@ func (l *loader) findProject(ctx context.Context) (org, project string, err erro
 	ef := bc.Earthfile
 
 	if ef.Version != nil {
-		l.hasher.HashVersion(*ef.Version)
+		l.hashVersion(*ef.Version)
 	}
 
 	for _, stmt := range ef.BaseRecipe {
@@ -314,7 +337,7 @@ func (l *loader) load(ctx context.Context) error {
 	ef := bc.Earthfile
 
 	if ef.Version != nil {
-		l.hasher.HashVersion(*ef.Version)
+		l.hashVersion(*ef.Version)
 	}
 
 	if l.target.Target == "base" {
@@ -352,9 +375,6 @@ func HashTarget(ctx context.Context, target domain.Target, conslog conslogging.C
 	if err != nil {
 		return "", "", nil, err
 	}
-	if !loaderInst.isPipeline {
-		return "", "", nil, errors.Wrap(ErrUnableToDetermineHash, "target is not a pipeline")
-	}
 
 	return org, project, loaderInst.hasher.GetHash(), nil
 }

tests/autoskip/Earthfile

@@ -9,7 +9,8 @@ WORKDIR /test
 test-all:
     BUILD +test-auto-skip
     BUILD +test-auto-skip-with-subdir
-    BUILD +test-auto-skip-requires-pipeline
+    BUILD +test-auto-skip-requires-project
+    BUILD +test-auto-skip-wait
 
 test-auto-skip:
     RUN echo hello > my-file
@@ -36,10 +37,14 @@ test-auto-skip-with-subdir:
     DO --pass-args +RUN_EARTHLY_ARGS --target=+allpipe --output_contains="ba1f2511fc30423bdbb183fe33f3dd0f"
     DO --pass-args +RUN_EARTHLY_ARGS --target=+allpipe --output_does_not_contain="ba1f2511fc30423bdbb183fe33f3dd0f" --output_contains="target .* has already been run; exiting"
 
-test-auto-skip-requires-pipeline:
+test-auto-skip-requires-project:
     RUN echo hello > my-file
-    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=simple.earth --target=+mytarget --output_contains="I was run"
-    RUN if ! grep "target is not a pipeline" earthly.output >/dev/null; then echo "no warning was displayed saying target must be a pipeline" && exit 1; fi
+    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=no-project.earth --target=+no-project --output_contains="I was run"
+    RUN if ! grep "PROJECT command missing" earthly.output >/dev/null; then echo "no warning displayed for missing PROJECT keyword" && exit 1; fi
+
+test-auto-skip-wait:
+    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=wait.earth --target=+test --output_contains="not skipped"
+    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=wait.earth --target=+test --output_contains="ec3d61867365de2deda79ce06f7afa849b765bb1"
 
 RUN_EARTHLY_ARGS:
     COMMAND

tests/autoskip/no-project.earth

@@ -0,0 +1,6 @@
+VERSION 0.7
+
+FROM alpine
+
+no-project:
+    RUN echo "I was run"

tests/autoskip/wait.earth

@@ -0,0 +1,17 @@
+VERSION 0.7
+
+PROJECT earthly-technologies/core
+
+FROM alpine
+
+foo:
+    RUN echo "not skipped"
+
+bar:
+    RUN echo bar > /tmp/x
+
+test:
+    WAIT
+        BUILD +foo
+        BUILD +bar
+    END

util/buildkitskipper/hasher/hasher.go

@@ -9,8 +9,6 @@ import (
 	"hash"
 	"io"
 	"os"
-
-	"github.com/earthly/earthly/ast/spec"
 )
 
 type Hasher struct {
@@ -30,24 +28,20 @@ func (h *Hasher) GetHash() []byte {
 	return h.h.Sum(nil)
 }
 
-func (h *Hasher) HashCommand(cmd spec.Command) {
-	dt, err := json.Marshal(cmd)
-	if err != nil {
-		panic(fmt.Sprintf("failed to hash command: %s", err)) // shouldn't happen
-	}
-	h.HashBytes(dt)
+func (h *Hasher) HashInt(i int) {
+	h.HashBytes([]byte(fmt.Sprintf("int:%d", i)))
 }
 
-func (h *Hasher) HashVersion(version spec.Version) {
-	dt, err := json.Marshal(version)
+func (h *Hasher) HashJSONMarshalled(v any) {
+	dt, err := json.Marshal(v)
 	if err != nil {
-		panic(fmt.Sprintf("failed to hash version: %s", err)) // shouldn't happen
+		panic(fmt.Sprintf("failed to hash command: %s", err)) // shouldn't happen
 	}
 	h.HashBytes(dt)
 }
 
 func (h *Hasher) HashString(s string) {
-	h.HashBytes([]byte(s))
+	h.HashBytes([]byte(fmt.Sprintf("str:%s", s)))
 }
 
 func (h *Hasher) HashBytes(b []byte) {

util/buildkitskipper/hasher/hasher_test.go

@@ -5,7 +5,6 @@ import (
 	"os"
 	"testing"
 
-	"github.com/earthly/earthly/ast/spec"
 	"github.com/earthly/earthly/util/buildkitskipper/hasher"
 )
 
@@ -22,28 +21,6 @@ func TestNilHasherIsNil(t *testing.T) {
 	Nil(t, h.GetHash())
 }
 
-func TestHashCommand(t *testing.T) {
-	h1 := hasher.New()
-	h1.HashCommand(spec.Command{
-		Name: "RUN",
-		Args: []string{"ls", "/foo"},
-	})
-	hash1 := h1.GetHash()
-	NotNil(t, hash1)
-	NotEqual(t, hash1, emptyHash)
-
-	h2 := hasher.New()
-	h2.HashCommand(spec.Command{
-		Name: "RUN",
-		Args: []string{"ls", "/bar"},
-	})
-	hash2 := h2.GetHash()
-	NotNil(t, hash2)
-	NotEqual(t, hash2, emptyHash)
-
-	NotEqual(t, hash1, hash2)
-}
-
 func TestHashEmptyFile(t *testing.T) {
 	file, err := os.CreateTemp("", "file-to-hash")
 	if err != nil {