Auto-skip: support for some remote targets (#3508)

Adds `BUILD`, `COPY`, `FROM` support for remote targets that use explicit SHAs or tags. --------- Co-authored-by: Alex Couture-Beil <[email protected]>
earthly · Nov 20, 2023 · 6c40169 · 6c40169
1 parent 7f43f54
commit 6c40169
Show file tree

Hide file tree

Showing 4 changed files with 95 additions and 9 deletions.
diff --git a/inputgraph/loader.go b/inputgraph/loader.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"regexp"
 	"sort"
 	"strings"
 
@@ -22,7 +23,10 @@ import (
 	"github.com/pkg/errors"
 )
 
-var ErrRemoteNotSupported = fmt.Errorf("remote targets not supported")
+var (
+	errUnsupportedRemoteTarget = errors.New("only remote targets referenced by a complete Git SHA or tag (e.g. tags/my-tag) are supported")
+	errCannotLoadRemoteTarget  = errors.New("cannot load remote target")
+)
 
 type loader struct {
 	conslog         conslogging.ConsoleLogger
@@ -65,6 +69,15 @@ func (l *loader) handleFrom(ctx context.Context, cmd spec.Command) error {
 		return nil
 	}
 
+	target, err := domain.ParseTarget(fromTarget)
+	if err == nil && target.IsRemote() {
+		if supportedRemoteTarget(target) {
+			l.hasher.HashString(target.StringCanonical())
+			return nil
+		}
+		return errUnsupportedRemoteTarget
+	}
+
 	return l.loadTargetFromString(ctx, fromTarget, args[1:], false)
 }
 
@@ -79,13 +92,24 @@ func (l *loader) handleBuild(ctx context.Context, cmd spec.Command) error {
 		return errors.New("missing BUILD arg")
 	}
 
+	targetName := args[0]
+
 	argCombos, err := flagutil.BuildArgMatrix(args)
 	if err != nil {
 		return errors.Wrap(err, "failed to compute arg matrix")
 	}
 
+	target, err := domain.ParseTarget(targetName)
+	if err == nil && target.IsRemote() {
+		if supportedRemoteTarget(target) {
+			l.hasher.HashString(target.StringCanonical())
+			return nil
+		}
+		return errUnsupportedRemoteTarget
+	}
+
 	for _, args := range argCombos {
-		err := l.loadTargetFromString(ctx, args[0], args[1:], opts.PassArgs)
+		err := l.loadTargetFromString(ctx, targetName, args[1:], opts.PassArgs)
 		if err != nil {
 			return err
 		}
@@ -120,13 +144,13 @@ func (l *loader) handleCopy(ctx context.Context, cmd spec.Command) error {
 	return nil
 }
 
-func hasShellExpr(s string) bool {
+func containsShellExpr(s string) bool {
 	return strings.Contains(s, "$(") && strings.Contains(s, ")")
 }
 
 func (l *loader) handleCopySrc(ctx context.Context, src string, mustExist bool) error {
 
-	if hasShellExpr(src) {
+	if containsShellExpr(src) {
 		return errors.Errorf("dynamic COPY source %q cannot be resolved", src)
 	}
 
@@ -177,7 +201,11 @@ func (l *loader) handleCopySrc(ctx context.Context, src string, mustExist bool)
 
 	// Remote targets aren't supported.
 	if artifactSrc.Target.IsRemote() {
-		return errors.New("unable to handle remote target")
+		if supportedRemoteTarget(artifactSrc.Target) {
+			l.hasher.HashString(artifactSrc.Target.StringCanonical())
+			return nil
+		}
+		return errUnsupportedRemoteTarget
 	}
 
 	targetName := artifactSrc.Target.LocalPath + "+" + artifactSrc.Target.Target
@@ -188,6 +216,12 @@ func (l *loader) handleCopySrc(ctx context.Context, src string, mustExist bool)
 	return nil
 }
 
+var sha1RE = regexp.MustCompile("^[0-9a-f]{40}$")
+
+func supportedRemoteTarget(t domain.Target) bool {
+	return strings.HasPrefix(t.GetTag(), "tags/") || sha1RE.MatchString(t.GetTag())
+}
+
 // expandCopyFiles expands a single COPY source into a slice containing all
 // nested files. The file names will then be used in our hash.
 func (l *loader) expandCopyFiles(src string, mustExist bool) ([]string, error) {
@@ -518,7 +552,7 @@ func (l *loader) forTarget(ctx context.Context, target domain.Target, args []str
 func (l *loader) loadTargetFromString(ctx context.Context, targetName string, args []string, passArgs bool) error {
 	// If the target name contains a variable that hasn't been expanded, we
 	// won't be able to explore the rest of the graph and generate a valid hash.
-	if hasShellExpr(targetName) {
+	if containsShellExpr(targetName) {
 		return errors.Errorf("dynamic target %q cannot be resolved", targetName)
 	}
 
@@ -554,7 +588,7 @@ func (l *loader) loadTargetFromString(ctx context.Context, targetName string, ar
 
 func (l *loader) findProject(ctx context.Context) (org, project string, err error) {
 	if l.target.IsRemote() {
-		return "", "", ErrRemoteNotSupported
+		return "", "", errCannotLoadRemoteTarget
 	}
 
 	resolver := buildcontext.NewResolver(nil, nil, l.conslog, "", "", "", 0, "")
@@ -588,7 +622,7 @@ func (l *loader) findProject(ctx context.Context) (org, project string, err erro
 
 func (l *loader) load(ctx context.Context) error {
 	if l.target.IsRemote() {
-		return ErrRemoteNotSupported
+		return errCannotLoadRemoteTarget
 	}
 
 	resolver := buildcontext.NewResolver(nil, nil, l.conslog, "", "", "", 0, "")

diff --git a/inputgraph/loader_hashing.go b/inputgraph/loader_hashing.go
@@ -1,6 +1,8 @@
 package inputgraph
 
-import "github.com/earthly/earthly/ast/spec"
+import (
+	"github.com/earthly/earthly/ast/spec"
+)
 
 func (l *loader) hashIfStatement(s spec.IfStatement) {
 	l.hasher.HashString("IF")

diff --git a/tests/autoskip/Earthfile b/tests/autoskip/Earthfile
@@ -24,6 +24,7 @@ test-all:
     BUILD +test-no-cache
     BUILD +test-shell-out
     BUILD +test-copy-if-exists
+    BUILD +test-remote-targets
 
 test-files:
     RUN echo hello > my-file
@@ -172,6 +173,26 @@ test-copy-if-exists:
 
     DO --pass-args +RUN_EARTHLY_ARGS --earthfile=simple.earth --target=+copy-if-exists --output_does_not_contain="target .* has already been run; exiting"
 
+test-remote-targets:
+    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=remote-target.earth --target=+valid-copy
+    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=remote-target.earth --target=+valid-copy --output_contains="target .* has already been run; exiting"
+
+    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=remote-target.earth --target=+valid-copy-sha
+    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=remote-target.earth --target=+valid-copy-sha --output_contains="target .* has already been run; exiting"
+
+    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=remote-target.earth --target=+valid-from
+    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=remote-target.earth --target=+valid-from --output_contains="target .* has already been run; exiting"
+
+    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=remote-target.earth --target=+valid-from-sha
+    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=remote-target.earth --target=+valid-from-sha --output_contains="target .* has already been run; exiting"
+
+    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=remote-target.earth --target=+invalid-copy-branch --output_contains="complete Git SHA or tag"
+
+    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=remote-target.earth --target=+valid-build
+    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=remote-target.earth --target=+valid-build --output_contains="target .* has already been run; exiting"
+
+    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=remote-target.earth --target=+invalid-build --output_contains="complete Git SHA or tag"
+
 RUN_EARTHLY_ARGS:
     COMMAND
     ARG earthfile

diff --git a/tests/autoskip/remote-target.earth b/tests/autoskip/remote-target.earth
@@ -0,0 +1,29 @@
+VERSION 0.7
+PROJECT testorg/testproj
+
+valid-copy:
+    FROM alpine
+    COPY github.com/earthly/earthly:tags/v0.7.21+license/LICENSE .
+
+valid-copy-sha:
+    FROM alpine
+    COPY github.com/earthly/earthly:6610b73131f94cbe594dba3b90665748f21a9b8d+license/LICENSE .
+
+valid-from:
+    FROM github.com/earthly/earthly:tags/v0.7.21+license
+    RUN echo "hello"
+
+valid-from-sha:
+    FROM github.com/earthly/earthly:6610b73131f94cbe594dba3b90665748f21a9b8d+license
+    RUN echo "hello"
+
+invalid-copy-branch:
+    FROM alpine
+    COPY github.com/earthly/earthly:main+license/LICENSE .
+
+valid-build:
+    BUILD github.com/earthly/earthly:6610b73131f94cbe594dba3b90665748f21a9b8d+license
+    BUILD github.com/earthly/earthly:tags/v0.7.21+license
+
+invalid-build:
+    BUILD github.com/earthly/earthly:main+license