Auto-skip: add support for arg matrices (#3444)

Adds support for the expansion of `BUILD` arguments when multiple values
are specified for a given argument.

Addresses: earthly/earthly#3045

earthfile2llb/interpreter.go

@@ -1200,7 +1200,7 @@ func (i *Interpreter) handleBuild(ctx context.Context, cmd spec.Command, async b
 		platformsSlice = []platutil.Platform{platutil.DefaultPlatform}
 	}
 
-	crossProductBuildArgs, err := buildArgMatrix(expandedBuildArgs)
+	crossProductBuildArgs, err := flagutil.BuildArgMatrix(expandedBuildArgs)
 	if err != nil {
 		return i.wrapError(err, cmd.SourceLocation, "build arg matrix")
 	}
@@ -2100,72 +2100,6 @@ func ParseLoad(loadStr string) (image string, target string, extraArgs []string,
 	return image, target, extraArgs, nil
 }
 
-type argGroup struct {
-	key    string
-	values []*string
-}
-
-func buildArgMatrix(args []string) ([][]string, error) {
-	groupedArgs := make([]argGroup, 0, len(args))
-	for _, arg := range args {
-		k, v, err := parseKeyValue(arg)
-		if err != nil {
-			return nil, err
-		}
-
-		found := false
-		for i, g := range groupedArgs {
-			if g.key == k {
-				groupedArgs[i].values = append(groupedArgs[i].values, v)
-				found = true
-				break
-			}
-		}
-		if !found {
-			groupedArgs = append(groupedArgs, argGroup{
-				key:    k,
-				values: []*string{v},
-			})
-		}
-	}
-	return crossProduct(groupedArgs, nil), nil
-}
-
-func crossProduct(ga []argGroup, prefix []string) [][]string {
-	if len(ga) == 0 {
-		return [][]string{prefix}
-	}
-	var ret [][]string
-	for _, v := range ga[0].values {
-		newPrefix := prefix[:]
-		var kv string
-		if v == nil {
-			kv = ga[0].key
-		} else {
-			kv = fmt.Sprintf("%s=%s", ga[0].key, *v)
-		}
-		newPrefix = append(newPrefix, kv)
-
-		cp := crossProduct(ga[1:], newPrefix)
-		ret = append(ret, cp...)
-	}
-	return ret
-}
-
-func parseKeyValue(arg string) (string, *string, error) {
-	var name string
-	splitArg := strings.SplitN(arg, "=", 2)
-	if len(splitArg) < 1 {
-		return "", nil, errors.Errorf("invalid build arg %s", splitArg)
-	}
-	name = splitArg[0]
-	var value *string
-	if len(splitArg) == 2 {
-		value = &splitArg[1]
-	}
-	return name, value, nil
-}
-
 // requiresShellOutOrCmdInvalid returns true if
 // cmd requires shelling out via $(...), or if the cmd is invalid.
 // This function is best-effort, and returns false on errors, errors

earthfile2llb/interpreter_test.go

@@ -1,30 +1 @@
 package earthfile2llb
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestBuildArgMatrix(t *testing.T) {
-	var tests = []struct {
-		in  []string
-		out [][]string
-	}{
-		{[]string{}, [][]string{nil}},
-		{[]string{"a=1"}, [][]string{{"a=1"}}},
-		{[]string{"a=1", "a=2", "a=3"}, [][]string{{"a=1"}, {"a=2"}, {"a=3"}}},
-		{[]string{"a=1", "b=2"}, [][]string{{"a=1", "b=2"}}},
-		{[]string{"a=1", "a=3", "b=2"}, [][]string{{"a=1", "b=2"}, {"a=3", "b=2"}}},
-		{[]string{"a=1", "a=3", "b=2", "b=4"}, [][]string{{"a=1", "b=2"}, {"a=1", "b=4"}, {"a=3", "b=2"}, {"a=3", "b=4"}}},
-		{[]string{"a=1", "b=2", "a=3", "b=4"}, [][]string{{"a=1", "b=2"}, {"a=1", "b=4"}, {"a=3", "b=2"}, {"a=3", "b=4"}}},
-		{[]string{"a=1", "b=2", "a=3", "b=4", "c=10"}, [][]string{{"a=1", "b=2", "c=10"}, {"a=1", "b=4", "c=10"}, {"a=3", "b=2", "c=10"}, {"a=3", "b=4", "c=10"}}},
-		{[]string{"a=1", "a=3", "a=7", "c=10"}, [][]string{{"a=1", "c=10"}, {"a=3", "c=10"}, {"a=7", "c=10"}}},
-	}
-
-	for _, tt := range tests {
-		ans, err := buildArgMatrix(tt.in)
-		assert.NoError(t, err)
-		assert.Equal(t, tt.out, ans)
-	}
-}

inputgraph/loader.go

@@ -84,11 +84,19 @@ func (l *loader) handleBuild(ctx context.Context, cmd spec.Command) error {
 		return errors.New("missing BUILD arg")
 	}
 
-	if requiresCrossProduct(args) {
-		return errors.New("unable to cross-product in BUILD")
+	argCombos, err := flagutil.BuildArgMatrix(args)
+	if err != nil {
+		return errors.Wrap(err, "failed to compute arg matrix")
 	}
 
-	return l.loadTargetFromString(ctx, args[0], args[1:], opts.PassArgs)
+	for _, args := range argCombos {
+		err := l.loadTargetFromString(ctx, args[0], args[1:], opts.PassArgs)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
 }
 
 func (l *loader) handleCopy(ctx context.Context, cmd spec.Command) error {

inputgraph/util.go

@@ -1,21 +1,5 @@
 package inputgraph
 
-import (
-	"strings"
-)
-
-func requiresCrossProduct(args []string) bool {
-	seen := map[string]struct{}{}
-	for _, s := range args {
-		k := strings.SplitN(s, "=", 2)[0]
-		if _, found := seen[k]; found {
-			return true
-		}
-		seen[k] = struct{}{}
-	}
-	return false
-}
-
 func copyVisited(m map[string]struct{}) map[string]struct{} {
 	m2 := map[string]struct{}{}
 	for k := range m {

tests/autoskip/Earthfile

@@ -18,6 +18,7 @@ test-all:
     BUILD +test-build-args
     BUILD +test-pass-args
     BUILD +test-copy-target-args
+    BUILD +test-arg-matrix
 
 test-files:
     RUN echo hello > my-file
@@ -114,6 +115,14 @@ test-copy-target-args:
     DO --pass-args +RUN_EARTHLY_ARGS --target=+copy-target-args --output_contains="+copy-target-args | changed"
     DO --pass-args +RUN_EARTHLY_ARGS --target=+copy-target-args --output_contains="target .* has already been run; exiting"
 
+test-arg-matrix:
+    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=matrix.earth --target=+arg-matrix --output_contains="Hello Bob. From Todd"
+    DO --pass-args +RUN_EARTHLY_ARGS --earthfile=matrix.earth --target=+arg-matrix --output_contains="target .* has already been run; exiting"
+
+    RUN sed -i s/Bill/Sam/g Earthfile
+
+    DO --pass-args +RUN_EARTHLY_ARGS --target=+arg-matrix --output_contains="Hello Sam. From Todd"
+    DO --pass-args +RUN_EARTHLY_ARGS --target=+arg-matrix --output_contains="target .* has already been run; exiting"
 
 RUN_EARTHLY_ARGS:
     COMMAND

tests/autoskip/matrix.earth

@@ -0,0 +1,13 @@
+VERSION 0.7
+
+PROJECT earthly-technologies/core
+
+FROM alpine
+
+foo:
+    ARG NAME
+    ARG SENDER
+    RUN echo "Hello $NAME. From $SENDER"
+
+arg-matrix:
+    BUILD +foo --NAME=Bob --NAME=John --NAME=Bill --NAME=Sarah --SENDER=Todd --SENDER=Owen

util/flagutil/matrix.go

@@ -0,0 +1,76 @@
+package flagutil
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+type argGroup struct {
+	key    string
+	values []*string
+}
+
+// BuildArgMatrix builds a 2-dimensional slice of arguments that contains all
+// combinations
+func BuildArgMatrix(args []string) ([][]string, error) {
+	groupedArgs := make([]argGroup, 0, len(args))
+	for _, arg := range args {
+		k, v, err := parseKeyValue(arg)
+		if err != nil {
+			return nil, err
+		}
+
+		found := false
+		for i, g := range groupedArgs {
+			if g.key == k {
+				groupedArgs[i].values = append(groupedArgs[i].values, v)
+				found = true
+				break
+			}
+		}
+		if !found {
+			groupedArgs = append(groupedArgs, argGroup{
+				key:    k,
+				values: []*string{v},
+			})
+		}
+	}
+	return crossProduct(groupedArgs, nil), nil
+}
+
+func crossProduct(ga []argGroup, prefix []string) [][]string {
+	if len(ga) == 0 {
+		return [][]string{prefix}
+	}
+	var ret [][]string
+	for _, v := range ga[0].values {
+		newPrefix := prefix[:]
+		var kv string
+		if v == nil {
+			kv = ga[0].key
+		} else {
+			kv = fmt.Sprintf("%s=%s", ga[0].key, *v)
+		}
+		newPrefix = append(newPrefix, kv)
+
+		cp := crossProduct(ga[1:], newPrefix)
+		ret = append(ret, cp...)
+	}
+	return ret
+}
+
+func parseKeyValue(arg string) (string, *string, error) {
+	var name string
+	splitArg := strings.SplitN(arg, "=", 2)
+	if len(splitArg) < 1 {
+		return "", nil, errors.Errorf("invalid build arg %s", splitArg)
+	}
+	name = splitArg[0]
+	var value *string
+	if len(splitArg) == 2 {
+		value = &splitArg[1]
+	}
+	return name, value, nil
+}

util/flagutil/matrix_test.go

@@ -0,0 +1,30 @@
+package flagutil
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBuildArgMatrix(t *testing.T) {
+	var tests = []struct {
+		in  []string
+		out [][]string
+	}{
+		{[]string{}, [][]string{nil}},
+		{[]string{"a=1"}, [][]string{{"a=1"}}},
+		{[]string{"a=1", "a=2", "a=3"}, [][]string{{"a=1"}, {"a=2"}, {"a=3"}}},
+		{[]string{"a=1", "b=2"}, [][]string{{"a=1", "b=2"}}},
+		{[]string{"a=1", "a=3", "b=2"}, [][]string{{"a=1", "b=2"}, {"a=3", "b=2"}}},
+		{[]string{"a=1", "a=3", "b=2", "b=4"}, [][]string{{"a=1", "b=2"}, {"a=1", "b=4"}, {"a=3", "b=2"}, {"a=3", "b=4"}}},
+		{[]string{"a=1", "b=2", "a=3", "b=4"}, [][]string{{"a=1", "b=2"}, {"a=1", "b=4"}, {"a=3", "b=2"}, {"a=3", "b=4"}}},
+		{[]string{"a=1", "b=2", "a=3", "b=4", "c=10"}, [][]string{{"a=1", "b=2", "c=10"}, {"a=1", "b=4", "c=10"}, {"a=3", "b=2", "c=10"}, {"a=3", "b=4", "c=10"}}},
+		{[]string{"a=1", "a=3", "a=7", "c=10"}, [][]string{{"a=1", "c=10"}, {"a=3", "c=10"}, {"a=7", "c=10"}}},
+	}
+
+	for _, tt := range tests {
+		ans, err := BuildArgMatrix(tt.in)
+		assert.NoError(t, err)
+		assert.Equal(t, tt.out, ans)
+	}
+}