Support wildcard/glob in COPY (#3995)

Address #3966

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to [Earthly](https://github.com/earthly/earthly) will be doc
 
 ## Unreleased
 
+### Added
+
+- New experimental wildcard-based copy, e.g. `COPY ./services/*+artifact/* .` which would invoke `COPY` for `./services/foo+artifact`, and `./services/bar+artifact` (assuming two services foo and bar, both having a `artifact` target in their respective Earthfile). Enable with the `VERSION --wildcard-copy` feature flag. [#3966](https://github.com/earthly/earthly/issues/3966).
+
 ## v0.8.7 - 2024-04-03
 
 ### Added

docs/earthfile/earthfile.md

@@ -381,6 +381,38 @@ The command may take a couple of possible forms. In the *classical form*, `COPY`
 
 The parameter `<src-artifact>` is an [artifact reference](../guides/importing.md#artifact-reference) and is generally of the form `<target-ref>/<artifact-path>`, where `<target-ref>` is the reference to the target which needs to be built in order to yield the artifact and `<artifact-path>` is the path within the artifact environment of the target, where the file or directory is located. The `<artifact-path>` may also be a wildcard.
 
+{% hint style='info' %}
+##### Globbing
+A target reference in a <src-artifact> may also include a glob expression.
+This is useful in order to invoke multiple targets that may exist in different Earthfiles in the filesystem, in a single `COPY` command.
+For example, consider the following filesystem:
+```bash
+services
+├── Earthfile
+├── service1
+│    └── Earthfile
+├── service2
+│   ├── Earthfile
+├── service3
+│   ├── Earthfile
+```
+
+where a `+mocks` target is defined in services1/Earthfile, services2/Earthfile and services3/Earthfile.
+The command `COPY ./services/*+mocks .` is equivalent to:
+```Earthfile
+    COPY ./services/service1+mocks .
+    COPY ./services/service2+mocks .
+    COPY ./services/service3+mocks .
+```
+
+A glob match occurs when an Earthfile in the glob expression path exists, and the named target is defined in the Earthfile.
+At least one match must be found for the command to succeed.
+
+This feature has experimental status. To use it, it must be enabled via `VERSION --wildcard-copy 0.8`.
+(This is not to be confused with the usage of wildcards in the artifact name, which is fully supported, e.g. `COPY ./services/service1+mocks/* .`)
+
+{% endhint %}
+
 The `COPY` command does not mark any saved images or artifacts of the referenced target for output, nor does it mark any push commands of the referenced target for pushing. For that, please use [`BUILD`](#build).
 
 Multiple `COPY` commands issued one after the other will build the referenced targets in parallel, if the targets don't depend on each other. The resulting artifacts will then be copied sequentially in the order in which the `COPY` commands were issued.
@@ -806,6 +838,37 @@ In Earthly v0.6+, what is being output and pushed is determined either by the ma
 
 If you are referencing a target via some other command, such as `COPY` and you would like for the outputs or pushes to be included, you can issue an equivalent `BUILD` command in addition to the `COPY`. For example
 
+{% hint style='info' %}
+##### Globbing
+A <target-ref> may also include a glob expression.
+This is useful in order to invoke multiple targets that may exist in different Earthfiles in the filesystem, in a single `BUILD` command.
+For example, consider the following filesystem:
+```bash
+services
+├── Earthfile
+├── service1
+│    └── Earthfile
+├── service2
+│   ├── Earthfile
+├── service3
+│   ├── Earthfile
+```
+
+where a `+compile` target is defined in services1/Earthfile, services2/Earthfile and services3/Earthfile.
+The command `BUILD ./services/*+compile .` is equivalent to:
+```Earthfile
+    BUILD ./services/service1+compile
+    BUILD ./services/service2+compile
+    BUILD ./services/service3+compile
+```
+
+A glob match occurs when an Earthfile in the glob expression path exists, and the named target is defined in the Earthfile.
+At least one match must be found for the command to succeed.
+
+This feature has experimental status. To use it, it must be enabled via `VERSION --wildcard-builds 0.8`.
+
+{% endhint %}
+
 ```Dockerfile
 my-target:
     COPY --platform=linux/amd64 (+some-target/some-file.txt --FOO=bar) ./
@@ -1100,7 +1163,7 @@ The `WITH DOCKER` clause only supports the command [`RUN`](#run). Other commands
 A typical example of a `WITH DOCKER` clause might be:
 
 ```Dockerfile
-FROM earthly/dind:alpine-3.19-docker-25.0.2-r0
+FROM earthly/dind:alpine-3.19-docker-25.0.3-r2
 WORKDIR /test
 COPY docker-compose.yml ./
 WITH DOCKER \
@@ -1122,7 +1185,7 @@ For information on using `WITH DOCKER` with podman see the [Podman guide](../gui
 ##### Note
 For performance reasons, it is recommended to use a Docker image that already contains `dockerd`. If `dockerd` is not found, Earthly will attempt to install it.
 
-Earthly provides officially supported images such as `earthly/dind:alpine-3.19-docker-25.0.2-r0` and `earthly/dind:ubuntu-23.04-docker-24.0.5-1` to be used together with `WITH DOCKER`.
+Earthly provides officially supported images such as `earthly/dind:alpine-3.19-docker-25.0.3-r2` and `earthly/dind:ubuntu-23.04-docker-24.0.5-1` to be used together with `WITH DOCKER`.
 {% endhint %}
 
 {% hint style='info' %}

docs/earthfile/features.md

@@ -87,9 +87,10 @@ VERSION [<flags>...] <version-number>
 | `--try`                                 | Experimental                                                                    | Enable the `TRY` / `FINALLY` / `END` block commands                                                                |
 | `--earthly-ci-runner-arg`               | Experimental                                                                    | Enable the `EARTHLY_CI_RUNNER` builtin ARG                                                                         |
 | `--wildcard-builds`                     | Experimental                                                                    | Alow for the expansion of wildcard (glob) paths for BUILD commands                                                 |
-| `--build-auto-skip`                     | Experimental                                                                    | Allow for `--auto-skip` to be used on individual BUILD commands                                                      |
-| `--allow-privileged-from-dockerfile`    | Experimental                                                                    | Allow the use of the `--allow-privileged` flag in the `FROM DOCKERFILE` command |
-| `--run-with-aws`                        | Experimental                                                                    | Make AWS credentials in the environment or ~/.aws available to `RUN` commands     |
+| `--build-auto-skip`                     | Experimental                                                                    | Allow for `--auto-skip` to be used on individual BUILD commands                                                    |
+| `--allow-privileged-from-dockerfile`    | Experimental                                                                    | Allow the use of the `--allow-privileged` flag in the `FROM DOCKERFILE` command                                    |
+| `--run-with-aws`                        | Experimental                                                                    | Make AWS credentials in the environment or ~/.aws available to `RUN` commands                                      |
+| `--wildcard-copy`                       | Experimental                                                                    | Alow for the expansion of wildcard (glob) paths for COPY commands                                                  |
 
 Note that the features flags are disabled by default in Earthly versions lower than the version listed in the "status" column above.
 

docs/guides/importing.md

@@ -51,7 +51,7 @@ my-target:
 
 In this example, the target `my-target` uses the import alias `hello-world` to reference a GitHub repository called `github.com/earthly/hello-world`, and the target `hello` within that repository. The `AS hello-world` part is optional, and is only needed if the import alias is different from the repository name.
 
-`BUILD` is used used to simply issue the build of the referenced target. Commands like `COPY` or `FROM` can be used to import artifacts or images, respectively.
+`BUILD` is used to simply issue the build of the referenced target. Commands like `COPY` or `FROM` can be used to import artifacts or images, respectively.
 
 ### Importing from other directories
 
@@ -96,7 +96,7 @@ Here are some examples:
 
 * `+build`
 * `./js+deps`
-* `github.com/earthly/earthly:v0.8.6+earthly`
+* `github.com/earthly/earthly:v0.8.7+earthly`
 * `my-import+build`
 
 ## Artifact reference
@@ -110,7 +110,7 @@ Here are some examples:
 * `+build/my-artifact`
 * `+build/some/artifact/deep/in/a/dir`
 * `./js+build/dist`
-* `github.com/earthly/earthly:v0.8.6+earthly/earthly`
+* `github.com/earthly/earthly:v0.8.7+earthly/earthly`
 * `my-import+build/my-artifact`
 
 ## Image reference
@@ -131,7 +131,7 @@ Here are some examples:
 
 * `+COMPILE`
 * `./js+NPM_INSTALL`
-* `github.com/earthly/earthly:v0.8.6+DOWNLOAD_DIND`
+* `github.com/earthly/earthly:v0.8.7+DOWNLOAD_DIND`
 * `my-import+COMPILE`
 
 For more information on functions, see the [Functions Guide](./functions.md).
@@ -162,6 +162,12 @@ Another form, is where a target, function or artifact is referenced from a diffe
 
 It is recommended that relative paths are used, for portability reasons: the working directory checked out by different users will be different, making absolute paths infeasible in most cases.
 
+{% hint style='info' %}
+##### Note
+When using a `Target ref` in a `BUILD` command or an `Artifact ref` in a `COPY` command, the ref to the target
+may also include a glob expression (e.g. `./parent/*+<target-name>`, `./parent/*+<target-name>/<artifact-path>`). Globbing in a target/artifact ref has experimental status. To use this feature, it must be enabled via `VERSION --wildcard-builds 0.8` (for `BUILD`) or `VERSION --wildcard-copy 0.8` (for `COPY`).
+{% endhint %}
+
 ### Remote
 
 Another form of a Earthfile reference is the remote form. In this form, the recipe and the build context are imported from a remote location. It has the following form:
@@ -170,7 +176,7 @@ Another form of a Earthfile reference is the remote form. In this form, the reci
 |----|----|----|----|
 | `<vendor>/<namespace>/<project>/path/in/project[:some-tag]` | `<vendor>/<namespace>/<project>/path/in/project[:some-tag]+<target-name>` | `<vendor>/<namespace>/<project>/path/in/project[:some-tag]+<target-name>/<artifact-path>` | `<vendor>/<namespace>/<project>/path/in/project[:some-tag]+<function-name>` |
 | `github.com/earthly/earthly/buildkitd` | `github.com/earthly/earthly/buildkitd+build` | `github.com/earthly/earthly/buildkitd+build/out.bin` | `github.com/earthly/earthly/buildkitd+COMPILE` |
-| `github.com/earthly/earthly:v0.8.6` | `github.com/earthly/earthly:v0.8.6+build` | `github.com/earthly/earthly:v0.8.6+build/out.bin` | `github.com/earthly/earthly:v0.8.6+COMPILE` |
+| `github.com/earthly/earthly:v0.8.7` | `github.com/earthly/earthly:v0.8.7+build` | `github.com/earthly/earthly:v0.8.7+build/out.bin` | `github.com/earthly/earthly:v0.8.7+COMPILE` |
 
 ### Import reference
 
@@ -180,7 +186,7 @@ Finally, the last form of Earthfile referencing is an import reference. Import r
 |----|----|----|----|----|
 | `IMPORT <full-earthfile-ref> AS <import-alias>` | `<import-alias>` | `<import-alias>+<target-name>` | `<import-alias>+<target-name>/<artifact-path>` | `<import-alias>+<function-name>` |
 | `IMPORT github.com/earthly/earthly/buildkitd` | `buildkitd` | `buildkitd+build` | `buildkitd+build/out.bin` | `buildkitd+COMPILE` |
-| `IMPORT github.com/earthly/earthly:v0.8.6` | `earthly` | `earthly+build` | `earthly+build/out.bin` | `earthly+COMPILE` |
+| `IMPORT github.com/earthly/earthly:v0.8.7` | `earthly` | `earthly+build` | `earthly+build/out.bin` | `earthly+COMPILE` |
 
 Here is an example in an Earthfile:
 

domain/artifact.go

@@ -14,6 +14,11 @@ type Artifact struct {
 	Artifact string
 }
 
+// Clone returns a copy of the Artifact
+func (a Artifact) Clone() Artifact {
+	return a
+}
+
 // String returns a string representation of the Artifact.
 func (ea Artifact) String() string {
 	return fmt.Sprintf("%s%s", ea.Target.String(), path.Join("/", escapePlus(ea.Artifact)))

earthfile2llb/cloneable.go

@@ -0,0 +1,5 @@
+package earthfile2llb
+
+type cloneable[T any] interface {
+	Clone() T
+}

earthfile2llb/converter.go

@@ -1705,61 +1705,31 @@ func (c *Converter) Pipeline(ctx context.Context) error {
 	return nil
 }
 
-func (c *Converter) ExpandWildcard(ctx context.Context, fullTargetName string, cmd spec.Command) ([]spec.Command, error) {
-	parsedTarget, err := domain.ParseTarget(fullTargetName)
+// ExpandWildcardCmds expands a glob expression in the specified fullTargetName and returns copies(clones) of the specified cmd for each match of the expression
+func (c *Converter) ExpandWildcardCmds(ctx context.Context, fullTargetName string, cmd spec.Command) ([]spec.Command, error) {
+	targets, err := c.expandWildcardTargets(ctx, fullTargetName)
 	if err != nil {
 		return nil, err
 	}
 
-	if strings.Contains(fullTargetName, "**") {
-		return nil, errors.New("globstar (**) pattern not yet supported")
-	}
+	return clonesWithExpandedTargets(targets, cmd, func(cmd *spec.Command, expandedTarget string) error {
+		for i := range cmd.Args {
+			cmd.Args[i] = strings.ReplaceAll(cmd.Args[i], fullTargetName, expandedTarget)
+		}
+		return nil
+	})
+}
 
-	matches, err := c.opt.Resolver.ExpandWildcard(ctx, c.opt.GwClient, c.platr, c.target, parsedTarget)
+// ExpandWildcardArtifacts expands a glob expression in the specified artifact's target and returns copies(clones) of the artifact for each match of the expression
+func (c *Converter) ExpandWildcardArtifacts(ctx context.Context, artifact domain.Artifact) ([]domain.Artifact, error) {
+	targets, err := c.expandWildcardTargets(ctx, artifact.Target.String())
 	if err != nil {
 		return nil, err
 	}
-
-	children := []spec.Command{}
-	for _, match := range matches {
-		childTargetName := fmt.Sprintf("./%s+%s", match, parsedTarget.GetName())
-
-		childTarget, err := domain.ParseTarget(childTargetName)
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed to parse target %q", childTargetName)
-		}
-
-		data, _, _, err := c.ResolveReference(ctx, childTarget)
-		if err != nil {
-			notExist := buildcontext.ErrEarthfileNotExist{}
-			if errors.As(err, &notExist) {
-				continue
-			}
-			return nil, errors.Wrapf(err, "unable to resolve target %q", childTargetName)
-		}
-
-		var found bool
-		for _, target := range data.Earthfile.Targets {
-			if target.Name == childTarget.GetName() {
-				found = true
-				break
-			}
-		}
-
-		if !found {
-			continue
-		}
-
-		cloned := cmd.Clone()
-		cloned.Args[len(cloned.Args)-1] = childTargetName
-		children = append(children, cloned)
-	}
-
-	if len(children) == 0 {
-		return nil, errors.Errorf("no matching targets found for pattern %q", parsedTarget.GetLocalPath())
-	}
-
-	return children, nil
+	return clonesWithExpandedTargets(targets, artifact, func(artifact *domain.Artifact, expandedTarget string) error {
+		artifact.Target, err = domain.ParseTarget(expandedTarget)
+		return err
+	})
 }
 
 // ResolveReference resolves a reference's build context given the current state: relativity to the Earthfile, imports etc.
@@ -2906,6 +2876,74 @@ func (c *Converter) newCmdID() int {
 	return cmdID
 }
 
+func (c *Converter) expandWildcardTargets(ctx context.Context, fullTargetName string) ([]string, error) {
+	parsedTarget, err := domain.ParseTarget(fullTargetName)
+	if err != nil {
+		return nil, err
+	}
+
+	if strings.Contains(fullTargetName, "**") {
+		return nil, errors.New("globstar (**) pattern not yet supported")
+	}
+
+	matches, err := c.opt.Resolver.ExpandWildcard(ctx, c.opt.GwClient, c.platr, c.target, parsedTarget)
+	if err != nil {
+		return nil, err
+	}
+
+	targets := make([]string, 0, len(matches))
+	for _, match := range matches {
+		childTargetName := fmt.Sprintf("./%s+%s", match, parsedTarget.GetName())
+
+		childTarget, err := domain.ParseTarget(childTargetName)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to parse target %q", childTargetName)
+		}
+
+		data, _, _, err := c.ResolveReference(ctx, childTarget)
+		if err != nil {
+			notExist := buildcontext.ErrEarthfileNotExist{}
+			if errors.As(err, &notExist) {
+				continue
+			}
+			return nil, errors.Wrapf(err, "unable to resolve target %q", childTargetName)
+		}
+
+		var found bool
+		for _, target := range data.Earthfile.Targets {
+			if target.Name == childTarget.GetName() {
+				found = true
+				break
+			}
+		}
+
+		if !found {
+			continue
+		}
+
+		targets = append(targets, childTargetName)
+	}
+
+	if len(targets) == 0 {
+		return nil, errors.Errorf("no matching targets found for pattern %q", parsedTarget.GetLocalPath())
+	}
+
+	return targets, nil
+}
+
+func clonesWithExpandedTargets[T cloneable[T]](expandedTargets []string, c T, setTarget func(t *T, expandedTarget string) error) ([]T, error) {
+	clones := make([]T, 0, len(expandedTargets))
+	for _, expandedTarget := range expandedTargets {
+		cloned := c.Clone()
+		err := setTarget(&cloned, expandedTarget)
+		if err != nil {
+			return nil, err
+		}
+		clones = append(clones, cloned)
+	}
+	return clones, nil
+}
+
 func joinWrap(a []string, before string, sep string, after string) string {
 	if len(a) > 0 {
 		return fmt.Sprintf("%s%s%s", before, strings.Join(a, sep), after)