Do not persist CACHE mount in child targets by default (#3510)

Persisting a `CACHE` in a child target can lead to bloated image sizes, the contents of a `CACHE` will no longer be copied by default; this can be overridden with the `CACHE --persist` option. Fixes #3509

ast/commandflag/flags.go

@@ -144,6 +144,7 @@ type CacheOpts struct {
 	Sharing string `long:"sharing" description:"The cache sharing mode: locked (default), shared, private"`
 	Mode    string `long:"chmod" description:"Apply a mode to the cache folder" default:"0644"`
 	ID      string `long:"id" description:"Cache ID, to reuse the same cache across different targets and Earthfiles"`
+	Persist bool   `long:"persist" description:"If should persist cache state in image"`
 }
 
 // NewForOpts creates and returns a ForOpts with default separators.

docs/earthfile/features.md

@@ -33,38 +33,39 @@ to require version `0.X` (or later), and could be rewritten as `VERSION 0.X`.
 
 ## Feature flags
 
-| Feature flag                        | status       | description                                                                                                   |
-|-------------------------------------|--------------|---------------------------------------------------------------------------------------------------------------|
-| `--use-registry-for-with-docker`    | 0.5          | Makes use of the embedded BuildKit Docker registry (instead of tar files) for `WITH DOCKER` loads and pulls   |
-| `--use-copy-include-patterns`       | 0.6          | Speeds up COPY transfers                                                                                      |
-| `--referenced-save-only`            | 0.6          | Changes the behavior of SAVE commands in a significant way                                                    |
-| `--for-in`                          | 0.6          | Enables support for `FOR ... IN ...` commands                                                                 |
-| `--require-force-for-unsafe-saves`  | 0.6          | Requires `--force` for saving artifacts locally outside the Earthfile's directory                             |
-| `--no-implicit-ignore`              | 0.6          | Eliminates implicit `.earthlyignore` entries, such as `Earthfile` and `.tmp-earthly-out`                      |
-| `--earthly-version-arg`             | 0.7          | Enables builtin ARGs: `EARTHLY_VERSION` and `EARTHLY_BUILD_SHA`                                               |
-| `--shell-out-anywhere`              | 0.7          | Allows shelling-out in any earthly command (including in the middle of `ARG`)                                 |
-| `--explicit-global`                 | 0.7          | Base target args must have a `--global` flag in order to be considered global args                            |
-| `--check-duplicate-images`          | 0.7          | Check for duplicate images during output                                                                      |
-| `--use-cache-command`               | 0.7          | Allow use of `CACHE` command in Earthfiles                                                                    |
-| `--use-host-command`                | 0.7          | Allow use of `HOST` command in Earthfiles                                                                     |
-| `--use-copy-link`                   | 0.7          | Use the equivalent of `COPY --link` for all copy-like operations                                              |
-| `--new-platform`                    | 0.7          | Enable new platform behavior                                                                                  |
-| `--no-tar-build-output`             | 0.7          | Do not print output when creating a tarball to load into `WITH DOCKER`                                        |
-| `--use-no-manifest-list`            | 0.7          | Enable the `SAVE IMAGE --no-manifest-list` option                                                             |
-| `--use-chmod`                       | 0.7          | Enable the `COPY --chmod` option                                                                              |
-| `--earthly-locally-arg`             | 0.7          | Enable the `EARTHLY_LOCALLY` arg                                                                              |
-| `--use-project-secrets`             | 0.7          | Enable project-based secret resolution                                                                        |
-| `--use-pipelines`                   | 0.7          | Enable the `PIPELINE` and `TRIGGER` commands                                                                  |
-| `--earthly-git-author-args`         | 0.7          | Enable the `EARTHLY_GIT_AUTHOR` and `EARTHLY_GIT_CO_AUTHORS` args                                             |
-| `--wait-block`                      | 0.7          | Enable the `WAIT` / `END` block commands                                                                      |
-| `--no-use-registry-for-with-docker` | Experimental | Disable `use-registry-for-with-docker`                                                                        |
-| `--try`                             | Experimental | Enable the `TRY` / `FINALLY` / `END` block commands                                                           |
-| `--no-network`                      | Experimental | Allow the use of `RUN --network=none` commands                                                                |
-| `--arg-scope-and-set`               | Experimental | Enable the `LET` / `SET` commands and nested `ARG` scoping                                                    |
-| `--earthly-ci-runner-arg`           | Experimental | Enable the `EARTHLY_CI_RUNNER` builtin ARG                                                                    |
-| `--use-docker-ignore`               | Experimental | Enable the use of `.dockerignore` files in `FROM DOCKERFILE` targets                                          |
-| `--pass-args`                       | Experimental | Enable the optional `--pass-args` flag for the `BUILD`, `FROM`, `COPY`, `WITH DOCKER --load` commands         |
-| `--global-cache`                    | Experimental | Enable global caches (shared across different Earthfiles), for cache mounts and `CACHE` commands having an ID |
+| Feature flag                        | status       | description                                                                                                        |
+|-------------------------------------|--------------|--------------------------------------------------------------------------------------------------------------------|
+| `--use-registry-for-with-docker`    | 0.5          | Makes use of the embedded BuildKit Docker registry (instead of tar files) for `WITH DOCKER` loads and pulls        |
+| `--use-copy-include-patterns`       | 0.6          | Speeds up COPY transfers                                                                                           |
+| `--referenced-save-only`            | 0.6          | Changes the behavior of SAVE commands in a significant way                                                         |
+| `--for-in`                          | 0.6          | Enables support for `FOR ... IN ...` commands                                                                      |
+| `--require-force-for-unsafe-saves`  | 0.6          | Requires `--force` for saving artifacts locally outside the Earthfile's directory                                  |
+| `--no-implicit-ignore`              | 0.6          | Eliminates implicit `.earthlyignore` entries, such as `Earthfile` and `.tmp-earthly-out`                           |
+| `--earthly-version-arg`             | 0.7          | Enables builtin ARGs: `EARTHLY_VERSION` and `EARTHLY_BUILD_SHA`                                                    |
+| `--shell-out-anywhere`              | 0.7          | Allows shelling-out in any earthly command (including in the middle of `ARG`)                                      |
+| `--explicit-global`                 | 0.7          | Base target args must have a `--global` flag in order to be considered global args                                 |
+| `--check-duplicate-images`          | 0.7          | Check for duplicate images during output                                                                           |
+| `--use-cache-command`               | 0.7          | Allow use of `CACHE` command in Earthfiles                                                                         |
+| `--use-host-command`                | 0.7          | Allow use of `HOST` command in Earthfiles                                                                          |
+| `--use-copy-link`                   | 0.7          | Use the equivalent of `COPY --link` for all copy-like operations                                                   |
+| `--new-platform`                    | 0.7          | Enable new platform behavior                                                                                       |
+| `--no-tar-build-output`             | 0.7          | Do not print output when creating a tarball to load into `WITH DOCKER`                                             |
+| `--use-no-manifest-list`            | 0.7          | Enable the `SAVE IMAGE --no-manifest-list` option                                                                  |
+| `--use-chmod`                       | 0.7          | Enable the `COPY --chmod` option                                                                                   |
+| `--earthly-locally-arg`             | 0.7          | Enable the `EARTHLY_LOCALLY` arg                                                                                   |
+| `--use-project-secrets`             | 0.7          | Enable project-based secret resolution                                                                             |
+| `--use-pipelines`                   | 0.7          | Enable the `PIPELINE` and `TRIGGER` commands                                                                       |
+| `--earthly-git-author-args`         | 0.7          | Enable the `EARTHLY_GIT_AUTHOR` and `EARTHLY_GIT_CO_AUTHORS` args                                                  |
+| `--wait-block`                      | 0.7          | Enable the `WAIT` / `END` block commands                                                                           |
+| `--no-use-registry-for-with-docker` | Experimental | Disable `use-registry-for-with-docker`                                                                             |
+| `--try`                             | Experimental | Enable the `TRY` / `FINALLY` / `END` block commands                                                                |
+| `--no-network`                      | Experimental | Allow the use of `RUN --network=none` commands                                                                     |
+| `--arg-scope-and-set`               | Experimental | Enable the `LET` / `SET` commands and nested `ARG` scoping                                                         |
+| `--earthly-ci-runner-arg`           | Experimental | Enable the `EARTHLY_CI_RUNNER` builtin ARG                                                                         |
+| `--use-docker-ignore`               | Experimental | Enable the use of `.dockerignore` files in `FROM DOCKERFILE` targets                                               |
+| `--pass-args`                       | Experimental | Enable the optional `--pass-args` flag for the `BUILD`, `FROM`, `COPY`, `WITH DOCKER --load` commands              |
+| `--global-cache`                    | Experimental | Enable global caches (shared across different Earthfiles), for cache mounts and `CACHE` commands having an ID      |
+| `--cache-persist-option`            | Experimental | Adds `CACHE --persist` option to persist cache content in images, Changes default `CACHE` behaviour to not persist |
 
 Note that the features flags are disabled by default in Earthly versions lower than the version listed in the "status" column above.
 

earthfile2llb/converter.go

@@ -103,7 +103,7 @@ type Converter struct {
 	directDeps          []*states.SingleTarget
 	buildContextFactory llbfactory.Factory
 	cacheContext        pllb.State
-	persistentCacheDirs map[string]llb.RunOption // maps path->mount
+	persistentCacheDirs map[string]states.CacheMount // maps path->mount
 	varCollection       *variables.Collection
 	ranSave             bool
 	cmdSet              bool
@@ -163,7 +163,7 @@ func NewConverter(ctx context.Context, target domain.Target, bc *buildcontext.Da
 		mts:                 mts,
 		buildContextFactory: bc.BuildContextFactory,
 		cacheContext:        pllb.Scratch(),
-		persistentCacheDirs: make(map[string]llb.RunOption),
+		persistentCacheDirs: make(map[string]states.CacheMount),
 		varCollection:       variables.NewCollection(newCollOpt),
 		ftrs:                bc.Features,
 		localWorkingDir:     filepath.Dir(bc.BuildFilePath),
@@ -186,7 +186,7 @@ func (c *Converter) From(ctx context.Context, imageName string, platform platuti
 	}
 	c.nonSaveCommand()
 	if len(c.persistentCacheDirs) > 0 {
-		c.persistentCacheDirs = make(map[string]llb.RunOption)
+		c.persistentCacheDirs = make(map[string]states.CacheMount)
 	}
 	c.cmdSet = false
 	err = c.checkOldPlatformIncompatibility(platform)
@@ -610,8 +610,8 @@ func (c *Converter) Run(ctx context.Context, opts ConvertRunOpts) error {
 	}
 	c.nonSaveCommand()
 
-	for _, cache := range c.persistentCacheDirs {
-		opts.extraRunOpts = append(opts.extraRunOpts, cache)
+	for _, state := range c.persistentCacheDirs {
+		opts.extraRunOpts = append(opts.extraRunOpts, state.RunOption)
 	}
 	_, err = c.internalRun(ctx, opts)
 	return err
@@ -624,8 +624,8 @@ func (c *Converter) RunExitCode(ctx context.Context, opts ConvertRunOpts) (int,
 		return 0, err
 	}
 	c.nonSaveCommand()
-	for _, cache := range c.persistentCacheDirs {
-		opts.extraRunOpts = append(opts.extraRunOpts, cache)
+	for _, state := range c.persistentCacheDirs {
+		opts.extraRunOpts = append(opts.extraRunOpts, state.RunOption)
 	}
 
 	var exitCodeFile string
@@ -688,17 +688,17 @@ func (c *Converter) RunExitCode(ctx context.Context, opts ConvertRunOpts) (int,
 // RunExpression runs an expression and returns its output. The run is transient - any state created
 // is not used in subsequent commands.
 func (c *Converter) RunExpression(ctx context.Context, expressionName string, opts ConvertRunOpts) (string, error) {
-	for _, cache := range c.persistentCacheDirs {
-		opts.extraRunOpts = append(opts.extraRunOpts, cache)
+	for _, state := range c.persistentCacheDirs {
+		opts.extraRunOpts = append(opts.extraRunOpts, state.RunOption)
 	}
 	return c.runCommand(ctx, expressionName, true, opts)
 }
 
 // RunCommand runs a command and returns its output. The run is transient - any state created
 // is not used in subsequent commands.
 func (c *Converter) RunCommand(ctx context.Context, commandName string, opts ConvertRunOpts) (string, error) {
-	for _, cache := range c.persistentCacheDirs {
-		opts.extraRunOpts = append(opts.extraRunOpts, cache)
+	for _, state := range c.persistentCacheDirs {
+		opts.extraRunOpts = append(opts.extraRunOpts, state.RunOption)
 	}
 	return c.runCommand(ctx, commandName, false, opts)
 }
@@ -1439,8 +1439,8 @@ func (c *Converter) WithDockerRun(ctx context.Context, args []string, opt WithDo
 
 	enableParallel := allowParallel && c.opt.ParallelConversion && c.ftrs.ParallelLoad
 
-	for _, cache := range c.persistentCacheDirs {
-		opt.extraRunOpts = append(opt.extraRunOpts, cache)
+	for _, state := range c.persistentCacheDirs {
+		opt.extraRunOpts = append(opt.extraRunOpts, state.RunOption)
 	}
 
 	if c.ftrs.UseRegistryForWithDocker {
@@ -1541,7 +1541,16 @@ func (c *Converter) Cache(ctx context.Context, mountTarget string, opts commandf
 		if err != nil {
 			return errors.Errorf("failed to parse mount mode %s", opts.Mode)
 		}
-		c.persistentCacheDirs[mountTarget] = pllb.AddMount(mountTarget, pllb.Scratch().File(pllb.Mkdir("/cache", os.FileMode(mountMode))), mountOpts...)
+		persisted := true // Without new --cache-persist-option we use old behaviour which is persisted
+		if c.ftrs.CachePersistOption {
+			persisted = opts.Persist
+		} else if opts.Persist {
+			return errors.Errorf("the --persist flag is only available when VERSION --cache-persist-option is enabled")
+		}
+		c.persistentCacheDirs[mountTarget] = states.CacheMount{
+			Persisted: persisted,
+			RunOption: pllb.AddMount(mountTarget, pllb.Scratch().File(pllb.Mkdir("/cache", os.FileMode(mountMode))), mountOpts...),
+		}
 	}
 	return nil
 }
@@ -2496,18 +2505,20 @@ func (c *Converter) checkAllowed(command cmdType) error {
 func (c *Converter) persistCache(srcState pllb.State) pllb.State {
 	dest := srcState
 	// User may have multiple CACHE commands in a single target
-	for dir, cache := range c.persistentCacheDirs {
-		// Copy the contents of the user's cache directory to the temporary backup.
-		// It's important to use DockerfileCopy here, since traditional llb.Copy()
-		// doesn't support adding mounts via RunOptions.
-		runOpts := []llb.RunOption{cache, llb.WithCustomName("persist cache directory")}
-		dest = llbutil.CopyWithRunOptions(
-			dest,
-			dir, // cache dir from external mount
-			dir, // cache dir on dest state (same location but without the mount)
-			c.platr,
-			runOpts...,
-		)
+	for dir, state := range c.persistentCacheDirs {
+		if state.Persisted {
+			// Copy the contents of the user's cache directory to the temporary backup.
+			// It's important to use DockerfileCopy here, since traditional llb.Copy()
+			// doesn't support adding mounts via RunOptions.
+			runOpts := []llb.RunOption{state.RunOption, llb.WithCustomName("persist cache directory: " + dir)}
+			dest = llbutil.CopyWithRunOptions(
+				dest,
+				dir, // cache dir from external mount
+				dir, // cache dir on dest state (same location but without the mount)
+				c.platr,
+				runOpts...,
+			)
+		}
 	}
 
 	return dest

features/features.go

@@ -61,6 +61,7 @@ type Features struct {
 	UseDockerIgnore                 bool `long:"use-docker-ignore" description:"fallback to .dockerignore incase .earthlyignore or .earthignore do not exist in a local \"FROM DOCKERFILE\" target"`
 	PassArgs                        bool `long:"pass-args" description:"Allow the use of the --pass-arg flag in FROM, BUILD, COPY, WITH DOCKER, and DO commands"`
 	GlobalCache                     bool `long:"global-cache" description:"enable global caches (shared across different Earthfiles), for cache mounts and CACHEs having an ID"`
+	CachePersistOption              bool `long:"cache-persist-option" description:"Adds option to persist caches, Changes default CACHE behaviour to not persist"`
 	GitRefs                         bool `long:"git-refs" description:"includes EARTHLY_GIT_REFS ARG"`
 	UseVisitedUpfrontHashCollection bool `long:"use-visited-upfront-hash-collection" description:"Uses a new target visitor implementation that computes upfront the hash of the visited targets and adds support for running all targets with the same name but different args in parallel"`
 

states/states.go

@@ -2,6 +2,7 @@ package states
 
 import (
 	"context"
+	"github.com/moby/buildkit/client/llb"
 	"sync"
 
 	"github.com/earthly/earthly/domain"
@@ -25,6 +26,14 @@ type MultiTarget struct {
 	Final *SingleTarget
 }
 
+// CacheMount holds run options needed to cache mounts, and some extra options.
+type CacheMount struct {
+	// Persisted should the cache be persisted to image.
+	Persisted bool
+	// RunOption Run options
+	RunOption llb.RunOption
+}
+
 // FinalTarget returns the final target of the states.
 func (mts *MultiTarget) FinalTarget() domain.Target {
 	return mts.Final.Target

tests/Earthfile

@@ -124,6 +124,7 @@ ga-no-qemu-group2:
     BUILD +test-cache-mount-mode
     BUILD +test-cache-mode
     BUILD +test-shared-cache
+    BUILD +test-cache-persist
     BUILD +test-visited-upfront-hash-collection
     BUILD +test-exec-stats
 
@@ -1444,6 +1445,22 @@ test-shared-cache:
       --extra_args=" --secret=content=filecontents " \
       --output_contains="filecontents"
 
+test-cache-persist:
+  DO +RUN_EARTHLY \
+      --earthfile=cache-persist.earth \
+      --target=+test-persisted \
+      --output_contains="| test persisted ok"
+
+  DO +RUN_EARTHLY \
+      --earthfile=cache-persist.earth \
+      --target=+test-non-persisted \
+      --output_contains="| test non persisted ok"
+
+  DO +RUN_EARTHLY \
+      --earthfile=cache-persist-no-flag.earth \
+      --target=+test-persisted \
+      --output_contains="| test persisted ok"
+
 test-visited-upfront-hash-collection:
   DO +RUN_EARTHLY \
       --earthfile=visited-upfront-hash-collection.earth \