From a3069ce1f9caaa4d7993b0a117ea9f80e66ff73a Mon Sep 17 00:00:00 2001 From: Mike Holly Date: Sat, 21 Oct 2023 11:14:17 -0700 Subject: [PATCH] Auto-skip arg expansion (#3404) This PR adds support for ARG passing and expansion to auto-skip. CLI args, `BUILD` args, `--pass-args`, and other command args are now used to generate the hash. Changing any of these will invalidate the auto-skip check. --- cmd/earthly/subcmd/build_cmd.go | 277 +++++++----- earthfile2llb/earthfile_info.go | 7 +- earthfile2llb/interpreter.go | 189 ++------- earthfile2llb/interpreter_test.go | 45 -- inputgraph/hash.go | 38 ++ .../{inputgraph_test.go => hash_test.go} | 29 +- inputgraph/{inputgraph.go => loader.go} | 394 +++++++++--------- inputgraph/loader_hashing.go | 43 ++ inputgraph/util.go | 37 ++ tests/autoskip/Earthfile | 86 +++- tests/autoskip/expand-args.earth | 60 +++ util/flagutil/parse.go | 124 ++++++ util/flagutil/parse_test.go | 49 ++- variables/builtin.go | 12 +- variables/collection.go | 1 + 15 files changed, 830 insertions(+), 561 deletions(-) create mode 100644 inputgraph/hash.go rename inputgraph/{inputgraph_test.go => hash_test.go} (76%) rename inputgraph/{inputgraph.go => loader.go} (55%) create mode 100644 inputgraph/loader_hashing.go create mode 100644 inputgraph/util.go create mode 100644 tests/autoskip/expand-args.earth diff --git a/cmd/earthly/subcmd/build_cmd.go b/cmd/earthly/subcmd/build_cmd.go index 654998d5..0be66e63 100644 --- a/cmd/earthly/subcmd/build_cmd.go +++ b/cmd/earthly/subcmd/build_cmd.go @@ -21,6 +21,7 @@ import ( "github.com/docker/cli/cli/config" "github.com/fatih/color" "github.com/joho/godotenv" + bkclient "github.com/moby/buildkit/client" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/session" "github.com/moby/buildkit/session/auth" @@ -262,113 +263,6 @@ func (a *Build) ActionBuildImp(cliCtx *cli.Context, flagArgs, nonFlagArgs []stri a.cli.Console().PrintPhaseHeader(builder.PhaseInit, false, "") a.warnIfArgContainsBuildArg(flagArgs) - var skipDB bk.BuildkitSkipper - var targetHash []byte - if a.cli.Flags().SkipBuildkit { - var orgName string - var projectName string - orgName, projectName, targetHash, err = inputgraph.HashTarget(cliCtx.Context, target, a.cli.Console()) - if err != nil { - a.cli.Console().Warnf("unable to calculate hash for %s: %s", target.String(), err.Error()) - } else { - skipDB, err = bk.NewBuildkitSkipper(a.cli.Flags().LocalSkipDB, orgName, projectName, target.GetName(), cloudClient) - if err != nil { - return err - } - exists, err := skipDB.Exists(cliCtx.Context, targetHash) - if err != nil { - a.cli.Console().Warnf("unable to check if target %s (hash %x) has already been run: %s", target.String(), targetHash, err.Error()) - } - if exists { - a.cli.Console().Printf("target %s (hash %x) has already been run; exiting", target.String(), targetHash) - return nil - } - } - } - - err = a.cli.InitFrontend(cliCtx) - if err != nil { - return errors.Wrapf(err, "could not init frontend") - } - - err = a.cli.ConfigureSatellite(cliCtx, cloudClient, gitCommitAuthor, gitConfigEmail) - if err != nil { - return errors.Wrapf(err, "could not configure satellite") - } - - // After configuring frontend and satellites, buildkit address should not be empty. - // It should be set to a local container, remote address, or satellite address at this point. - if a.cli.Flags().BuildkitdSettings.BuildkitAddress == "" { - return errors.New("could not determine buildkit address - is Docker or Podman running?") - } - - var runnerName string - isLocal := containerutil.IsLocal(a.cli.Flags().BuildkitdSettings.BuildkitAddress) - if isLocal { - hostname, err := os.Hostname() - if err != nil { - a.cli.Console().Warnf("failed to get hostname: %v", err) - hostname = "unknown" - } - runnerName = fmt.Sprintf("local:%s", hostname) - } else { - if a.cli.Flags().SatelliteName != "" { - runnerName = fmt.Sprintf("sat:%s/%s", a.cli.Flags().OrgName, a.cli.Flags().SatelliteName) - } else { - runnerName = fmt.Sprintf("bk:%s", a.cli.Flags().BuildkitdSettings.BuildkitAddress) - } - } - if !isLocal && (a.cli.Flags().UseInlineCache || a.cli.Flags().SaveInlineCache) { - a.cli.Console().Warnf("Note that inline cache (--use-inline-cache and --save-inline-cache) occasionally cause builds to get stuck at 100%% CPU on Satellites and remote Buildkit.") - a.cli.Console().Warnf("") // newline - } - if isLocal && !a.cli.Flags().ContainerFrontend.IsAvailable(cliCtx.Context) { - return errors.New("Frontend is not available to perform the build. Is Docker installed and running?") - } - - bkClient, err := buildkitd.NewClient( - cliCtx.Context, - a.cli.Console(), - a.cli.Flags().BuildkitdImage, - a.cli.Flags().ContainerName, - a.cli.Flags().InstallationName, - a.cli.Flags().ContainerFrontend, - a.cli.Version(), - a.cli.Flags().BuildkitdSettings, - ) - if err != nil { - return errors.Wrap(err, "build new buildkitd client") - } - defer bkClient.Close() - a.cli.SetAnaMetaIsRemoteBK(!isLocal) - - nativePlatform, err := platutil.GetNativePlatformViaBkClient(cliCtx.Context, bkClient) - if err != nil { - return errors.Wrap(err, "get native platform via buildkit client") - } - if a.cli.Flags().Logstream { - a.cli.LogbusSetup().SetDefaultPlatform(platforms.Format(nativePlatform)) - } - platr := platutil.NewResolver(nativePlatform) - a.cli.SetAnaMetaBKPlatform(platforms.Format(nativePlatform)) - a.cli.SetAnaMetaUserPlatform(platforms.Format(platr.LLBUser())) - platr.AllowNativeAndUser = true - platformsSlice := make([]platutil.Platform, 0, len(a.platformsStr.Value())) - for _, p := range a.platformsStr.Value() { - platform, err := platr.Parse(p) - if err != nil { - return errors.Wrapf(err, "parse platform %s", p) - } - platformsSlice = append(platformsSlice, platform) - } - switch len(platformsSlice) { - case 0: - case 1: - platr.UpdatePlatform(platformsSlice[0]) - default: - return errors.Errorf("multi-platform builds are not yet supported on the command line. You may, however, create a target with the instruction BUILD --platform ... --platform ... %s", target) - } - showUnexpectedEnvWarnings := true dotEnvMap, err := godotenv.Read(a.cli.Flags().EnvFile) if err != nil { @@ -416,6 +310,62 @@ func (a *Build) ActionBuildImp(cliCtx *cli.Context, flagArgs, nonFlagArgs []stri } } + overridingVars, err := common.CombineVariables(argMap, flagArgs, a.buildArgs.Value()) + if err != nil { + return err + } + + skipDB, targetHash, doSkip, err := a.initAutoSkip(cliCtx.Context, target, overridingVars, cloudClient) + if err != nil { + return err + } + if doSkip { + return nil + } + + err = a.cli.InitFrontend(cliCtx) + if err != nil { + return errors.Wrapf(err, "could not init frontend") + } + + err = a.cli.ConfigureSatellite(cliCtx, cloudClient, gitCommitAuthor, gitConfigEmail) + if err != nil { + return errors.Wrapf(err, "could not configure satellite") + } + + // After configuring frontend and satellites, buildkit address should not be empty. + // It should be set to a local container, remote address, or satellite address at this point. + if a.cli.Flags().BuildkitdSettings.BuildkitAddress == "" { + return errors.New("could not determine buildkit address - is Docker or Podman running?") + } + + bkClient, err := buildkitd.NewClient( + cliCtx.Context, + a.cli.Console(), + a.cli.Flags().BuildkitdImage, + a.cli.Flags().ContainerName, + a.cli.Flags().InstallationName, + a.cli.Flags().ContainerFrontend, + a.cli.Version(), + a.cli.Flags().BuildkitdSettings, + ) + if err != nil { + return errors.Wrap(err, "build new buildkitd client") + } + defer bkClient.Close() + + platr, err := a.platformResolver(cliCtx.Context, bkClient, target) + if err != nil { + return err + } + + runnerName, isLocal, err := a.runnerName(cliCtx.Context) + if err != nil { + return err + } + + a.cli.SetAnaMetaIsRemoteBK(!isLocal) + localhostProvider, err := localhostprovider.NewLocalhostProvider() if err != nil { return errors.Wrap(err, "failed to create localhostprovider") @@ -512,11 +462,6 @@ func (a *Build) ActionBuildImp(cliCtx *cli.Context, flagArgs, nonFlagArgs []stri enttlmnts = append(enttlmnts, entitlements.EntitlementSecurityInsecure) } - overridingVars, err := common.CombineVariables(argMap, flagArgs, a.buildArgs.Value()) - if err != nil { - return err - } - imageResolveMode := llb.ResolveModePreferLocal if a.cli.Flags().Pull { imageResolveMode = llb.ResolveModeForcePull @@ -807,6 +752,114 @@ func receiveFileVersion2(ctx context.Context, conn io.ReadWriteCloser, localArti return f.Close() } +// runnerName returns the name of the local or remote BK "runner"; which is a +// representation of what BuildKit instance is being used, +// e.g. local:, sat:/, or bk: +func (a *Build) runnerName(ctx context.Context) (string, bool, error) { + var runnerName string + isLocal := containerutil.IsLocal(a.cli.Flags().BuildkitdSettings.BuildkitAddress) + if isLocal { + hostname, err := os.Hostname() + if err != nil { + a.cli.Console().Warnf("failed to get hostname: %v", err) + hostname = "unknown" + } + runnerName = fmt.Sprintf("local:%s", hostname) + } else { + if a.cli.Flags().SatelliteName != "" { + runnerName = fmt.Sprintf("sat:%s/%s", a.cli.Flags().OrgName, a.cli.Flags().SatelliteName) + } else { + runnerName = fmt.Sprintf("bk:%s", a.cli.Flags().BuildkitdSettings.BuildkitAddress) + } + } + if !isLocal && (a.cli.Flags().UseInlineCache || a.cli.Flags().SaveInlineCache) { + a.cli.Console().Warnf("Note that inline cache (--use-inline-cache and --save-inline-cache) occasionally cause builds to get stuck at 100%% CPU on Satellites and remote Buildkit.") + a.cli.Console().Warnf("") // newline + } + if isLocal && !a.cli.Flags().ContainerFrontend.IsAvailable(ctx) { + return "", false, errors.New("Frontend is not available to perform the build. Is Docker installed and running?") + } + return runnerName, isLocal, nil +} + +func (a *Build) platformResolver(ctx context.Context, bkClient *bkclient.Client, target domain.Target) (*platutil.Resolver, error) { + nativePlatform, err := platutil.GetNativePlatformViaBkClient(ctx, bkClient) + if err != nil { + return nil, errors.Wrap(err, "get native platform via buildkit client") + } + if a.cli.Flags().Logstream { + a.cli.LogbusSetup().SetDefaultPlatform(platforms.Format(nativePlatform)) + } + platr := platutil.NewResolver(nativePlatform) + a.cli.SetAnaMetaBKPlatform(platforms.Format(nativePlatform)) + a.cli.SetAnaMetaUserPlatform(platforms.Format(platr.LLBUser())) + platr.AllowNativeAndUser = true + platformsSlice := make([]platutil.Platform, 0, len(a.platformsStr.Value())) + for _, p := range a.platformsStr.Value() { + platform, err := platr.Parse(p) + if err != nil { + return nil, errors.Wrapf(err, "parse platform %s", p) + } + platformsSlice = append(platformsSlice, platform) + } + switch len(platformsSlice) { + case 0: + case 1: + platr.UpdatePlatform(platformsSlice[0]) + default: + return nil, errors.Errorf("multi-platform builds are not yet supported on the command line. You may, however, create a target with the instruction BUILD --platform ... --platform ... %s", target) + } + + return platr, nil +} + +func (a *Build) initAutoSkip(ctx context.Context, target domain.Target, overridingVars *variables.Scope, client *cloud.Client) (bk.BuildkitSkipper, []byte, bool, error) { + if !a.cli.Flags().SkipBuildkit { + return nil, nil, false, nil + } + + var ( + skipDB bk.BuildkitSkipper + targetHash []byte + orgName string + projectName string + ) + + console := a.cli.Console() + + orgName, projectName, targetHash, err := inputgraph.HashTarget(ctx, inputgraph.HashOpt{ + Target: target, + Console: a.cli.Console(), + CI: a.cli.Flags().CI, + Push: a.cli.Flags().Push, + BuiltinArgs: variables.DefaultArgs{EarthlyVersion: a.cli.Version(), EarthlyBuildSha: a.cli.GitSHA()}, + OverridingVars: overridingVars, + EarthlyCIRunner: a.cli.Flags().EarthlyCIRunner, + }) + if err != nil { + console.Warnf("unable to calculate hash for %s: %s", target.String(), err.Error()) + return nil, nil, false, nil + } + + skipDB, err = bk.NewBuildkitSkipper(a.cli.Flags().LocalSkipDB, orgName, projectName, target.GetName(), client) + if err != nil { + return nil, nil, false, err + } + + exists, err := skipDB.Exists(ctx, targetHash) + if err != nil { + console.Warnf("unable to check if target %s (hash %x) has already been run: %s", target.String(), targetHash, err.Error()) + return nil, nil, false, nil + } + + if exists { + console.Printf("target %s (hash %x) has already been run; exiting", target.String(), targetHash) + return nil, nil, true, nil + } + + return skipDB, targetHash, false, nil +} + func (a *Build) logShareLink(ctx context.Context, cloudClient *cloud.Client, target domain.Target, clean *cleanup.Collection) (string, bool, func()) { if a.cli.Cfg().Global.DisableLogSharing { diff --git a/earthfile2llb/earthfile_info.go b/earthfile2llb/earthfile_info.go index 03071db7..3eb3f5c9 100644 --- a/earthfile2llb/earthfile_info.go +++ b/earthfile2llb/earthfile_info.go @@ -8,6 +8,7 @@ import ( "github.com/earthly/earthly/ast/spec" "github.com/earthly/earthly/buildcontext" "github.com/earthly/earthly/domain" + "github.com/earthly/earthly/util/flagutil" "github.com/earthly/earthly/util/platutil" gwclient "github.com/moby/buildkit/frontend/gateway/client" "github.com/pkg/errors" @@ -54,7 +55,7 @@ func GetTargetArgs(ctx context.Context, resolver *buildcontext.Resolver, gwClien isBase := t.Name == "base" // since Arg opts are ignored (and feature flags are not available) we set explicitGlobalArgFlag as false explicitGlobal := false - _, argName, _, err := parseArgArgs(ctx, *stmt.Command, isBase, explicitGlobal) + _, argName, _, err := flagutil.ParseArgArgs(ctx, *stmt.Command, isBase, explicitGlobal) if err != nil { return nil, errors.Wrapf(err, "failed to parse ARG arguments %v", stmt.Command.Args) } @@ -71,7 +72,7 @@ func ArgName(ctx context.Context, cmd spec.Command, isBase bool, explicitGlobal if cmd.Name != "ARG" { return "", nil, false, false, errors.Errorf("ArgName was called with non-arg command type '%v'", cmd.Name) } - opts, argName, dflt, err := parseArgArgs(ctx, cmd, isBase, explicitGlobal) + opts, argName, dflt, err := flagutil.ParseArgArgs(ctx, cmd, isBase, explicitGlobal) if err != nil { return "", nil, false, false, errors.Wrapf(err, "could not parse opts for ARG [%v]", cmd) } @@ -97,7 +98,7 @@ func ArtifactName(ctx context.Context, cmd spec.Command) (string, *string, error // ImageNames returns the parsed names of a SAVE IMAGE command. func ImageNames(ctx context.Context, cmd spec.Command) ([]string, error) { var opts commandflag.SaveImageOpts - args, err := parseArgs("SAVE IMAGE", &opts, getArgsCopy(cmd)) + args, err := flagutil.ParseArgs("SAVE IMAGE", &opts, flagutil.GetArgsCopy(cmd)) if err != nil { return nil, errors.Wrapf(err, "invalid SAVE IMAGE arguments %v", cmd.Args) } diff --git a/earthfile2llb/interpreter.go b/earthfile2llb/interpreter.go index 4b5801b6..bd4974c6 100644 --- a/earthfile2llb/interpreter.go +++ b/earthfile2llb/interpreter.go @@ -22,7 +22,6 @@ import ( "github.com/earthly/earthly/util/flagutil" "github.com/earthly/earthly/util/platutil" "github.com/earthly/earthly/util/shell" - "github.com/earthly/earthly/util/stringutil" "github.com/earthly/earthly/variables" "github.com/docker/go-connections/nat" @@ -197,7 +196,7 @@ func (i *Interpreter) handleStatement(ctx context.Context, stmt spec.Statement) func (i *Interpreter) handleCommand(ctx context.Context, cmd spec.Command) (err error) { // The AST should not be modified by any operation. This is a consistency check. - argsCopy := getArgsCopy(cmd) + argsCopy := flagutil.GetArgsCopy(cmd) defer func() { if err != nil { return @@ -350,7 +349,7 @@ func (i *Interpreter) handleIfExpression(ctx context.Context, expression []strin return false, i.errorf(sl, "not enough arguments for IF") } opts := commandflag.IfOpts{} - args, err := parseArgs("IF", &opts, expression) + args, err := flagutil.ParseArgsCleaned("IF", &opts, expression) if err != nil { return false, i.wrapError(err, sl, "invalid IF arguments %v", expression) } @@ -421,7 +420,7 @@ func (i *Interpreter) handleForArgs(ctx context.Context, forArgs []string, sl *s opts := commandflag.ForOpts{ Separators: "\n\t ", } - args, err := parseArgs("FOR", &opts, forArgs) + args, err := flagutil.ParseArgsCleaned("FOR", &opts, forArgs) if err != nil { return "", nil, i.wrapError(err, sl, "invalid FOR arguments %v", forArgs) } @@ -514,7 +513,7 @@ func (i *Interpreter) handleTry(ctx context.Context, tryStmt spec.TryStatement) return i.errorf(tryStmt.SourceLocation, "CATCH/FINALLY body only (currently) supports SAVE ARTIFACT ... AS LOCAL commands; got %s", cmd.Command.Name) } opts := commandflag.SaveArtifactOpts{} - args, err := parseArgs("SAVE ARTIFACT", &opts, getArgsCopy(*cmd.Command)) + args, err := flagutil.ParseArgsCleaned("SAVE ARTIFACT", &opts, flagutil.GetArgsCopy(*cmd.Command)) if err != nil { return i.wrapError(err, cmd.Command.SourceLocation, "invalid SAVE ARTIFACT arguments %v", cmd.Command.Args) } @@ -575,7 +574,7 @@ func (i *Interpreter) handleFrom(ctx context.Context, cmd spec.Command) error { return i.pushOnlyErr(cmd.SourceLocation) } opts := commandflag.FromOpts{} - args, err := parseArgs("FROM", &opts, getArgsCopy(cmd)) + args, err := flagutil.ParseArgsCleaned("FROM", &opts, flagutil.GetArgsCopy(cmd)) if err != nil { return i.wrapError(err, cmd.SourceLocation, "invalid FROM arguments %v", cmd.Args) } @@ -679,7 +678,7 @@ func (i *Interpreter) handleRun(ctx context.Context, cmd spec.Command) error { return i.errorf(cmd.SourceLocation, "not enough arguments for RUN") } opts := commandflag.RunOpts{} - args, err := parseArgsWithValueModifier("RUN", &opts, getArgsCopy(cmd), i.flagValModifierFuncWithContext(ctx)) + args, err := flagutil.ParseArgsWithValueModifierCleaned("RUN", &opts, flagutil.GetArgsCopy(cmd), i.flagValModifierFuncWithContext(ctx)) if err != nil { return i.wrapError(err, cmd.SourceLocation, "invalid RUN arguments %v", cmd.Args) } @@ -797,7 +796,7 @@ func (i *Interpreter) handleFromDockerfile(ctx context.Context, cmd spec.Command return i.pushOnlyErr(cmd.SourceLocation) } opts := commandflag.FromDockerfileOpts{} - args, err := parseArgs("FROM DOCKERFILE", &opts, getArgsCopy(cmd)) + args, err := flagutil.ParseArgsCleaned("FROM DOCKERFILE", &opts, flagutil.GetArgsCopy(cmd)) if err != nil { return i.wrapError(err, cmd.SourceLocation, "invalid FROM DOCKERFILE arguments %v", cmd.Args) } @@ -875,7 +874,7 @@ func (i *Interpreter) handleCopy(ctx context.Context, cmd spec.Command) error { return i.pushOnlyErr(cmd.SourceLocation) } opts := commandflag.CopyOpts{} - args, err := parseArgs("COPY", &opts, getArgsCopy(cmd)) + args, err := flagutil.ParseArgsCleaned("COPY", &opts, flagutil.GetArgsCopy(cmd)) if err != nil { return i.wrapError(err, cmd.SourceLocation, "invalid COPY arguments %v", cmd.Args) } @@ -926,9 +925,9 @@ func (i *Interpreter) handleCopy(ctx context.Context, cmd spec.Command) error { for index, src := range srcs { var artifactSrc domain.Artifact var parseErr error - if isInParamsForm(src) { + if flagutil.IsInParamsForm(src) { // COPY ( ) ... - artifactStr, extraArgs, err := parseParams(src) + artifactStr, extraArgs, err := flagutil.ParseParams(src) if err != nil { return i.wrapError(err, cmd.SourceLocation, "parse params %s", src) } @@ -1045,7 +1044,7 @@ func parseSaveArtifactArgs(args []string) (from, to, asLocal string, _ bool) { func (i *Interpreter) handleSaveArtifact(ctx context.Context, cmd spec.Command) error { opts := commandflag.SaveArtifactOpts{} - args, err := parseArgs("SAVE ARTIFACT", &opts, getArgsCopy(cmd)) + args, err := flagutil.ParseArgsCleaned("SAVE ARTIFACT", &opts, flagutil.GetArgsCopy(cmd)) if err != nil { return i.wrapError(err, cmd.SourceLocation, "invalid SAVE ARTIFACT arguments %v", cmd.Args) } @@ -1101,7 +1100,7 @@ func (i *Interpreter) handleSaveArtifact(ctx context.Context, cmd spec.Command) func (i *Interpreter) handleSaveImage(ctx context.Context, cmd spec.Command) error { opts := commandflag.SaveImageOpts{} - args, err := parseArgs("SAVE IMAGE", &opts, getArgsCopy(cmd)) + args, err := flagutil.ParseArgsCleaned("SAVE IMAGE", &opts, flagutil.GetArgsCopy(cmd)) if err != nil { return i.wrapError(err, cmd.SourceLocation, "invalid SAVE IMAGE arguments %v", cmd.Args) } @@ -1154,7 +1153,7 @@ func (i *Interpreter) handleBuild(ctx context.Context, cmd spec.Command, async b return i.pushOnlyErr(cmd.SourceLocation) } opts := commandflag.BuildOpts{} - args, err := parseArgs("BUILD", &opts, getArgsCopy(cmd)) + args, err := flagutil.ParseArgsCleaned("BUILD", &opts, flagutil.GetArgsCopy(cmd)) if err != nil { return i.wrapError(err, cmd.SourceLocation, "invalid BUILD arguments %v", cmd.Args) } @@ -1274,7 +1273,7 @@ func (i *Interpreter) handleCmd(ctx context.Context, cmd spec.Command) error { return i.pushOnlyErr(cmd.SourceLocation) } withShell := !cmd.ExecMode - cmdArgs := getArgsCopy(cmd) + cmdArgs := flagutil.GetArgsCopy(cmd) if withShell { for index, arg := range cmdArgs { expandedCmd, err := i.expandArgs(ctx, arg, false, false) @@ -1296,7 +1295,7 @@ func (i *Interpreter) handleEntrypoint(ctx context.Context, cmd spec.Command) er return i.pushOnlyErr(cmd.SourceLocation) } withShell := !cmd.ExecMode - entArgs := getArgsCopy(cmd) + entArgs := flagutil.GetArgsCopy(cmd) if withShell { for index, arg := range entArgs { expandedEntrypoint, err := i.expandArgs(ctx, arg, false, false) @@ -1320,7 +1319,7 @@ func (i *Interpreter) handleExpose(ctx context.Context, cmd spec.Command) error if len(cmd.Args) == 0 { return i.errorf(cmd.SourceLocation, "no arguments provided to the EXPOSE command") } - ports := getArgsCopy(cmd) + ports := flagutil.GetArgsCopy(cmd) for index, port := range ports { expandedPort, err := i.expandArgs(ctx, port, false, false) if err != nil { @@ -1355,7 +1354,7 @@ func (i *Interpreter) handleVolume(ctx context.Context, cmd spec.Command) error if len(cmd.Args) == 0 { return i.errorf(cmd.SourceLocation, "no arguments provided to the VOLUME command") } - volumes := getArgsCopy(cmd) + volumes := flagutil.GetArgsCopy(cmd) for index, volume := range volumes { expandedVolume, err := i.expandArgs(ctx, volume, false, false) if err != nil { @@ -1398,52 +1397,11 @@ func (i *Interpreter) handleEnv(ctx context.Context, cmd spec.Command) error { return nil } -var errInvalidSyntax = errors.New("invalid syntax") -var errRequiredArgHasDefault = errors.New("required ARG cannot have a default value") -var errGlobalArgNotInBase = errors.New("global ARG can only be set in the base target") - -// parseArgArgs parses the ARG command's arguments -// and returns the argOpts, key, value (or nil if missing), or error -func parseArgArgs(ctx context.Context, cmd spec.Command, isBaseTarget bool, explicitGlobalFeature bool) (commandflag.ArgOpts, string, *string, error) { - var opts commandflag.ArgOpts - args, err := parseArgs("ARG", &opts, getArgsCopy(cmd)) - if err != nil { - return commandflag.ArgOpts{}, "", nil, err - } - if opts.Global { - // since the global flag is part of the struct, we need to manually return parsing error if it's used while the feature flag is off - if !explicitGlobalFeature { - return commandflag.ArgOpts{}, "", nil, errors.New("unknown flag --global") - } - // global flag can only bet set on base targets - if !isBaseTarget { - return commandflag.ArgOpts{}, "", nil, errGlobalArgNotInBase - } - } else if !explicitGlobalFeature { - // if the feature flag is off, all base target args are considered global - opts.Global = isBaseTarget - } - switch len(args) { - case 3: - if args[1] != "=" { - return commandflag.ArgOpts{}, "", nil, errInvalidSyntax - } - if opts.Required { - return commandflag.ArgOpts{}, "", nil, errRequiredArgHasDefault - } - return opts, args[0], &args[2], nil - case 1: - return opts, args[0], nil, nil - default: - return commandflag.ArgOpts{}, "", nil, errInvalidSyntax - } -} - func (i *Interpreter) handleArg(ctx context.Context, cmd spec.Command) error { if i.pushOnlyAllowed { return i.pushOnlyErr(cmd.SourceLocation) } - opts, key, valueOrNil, err := parseArgArgs(ctx, cmd, i.isBase, i.converter.ftrs.ExplicitGlobal) + opts, key, valueOrNil, err := flagutil.ParseArgArgs(ctx, cmd, i.isBase, i.converter.ftrs.ExplicitGlobal) if err != nil { return i.wrapError(err, cmd.SourceLocation, "invalid ARG arguments %v", cmd.Args) } @@ -1468,13 +1426,13 @@ func (i *Interpreter) handleLet(ctx context.Context, cmd spec.Command) error { return i.pushOnlyErr(cmd.SourceLocation) } var opts commandflag.LetOpts - argsCpy := getArgsCopy(cmd) - args, err := parseArgs("LET", &opts, argsCpy) + argsCpy := flagutil.GetArgsCopy(cmd) + args, err := flagutil.ParseArgsCleaned("LET", &opts, argsCpy) if err != nil { return errors.Wrap(err, "failed to parse LET args") } if len(args) != 3 || args[1] != "=" { - return hint.Wrap(errInvalidSyntax, "LET requires a variable assignment, e.g. LET foo = bar") + return hint.Wrap(flagutil.ErrInvalidSyntax, "LET requires a variable assignment, e.g. LET foo = bar") } key := args[0] @@ -1493,16 +1451,16 @@ func (i *Interpreter) handleLet(ctx context.Context, cmd spec.Command) error { func parseSetArgs(ctx context.Context, cmd spec.Command) (name, value string, _ error) { var opts commandflag.SetOpts - argsCpy := getArgsCopy(cmd) - args, err := parseArgs("SET", &opts, argsCpy) + argsCpy := flagutil.GetArgsCopy(cmd) + args, err := flagutil.ParseArgsCleaned("SET", &opts, argsCpy) if err != nil { return "", "", errors.Wrap(err, "failed to parse SET args") } if len(args) != 3 { - return "", "", errInvalidSyntax + return "", "", flagutil.ErrInvalidSyntax } if args[1] != "=" { - return "", "", errInvalidSyntax + return "", "", flagutil.ErrInvalidSyntax } return args[0], args[2], nil } @@ -1574,7 +1532,7 @@ func (i *Interpreter) handleGitClone(ctx context.Context, cmd spec.Command) erro return i.pushOnlyErr(cmd.SourceLocation) } opts := commandflag.GitCloneOpts{} - args, err := parseArgs("GIT CLONE", &opts, getArgsCopy(cmd)) + args, err := flagutil.ParseArgsCleaned("GIT CLONE", &opts, flagutil.GetArgsCopy(cmd)) if err != nil { return i.wrapError(err, cmd.SourceLocation, "invalid GIT CLONE arguments %v", cmd.Args) } @@ -1612,7 +1570,7 @@ func (i *Interpreter) handleHealthcheck(ctx context.Context, cmd spec.Command) e return i.pushOnlyErr(cmd.SourceLocation) } opts := commandflag.HealthCheckOpts{} - args, err := parseArgs("HEALTHCHECK", &opts, getArgsCopy(cmd)) + args, err := flagutil.ParseArgsCleaned("HEALTHCHECK", &opts, flagutil.GetArgsCopy(cmd)) if err != nil { return i.wrapError(err, cmd.SourceLocation, "invalid HEALTHCHECK arguments %v", cmd.Args) } @@ -1660,7 +1618,7 @@ func (i *Interpreter) handleWithDocker(ctx context.Context, cmd spec.Command) er return i.errorf(cmd.SourceLocation, "cannot use WITH DOCKER within WITH DOCKER") } opts := commandflag.WithDockerOpts{} - args, err := parseArgs("WITH DOCKER", &opts, getArgsCopy(cmd)) + args, err := flagutil.ParseArgsCleaned("WITH DOCKER", &opts, flagutil.GetArgsCopy(cmd)) if err != nil { return i.wrapError(err, cmd.SourceLocation, "invalid WITH DOCKER arguments %v", cmd.Args) } @@ -1777,7 +1735,7 @@ func (i *Interpreter) handleUserCommand(ctx context.Context, cmd spec.Command) e func (i *Interpreter) handleDo(ctx context.Context, cmd spec.Command) error { opts := commandflag.DoOpts{} - args, err := parseArgs("DO", &opts, getArgsCopy(cmd)) + args, err := flagutil.ParseArgsCleaned("DO", &opts, flagutil.GetArgsCopy(cmd)) if err != nil { return i.wrapError(err, cmd.SourceLocation, "invalid DO arguments %v", cmd.Args) } @@ -1833,7 +1791,7 @@ func (i *Interpreter) handleDo(ctx context.Context, cmd spec.Command) error { func (i *Interpreter) handleImport(ctx context.Context, cmd spec.Command) error { opts := commandflag.ImportOpts{} - args, err := parseArgs("IMPORT", &opts, getArgsCopy(cmd)) + args, err := flagutil.ParseArgsCleaned("IMPORT", &opts, flagutil.GetArgsCopy(cmd)) if err != nil { return i.wrapError(err, cmd.SourceLocation, "invalid IMPORT arguments %v", cmd.Args) } @@ -1923,7 +1881,7 @@ func (i *Interpreter) handlePipeline(ctx context.Context, cmd spec.Command) erro } var opts commandflag.PipelineOpts - _, err := parseArgs("PIPELINE", &opts, getArgsCopy(cmd)) + _, err := flagutil.ParseArgsCleaned("PIPELINE", &opts, flagutil.GetArgsCopy(cmd)) if err != nil { return i.wrapError(err, cmd.SourceLocation, "invalid PIPELINE arguments") } @@ -1958,7 +1916,7 @@ func (i *Interpreter) handleCache(ctx context.Context, cmd spec.Command) error { return i.errorf(cmd.SourceLocation, "the CACHE command is not supported in this version") } opts := commandflag.CacheOpts{} - args, err := parseArgs("CACHE", &opts, getArgsCopy(cmd)) + args, err := flagutil.ParseArgsCleaned("CACHE", &opts, flagutil.GetArgsCopy(cmd)) if err != nil { return i.wrapError(err, cmd.SourceLocation, "invalid CACHE arguments %v", cmd.Args) } @@ -2133,8 +2091,8 @@ func ParseLoad(loadStr string) (image string, target string, extraArgs []string, target = strings.Join(words, " ") } } - if isInParamsForm(target) { - target, extraArgs, err = parseParams(target) + if flagutil.IsInParamsForm(target) { + target, extraArgs, err = flagutil.ParseParams(target) if err != nil { return "", "", nil, err } @@ -2142,12 +2100,6 @@ func ParseLoad(loadStr string) (image string, target string, extraArgs []string, return image, target, extraArgs, nil } -func getArgsCopy(cmd spec.Command) []string { - argsCopy := make([]string, len(cmd.Args)) - copy(argsCopy, cmd.Args) - return argsCopy -} - type argGroup struct { key string values []*string @@ -2214,69 +2166,6 @@ func parseKeyValue(arg string) (string, *string, error) { return name, value, nil } -func isInParamsForm(str string) bool { - return (strings.HasPrefix(str, "\"(") && strings.HasSuffix(str, "\")")) || - (strings.HasPrefix(str, "(") && strings.HasSuffix(str, ")")) -} - -// parseParams turns "(+target --flag=something)" into "+target" and []string{"--flag=something"}, -// or "\"(+target --flag=something)\"" into "+target" and []string{"--flag=something"} -func parseParams(str string) (string, []string, error) { - if !isInParamsForm(str) { - return "", nil, errors.New("params atom not in ( ... )") - } - if strings.HasPrefix(str, "\"(") { - str = str[2 : len(str)-2] // remove \"( and )\" - } else { - str = str[1 : len(str)-1] // remove ( and ) - } - var parts []string - var part []rune - nextEscaped := false - inQuotes := false - for _, char := range str { - switch char { - case '"': - if !nextEscaped { - inQuotes = !inQuotes - } - nextEscaped = false - case '\\': - nextEscaped = true - case ' ', '\t', '\n': - if !inQuotes && !nextEscaped { - if len(part) > 0 { - parts = append(parts, string(part)) - part = []rune{} - nextEscaped = false - continue - } else { - nextEscaped = false - continue - } - } - nextEscaped = false - default: - nextEscaped = false - } - part = append(part, char) - } - if nextEscaped { - return "", nil, errors.New("unterminated escape sequence") - } - if inQuotes { - return "", nil, errors.New("no ending quotes") - } - if len(part) > 0 { - parts = append(parts, string(part)) - } - - if len(parts) < 1 { - return "", nil, errors.New("invalid empty params") - } - return parts[0], parts[1:], nil -} - // requiresShellOutOrCmdInvalid returns true if // cmd requires shelling out via $(...), or if the cmd is invalid. // This function is best-effort, and returns false on errors, errors @@ -2345,13 +2234,3 @@ func baseTarget(ref domain.Reference) domain.Target { Target: "base", } } - -func parseArgs(cmdName string, opts interface{}, args []string) ([]string, error) { - processed := stringutil.ProcessParamsAndQuotes(args) - return flagutil.ParseArgs(cmdName, opts, processed) -} - -func parseArgsWithValueModifier(cmdName string, opts interface{}, args []string, argumentModFunc flagutil.ArgumentModFunc) ([]string, error) { - processed := stringutil.ProcessParamsAndQuotes(args) - return flagutil.ParseArgsWithValueModifier(cmdName, opts, processed, argumentModFunc) -} diff --git a/earthfile2llb/interpreter_test.go b/earthfile2llb/interpreter_test.go index b8b610a4..1d0436a3 100644 --- a/earthfile2llb/interpreter_test.go +++ b/earthfile2llb/interpreter_test.go @@ -28,48 +28,3 @@ func TestBuildArgMatrix(t *testing.T) { assert.Equal(t, tt.out, ans) } } - -func TestParseParams(t *testing.T) { - var tests = []struct { - in string - first string - args []string - }{ - {"(+target/art --flag=something)", "+target/art", []string{"--flag=something"}}, - {"(+target/art --flag=something\"\")", "+target/art", []string{"--flag=something\"\""}}, - {"( \n +target/art \t \n --flag=something\t )", "+target/art", []string{"--flag=something"}}, - {"(+target/art --flag=something\\ --another=something)", "+target/art", []string{"--flag=something\\ --another=something"}}, - {"(+target/art --flag=something --another=something)", "+target/art", []string{"--flag=something", "--another=something"}}, - {"(+target/art --flag=\"something in quotes\")", "+target/art", []string{"--flag=\"something in quotes\""}}, - {"(+target/art --flag=\\\"something --not=in-quotes\\\")", "+target/art", []string{"--flag=\\\"something", "--not=in-quotes\\\""}}, - {"(+target/art --flag=look-ma-a-\\))", "+target/art", []string{"--flag=look-ma-a-\\)"}}, - } - - for _, tt := range tests { - t.Run(tt.in, func(t *testing.T) { - actualFirst, actualArgs, err := parseParams(tt.in) - assert.NoError(t, err) - assert.Equal(t, tt.first, actualFirst) - assert.Equal(t, tt.args, actualArgs) - }) - - } -} - -func TestNegativeParseParams(t *testing.T) { - var tests = []struct { - in string - }{ - {"+target/art --flag=something)"}, - {"(+target/art --flag=something"}, - {"(+target/art --flag=\"something)"}, - {"(+target/art --flag=something\\)"}, - {"()"}, - {"( \t\n )"}, - } - - for _, tt := range tests { - _, _, err := parseParams(tt.in) - assert.Error(t, err) - } -} diff --git a/inputgraph/hash.go b/inputgraph/hash.go new file mode 100644 index 00000000..8851b76f --- /dev/null +++ b/inputgraph/hash.go @@ -0,0 +1,38 @@ +package inputgraph + +import ( + "context" + "errors" + + "github.com/earthly/earthly/conslogging" + "github.com/earthly/earthly/domain" + "github.com/earthly/earthly/variables" +) + +// HashOpt contains all of the options available to the hasher. +type HashOpt struct { + Target domain.Target + Console conslogging.ConsoleLogger + CI bool + Push bool + BuiltinArgs variables.DefaultArgs + OverridingVars *variables.Scope + EarthlyCIRunner bool +} + +// HashTarget produces a hash from an Earthly target. +func HashTarget(ctx context.Context, opt HashOpt) (org, project string, hash []byte, err error) { + l := newLoader(ctx, opt) + + org, project, err = l.findProject(ctx) + if err != nil { + return "", "", nil, err + } + + err = l.load(ctx) + if err != nil { + return "", "", nil, errors.Join(ErrUnableToDetermineHash, err) + } + + return org, project, l.hasher.GetHash(), nil +} diff --git a/inputgraph/inputgraph_test.go b/inputgraph/hash_test.go similarity index 76% rename from inputgraph/inputgraph_test.go rename to inputgraph/hash_test.go index 59daa317..22dd01c7 100644 --- a/inputgraph/inputgraph_test.go +++ b/inputgraph/hash_test.go @@ -25,13 +25,14 @@ func TestHashTargetWithDocker(t *testing.T) { ctx := context.Background() cons := conslogging.New(os.Stderr, &sync.Mutex{}, conslogging.NoColor, 0, conslogging.Info) - org, project, hash, err := HashTarget(ctx, target, cons) + hashOpt := HashOpt{Console: cons, Target: target} + org, project, hash, err := HashTarget(ctx, hashOpt) r.NoError(err) r.Equal("earthly-technologies", org) r.Equal("core", project) hex := fmt.Sprintf("%x", hash) - r.Equal("9d2903bc18c99831f4a299090abaf94d25d89321", hex) + r.Equal("f208c09aa799764ae27686f551a60a62769d4183", hex) path := "./testdata/with-docker/Earthfile" @@ -55,11 +56,12 @@ func TestHashTargetWithDocker(t *testing.T) { Target: "with-docker-load", } - _, _, hash, err = HashTarget(ctx, target, cons) + hashOpt = HashOpt{Console: cons, Target: target} + _, _, hash, err = HashTarget(ctx, hashOpt) r.NoError(err) hex = fmt.Sprintf("%x", hash) - r.Equal("84b6f722421695a7ded144c1b72efb3b8f3339c6", hex) + r.Equal("b638f270f40d93abc4618775470602b41e5c0755", hex) } func copyFile(src, dst string) error { @@ -117,25 +119,12 @@ func TestHashTargetWithDockerNoAlias(t *testing.T) { ctx := context.Background() cons := conslogging.New(os.Stderr, &sync.Mutex{}, conslogging.NoColor, 0, conslogging.Info) - org, project, hash, err := HashTarget(ctx, target, cons) + hashOpt := HashOpt{Console: cons, Target: target} + org, project, hash, err := HashTarget(ctx, hashOpt) r.NoError(err) r.Equal("earthly-technologies", org) r.Equal("core", project) hex := fmt.Sprintf("%x", hash) - r.Equal("d73e37689c7780cbff2cba2de1a23141618b7b14", hex) -} - -func TestHashTargetWithDockerArgs(t *testing.T) { - r := require.New(t) - target := domain.Target{ - LocalPath: "./testdata/with-docker", - Target: "with-docker-load-args", - } - - ctx := context.Background() - cons := conslogging.New(os.Stderr, &sync.Mutex{}, conslogging.NoColor, 0, conslogging.Info) - - _, _, _, err := HashTarget(ctx, target, cons) - r.Error(err) + r.Equal("2b25818489d6b44508829424009c4b05e6f14c7a", hex) } diff --git a/inputgraph/inputgraph.go b/inputgraph/loader.go similarity index 55% rename from inputgraph/inputgraph.go rename to inputgraph/loader.go index 791a467f..2583ae37 100644 --- a/inputgraph/inputgraph.go +++ b/inputgraph/loader.go @@ -15,9 +15,10 @@ import ( "github.com/earthly/earthly/conslogging" "github.com/earthly/earthly/domain" "github.com/earthly/earthly/earthfile2llb" + "github.com/earthly/earthly/features" "github.com/earthly/earthly/util/buildkitskipper/hasher" "github.com/earthly/earthly/util/flagutil" - "github.com/earthly/earthly/util/stringutil" + "github.com/earthly/earthly/variables" "github.com/pkg/errors" ) @@ -26,91 +27,83 @@ var ( ErrUnableToDetermineHash = fmt.Errorf("unable to determine hash") ) -func argsContainsStr(args []string, substr string) bool { - for _, s := range args { - if strings.Contains(s, substr) { - return true - } - } - return false -} - -func requiresCrossProduct(args []string) bool { - seen := map[string]struct{}{} - for _, s := range args { - k := strings.SplitN(s, "=", 2)[0] - if _, found := seen[k]; found { - return true - } - seen[k] = struct{}{} +type loader struct { + conslog conslogging.ConsoleLogger + target domain.Target + visited map[string]struct{} + hasher *hasher.Hasher + varCollection *variables.Collection + features *features.Features + isBaseTarget bool + ci bool + push bool + builtinArgs variables.DefaultArgs + overridingVars *variables.Scope + earthlyCIRunner bool +} + +func newLoader(ctx context.Context, opt HashOpt) *loader { + // Other important values are set by load(). + return &loader{ + conslog: opt.Console, + target: opt.Target, + visited: map[string]struct{}{}, + hasher: hasher.New(), + isBaseTarget: opt.Target.Target == "base", + ci: opt.CI, + push: opt.Push, + builtinArgs: opt.BuiltinArgs, + overridingVars: opt.OverridingVars, + earthlyCIRunner: opt.EarthlyCIRunner, } - return false -} - -func getArgsCopy(cmd spec.Command) []string { - argsCopy := make([]string, len(cmd.Args)) - copy(argsCopy, cmd.Args) - return argsCopy -} - -func parseArgs(cmdName string, opts interface{}, args []string) ([]string, error) { - processed := stringutil.ProcessParamsAndQuotes(args) - return flagutil.ParseArgs(cmdName, opts, processed) } func (l *loader) handleFrom(ctx context.Context, cmd spec.Command) error { opts := commandflag.FromOpts{} - args, err := parseArgs(command.From, &opts, getArgsCopy(cmd)) + args, err := flagutil.ParseArgsCleaned(command.From, &opts, flagutil.GetArgsCopy(cmd)) if err != nil { return err } - if argsContainsStr(args, "$") { - return errors.Wrap(ErrUnableToDetermineHash, "unable to handle arg in FROM") - } + fromTarget := args[0] if !strings.Contains(fromTarget, "+") { return nil } - return l.loadTargetFromString(ctx, fromTarget) + + return l.loadTargetFromString(ctx, fromTarget, args[1:], false) } func (l *loader) handleBuild(ctx context.Context, cmd spec.Command) error { opts := commandflag.BuildOpts{} - args, err := parseArgs(command.Build, &opts, getArgsCopy(cmd)) + args, err := flagutil.ParseArgsCleaned(command.Build, &opts, flagutil.GetArgsCopy(cmd)) if err != nil { return err } + if len(args) < 1 { - return errors.Wrap(ErrUnableToDetermineHash, "missing BUILD arg") - } - targetName := args[0] - if strings.Contains(targetName, "$") { - return errors.Wrap(ErrUnableToDetermineHash, "unable to handle arg in BUILD") + return errors.New("missing BUILD arg") } + if requiresCrossProduct(args) { - return errors.Wrap(ErrUnableToDetermineHash, "unable to cross-product in BUILD") + return errors.New("unable to cross-product in BUILD") } - return l.loadTargetFromString(ctx, targetName) + + return l.loadTargetFromString(ctx, args[0], args[1:], opts.PassArgs) } func (l *loader) handleCopy(ctx context.Context, cmd spec.Command) error { opts := commandflag.CopyOpts{} - - args, err := parseArgs(command.Copy, &opts, getArgsCopy(cmd)) + args, err := flagutil.ParseArgsCleaned(command.Copy, &opts, flagutil.GetArgsCopy(cmd)) if err != nil { return err } if opts.From != "" { - return errors.Wrap(ErrUnableToDetermineHash, "COPY --from is not supported") + return errors.New("COPY --from is not supported") } if len(args) < 2 { - return errors.Wrap(ErrUnableToDetermineHash, "COPY must include a source and destination") - } - - if argsContainsStr(args, "$") { - return errors.Wrap(ErrUnableToDetermineHash, "unable to handle COPY with arg") + return errors.New("COPY must include a source and destination") } srcs := args[:len(args)-1] @@ -125,9 +118,34 @@ func (l *loader) handleCopy(ctx context.Context, cmd spec.Command) error { func (l *loader) handleCopySrc(ctx context.Context, src string, isDir bool) error { - artifactSrc, parseErr := domain.ParseArtifact(src) - if parseErr != nil { - // COPY classical (not from another target) + var ( + classical bool + artifactSrc domain.Artifact + extraArgs []string + err error + ) + + // Complex form with args: (+target --arg=1) + if flagutil.IsInParamsForm(src) { + var artifactName string + classical = false + artifactName, extraArgs, err = flagutil.ParseParams(src) + if err != nil { + return errors.Wrap(err, "failed to parse COPY params") + } + artifactSrc, err = domain.ParseArtifact(artifactName) + if err != nil { + return errors.Wrap(err, "failed to parse artifact") + } + } else { // Simpler form: '+target/artifact' or 'file/path' + artifactSrc, err = domain.ParseArtifact(src) + if err != nil { + classical = true + } + } + + // COPY classical (not from another target) + if classical { path := filepath.Join(l.target.GetLocalPath(), src) files, err := l.expandCopyFiles(path) if err != nil { @@ -136,19 +154,19 @@ func (l *loader) handleCopySrc(ctx context.Context, src string, isDir bool) erro sort.Strings(files) for _, file := range files { if err := l.hasher.HashFile(ctx, file); err != nil { - return errors.Wrapf(ErrUnableToDetermineHash, "failed to hash file %s: %s", path, err) + return errors.Wrapf(err, "failed to hash file %s", path) } } return nil } - // COPY from a different target + // Remote targets aren't supported. if artifactSrc.Target.IsRemote() { - return errors.Wrap(ErrUnableToDetermineHash, "unable to handle remote target") + return errors.New("unable to handle remote target") } targetName := artifactSrc.Target.LocalPath + "+" + artifactSrc.Target.Target - if err := l.loadTargetFromString(ctx, targetName); err != nil { + if err := l.loadTargetFromString(ctx, targetName, extraArgs, false); err != nil { return err } @@ -159,7 +177,7 @@ func (l *loader) handleCopySrc(ctx context.Context, src string, isDir bool) erro // nested files. The file names will then be used in our hash. func (l *loader) expandCopyFiles(src string) ([]string, error) { if strings.Contains(src, "**") { - return nil, errors.Wrap(ErrUnableToDetermineHash, "globstar (**) not supported") + return nil, errors.New("globstar (**) not supported") } if strings.Contains(src, "*") { @@ -216,30 +234,30 @@ func (l *loader) expandDirs(dirs ...string) ([]string, error) { return uniqStrs(ret), nil } -func uniqStrs(all []string) []string { - m := map[string]struct{}{} - for _, v := range all { - m[v] = struct{}{} - } +func (l *loader) expandArgs(ctx context.Context, args []string) ([]string, error) { ret := []string{} - for k := range m { - ret = append(ret, k) + for _, arg := range args { + expanded, err := l.varCollection.Expand(arg, func(cmd string) (string, error) { + return "", errors.New("shell-out is not supported") + }) + if err != nil { + return nil, err + } + ret = append(ret, expanded) } - return ret + return ret, nil } -func (l *loader) handlePipeline(ctx context.Context, cmd spec.Command) error { - opts := commandflag.PipelineOpts{} - _, err := parseArgs(command.Copy, &opts, getArgsCopy(cmd)) +func (l *loader) handleCommand(ctx context.Context, cmd spec.Command) error { + // All commands are expanded and hashed at a minimum. + var err error + cmd.Args, err = l.expandArgs(ctx, cmd.Args) if err != nil { return err } - l.isPipeline = !opts.NoPipelineCache - return nil -} - -func (l *loader) handleCommand(ctx context.Context, cmd spec.Command) error { l.hashCommand(cmd) + + // Some commands require more processing. switch cmd.Name { case command.From: return l.handleFrom(ctx, cmd) @@ -247,44 +265,42 @@ func (l *loader) handleCommand(ctx context.Context, cmd spec.Command) error { return l.handleBuild(ctx, cmd) case command.Copy: return l.handleCopy(ctx, cmd) - case command.Pipeline: - return l.handlePipeline(ctx, cmd) - case command.SaveImage: - return l.handleSaveImage(ctx, cmd) - case command.Run: - return l.handleRun(ctx, cmd) case command.Arg: return l.handleArg(ctx, cmd) - case command.SaveArtifact: - return l.handleSaveArtifact(ctx, cmd) default: - return errors.Errorf("unhandled command: %s", cmd.Name) + return nil } } -func (l *loader) handleSaveImage(ctx context.Context, cmd spec.Command) error { - l.hashCommand(cmd) - return nil -} +func (l *loader) handleArg(ctx context.Context, cmd spec.Command) error { + opts, key, valueOrNil, err := flagutil.ParseArgArgs(ctx, cmd, l.isBaseTarget, l.features.ExplicitGlobal) + if err != nil { + return errors.Wrap(err, "failed to parse ARG args") + } -func (l *loader) handleSaveArtifact(ctx context.Context, cmd spec.Command) error { - l.hashCommand(cmd) - return nil -} + declOpts := []variables.DeclareOpt{ + variables.AsArg(), + } -func (l *loader) handleRun(ctx context.Context, cmd spec.Command) error { - l.hashCommand(cmd) - return nil -} + if valueOrNil != nil { + declOpts = append(declOpts, variables.WithValue(*valueOrNil)) + } + + if opts.Global { + declOpts = append(declOpts, variables.AsGlobal()) + } + + _, _, err = l.varCollection.DeclareVar(key, declOpts...) + if err != nil { + return errors.Wrap(err, "failed to declare variable") + } -func (l *loader) handleArg(ctx context.Context, cmd spec.Command) error { - l.hashCommand(cmd) return nil } func (l *loader) handleWith(ctx context.Context, with spec.WithStatement) error { if with.Command.Name != command.Docker { - return errors.Wrap(ErrUnableToDetermineHash, "expected WITH DOCKER") + return errors.New("expected WITH DOCKER") } err := l.handleWithDocker(ctx, with.Command) if err != nil { @@ -294,24 +310,24 @@ func (l *loader) handleWith(ctx context.Context, with spec.WithStatement) error } func (l *loader) handleWithDocker(ctx context.Context, cmd spec.Command) error { - l.hashCommand(cmd) // special case since handleWithDocker doesn't get called from handleCommand + // Special case since handleWithDocker doesn't get called from handleCommand. + var err error + cmd.Args, err = l.expandArgs(ctx, cmd.Args) + if err != nil { + return err + } + l.hashCommand(cmd) opts := commandflag.WithDockerOpts{} - _, err := parseArgs("WITH DOCKER", &opts, getArgsCopy(cmd)) + _, err = flagutil.ParseArgsCleaned("WITH DOCKER", &opts, flagutil.GetArgsCopy(cmd)) if err != nil { - return errors.Wrap(ErrUnableToDetermineHash, "failed to parse WITH DOCKER flags") + return errors.New("failed to parse WITH DOCKER flags") } for _, load := range opts.Loads { - if strings.Contains(load, "$") { - return errors.Wrap(ErrUnableToDetermineHash, "unable to handle arg in WITH DOCKER --load") - } _, target, extraArgs, err := earthfile2llb.ParseLoad(load) if err != nil { return errors.Wrap(err, "failed to parse --load value") } - if len(extraArgs) > 0 { - return errors.Wrap(ErrUnableToDetermineHash, "--load args are not yet supported") - } - err = l.loadTargetFromString(ctx, target) + err = l.loadTargetFromString(ctx, target, extraArgs, false) if err != nil { return err } @@ -358,7 +374,7 @@ func (l *loader) handleWait(ctx context.Context, waitStmt spec.WaitStatement) er } func (l *loader) handleTry(ctx context.Context, tryStmt spec.TryStatement) error { - return errors.Wrap(ErrUnableToDetermineHash, "try not supported") + return errors.New("try not supported") } func (l *loader) handleStatement(ctx context.Context, stmt spec.Statement) error { @@ -380,7 +396,7 @@ func (l *loader) handleStatement(ctx context.Context, stmt spec.Statement) error if stmt.Try != nil { return l.handleTry(ctx, *stmt.Try) } - return errors.Wrap(ErrUnableToDetermineHash, "unexpected statement type") + return errors.New("unexpected statement type") } func (l *loader) loadBlock(ctx context.Context, b spec.Block) error { @@ -393,94 +409,84 @@ func (l *loader) loadBlock(ctx context.Context, b spec.Block) error { return nil } -func (l *loader) hashIfStatement(s spec.IfStatement) { - l.hasher.HashString("IF") - l.hasher.HashJSONMarshalled(s.Expression) - l.hasher.HashBool(s.ExecMode) - l.hasher.HashInt(len(s.IfBody)) - l.hasher.HashInt(len(s.ElseIf)) - if s.ElseBody != nil { - l.hasher.HashInt(len(*s.ElseBody)) - } -} - -func (l *loader) hashElseIf(e spec.ElseIf) { - l.hasher.HashString("ELSE IF") - l.hasher.HashJSONMarshalled(e.Expression) - l.hasher.HashBool(e.ExecMode) - l.hasher.HashInt(len(e.Body)) -} - -func (l *loader) hashWaitStatement(w spec.WaitStatement) { - l.hasher.HashString("WAIT") - l.hasher.HashInt(len(w.Body)) - l.hasher.HashJSONMarshalled(w.Args) -} +func (l *loader) forTarget(ctx context.Context, target domain.Target, args []string, passArgs bool) (*loader, error) { + fullTargetName := target.String() -func (l *loader) hashVersion(v spec.Version) { - l.hasher.HashString("VERSION") - l.hasher.HashJSONMarshalled(v.Args) -} + visited := copyVisited(l.visited) + visited[fullTargetName] = struct{}{} -func (l *loader) hashCommand(c spec.Command) { - l.hasher.HashString(c.Name) - l.hasher.HashJSONMarshalled(c.Args) - l.hasher.HashBool(c.ExecMode) -} + flagArgs, err := variables.ParseFlagArgs(args) + if err != nil { + return nil, err + } -func (l *loader) hashFor(f spec.ForStatement) { - l.hasher.HashString("FOR") - l.hasher.HashJSONMarshalled(f.Args) -} + overriding, err := variables.ParseCommandLineArgs(flagArgs) + if err != nil { + return nil, err + } -func copyVisited(m map[string]struct{}) map[string]struct{} { - m2 := map[string]struct{}{} - for k := range m { - m2[k] = struct{}{} + if passArgs { + overriding = variables.CombineScopes(overriding, l.overridingVars) } - return m2 + + return &loader{ + conslog: l.conslog, + target: target, + visited: visited, + hasher: l.hasher, + isBaseTarget: target.Target == "base", + ci: l.ci, + push: l.push, + builtinArgs: l.builtinArgs, + overridingVars: overriding, + earthlyCIRunner: l.earthlyCIRunner, + }, nil } -func (l *loader) loadTargetFromString(ctx context.Context, targetName string) error { +func (l *loader) loadTargetFromString(ctx context.Context, targetName string, args []string, passArgs bool) error { relTarget, err := domain.ParseTarget(targetName) if err != nil { return errors.Wrapf(err, "parse target name %s", targetName) } + targetRef, err := domain.JoinReferences(l.target, relTarget) if err != nil { return errors.Wrapf(err, "failed to join %s and %s", l.target, relTarget) } + target := targetRef.(domain.Target) fullTargetName := target.String() if fullTargetName == "" { return fmt.Errorf("missing target string") } + if _, exists := l.visited[fullTargetName]; exists { - // prevent infinite loops; the converter does a better job since it also looks at args and if conditions - return errors.Wrapf(ErrUnableToDetermineHash, "circular dependency detected; %s already called", fullTargetName) + // Prevent infinite loops; the converter does a better job since it also + // looks at args and if conditions. + return errors.Errorf("circular dependency detected; %s already called", fullTargetName) } - visited := copyVisited(l.visited) - visited[fullTargetName] = struct{}{} - loaderInst := &loader{ - conslog: l.conslog, - target: target, - visited: visited, - hasher: l.hasher, + + targetLoader, err := l.forTarget(ctx, target, args, passArgs) + if err != nil { + return errors.Wrapf(err, "failed to create loader for target %q", targetName) } - return loaderInst.load(ctx) + + return targetLoader.load(ctx) } func (l *loader) findProject(ctx context.Context) (org, project string, err error) { if l.target.IsRemote() { return "", "", ErrRemoteNotSupported } + resolver := buildcontext.NewResolver(nil, nil, l.conslog, "", "", "", 0, "") - bc, err := resolver.Resolve(ctx, nil, nil, l.target) + + buildCtx, err := resolver.Resolve(ctx, nil, nil, l.target) if err != nil { return "", "", err } - ef := bc.Earthfile + ef := buildCtx.Earthfile if ef.Version != nil { l.hashVersion(*ef.Version) } @@ -489,29 +495,47 @@ func (l *loader) findProject(ctx context.Context) (org, project string, err erro if stmt.Command != nil && stmt.Command.Name == command.Project { args := stmt.Command.Args if len(args) != 1 { - return "", "", errors.Wrapf(ErrUnableToDetermineHash, "failed to parse PROJECT command") + return "", "", errors.New("failed to parse PROJECT command") } parts := strings.Split(args[0], "/") if len(parts) != 2 { - return "", "", errors.Wrapf(ErrUnableToDetermineHash, "failed to parse PROJECT command") + return "", "", errors.New("failed to parse PROJECT command") } return parts[0], parts[1], nil } } - return "", "", errors.Wrapf(ErrUnableToDetermineHash, "PROJECT command missing") + + return "", "", errors.New("PROJECT command missing") } func (l *loader) load(ctx context.Context) error { if l.target.IsRemote() { return ErrRemoteNotSupported } + resolver := buildcontext.NewResolver(nil, nil, l.conslog, "", "", "", 0, "") - bc, err := resolver.Resolve(ctx, nil, nil, l.target) + + buildCtx, err := resolver.Resolve(ctx, nil, nil, l.target) if err != nil { return err } - ef := bc.Earthfile + l.features = buildCtx.Features + + collOpt := variables.NewCollectionOpt{ + Console: l.conslog, + Target: l.target, + CI: l.ci, + Push: l.push, + BuiltinArgs: l.builtinArgs, + OverridingVars: l.overridingVars, + EarthlyCIRunner: l.earthlyCIRunner, + GitMeta: buildCtx.GitMetadata, + Features: l.features, + } + l.varCollection = variables.NewCollection(collOpt) + + ef := buildCtx.Earthfile if ef.Version != nil { l.hashVersion(*ef.Version) } @@ -519,38 +543,12 @@ func (l *loader) load(ctx context.Context) error { if l.target.Target == "base" { return l.loadBlock(ctx, ef.BaseRecipe) } + for _, t := range ef.Targets { if t.Name == l.target.Target { return l.loadBlock(ctx, t.Recipe) } } - return fmt.Errorf("target %s not found", l.target.Target) -} - -type loader struct { - conslog conslogging.ConsoleLogger - target domain.Target - visited map[string]struct{} - hasher *hasher.Hasher - isPipeline bool -} - -func HashTarget(ctx context.Context, target domain.Target, conslog conslogging.ConsoleLogger) (org, project string, hash []byte, err error) { - loaderInst := &loader{ - conslog: conslog, - target: target, - visited: map[string]struct{}{}, - hasher: hasher.New(), - } - org, project, err = loaderInst.findProject(ctx) - if err != nil { - return "", "", nil, err - } - err = loaderInst.load(ctx) - if err != nil { - return "", "", nil, err - } - - return org, project, loaderInst.hasher.GetHash(), nil + return fmt.Errorf("target %s not found", l.target.Target) } diff --git a/inputgraph/loader_hashing.go b/inputgraph/loader_hashing.go new file mode 100644 index 00000000..79de24ca --- /dev/null +++ b/inputgraph/loader_hashing.go @@ -0,0 +1,43 @@ +package inputgraph + +import "github.com/earthly/earthly/ast/spec" + +func (l *loader) hashIfStatement(s spec.IfStatement) { + l.hasher.HashString("IF") + l.hasher.HashJSONMarshalled(s.Expression) + l.hasher.HashBool(s.ExecMode) + l.hasher.HashInt(len(s.IfBody)) + l.hasher.HashInt(len(s.ElseIf)) + if s.ElseBody != nil { + l.hasher.HashInt(len(*s.ElseBody)) + } +} + +func (l *loader) hashElseIf(e spec.ElseIf) { + l.hasher.HashString("ELSE IF") + l.hasher.HashJSONMarshalled(e.Expression) + l.hasher.HashBool(e.ExecMode) + l.hasher.HashInt(len(e.Body)) +} + +func (l *loader) hashWaitStatement(w spec.WaitStatement) { + l.hasher.HashString("WAIT") + l.hasher.HashInt(len(w.Body)) + l.hasher.HashJSONMarshalled(w.Args) +} + +func (l *loader) hashVersion(v spec.Version) { + l.hasher.HashString("VERSION") + l.hasher.HashJSONMarshalled(v.Args) +} + +func (l *loader) hashCommand(c spec.Command) { + l.hasher.HashString(c.Name) + l.hasher.HashJSONMarshalled(c.Args) + l.hasher.HashBool(c.ExecMode) +} + +func (l *loader) hashFor(f spec.ForStatement) { + l.hasher.HashString("FOR") + l.hasher.HashJSONMarshalled(f.Args) +} diff --git a/inputgraph/util.go b/inputgraph/util.go new file mode 100644 index 00000000..a806ce61 --- /dev/null +++ b/inputgraph/util.go @@ -0,0 +1,37 @@ +package inputgraph + +import ( + "strings" +) + +func requiresCrossProduct(args []string) bool { + seen := map[string]struct{}{} + for _, s := range args { + k := strings.SplitN(s, "=", 2)[0] + if _, found := seen[k]; found { + return true + } + seen[k] = struct{}{} + } + return false +} + +func copyVisited(m map[string]struct{}) map[string]struct{} { + m2 := map[string]struct{}{} + for k := range m { + m2[k] = struct{}{} + } + return m2 +} + +func uniqStrs(all []string) []string { + m := map[string]struct{}{} + for _, v := range all { + m[v] = struct{}{} + } + ret := []string{} + for k := range m { + ret = append(ret, k) + } + return ret +} diff --git a/tests/autoskip/Earthfile b/tests/autoskip/Earthfile index ac3e1243..f9500b2d 100644 --- a/tests/autoskip/Earthfile +++ b/tests/autoskip/Earthfile @@ -7,15 +7,19 @@ IMPORT .. AS tests WORKDIR /test test-all: - BUILD +test-auto-skip - BUILD +test-auto-skip-with-subdir - BUILD +test-auto-skip-requires-project - BUILD +test-auto-skip-wait - BUILD +test-auto-skip-if-else - BUILD +test-auto-skip-for-in - BUILD +test-auto-skip-copy-glob - -test-auto-skip: + BUILD +test-files + BUILD +test-with-subdir + BUILD +test-requires-project + BUILD +test-wait + BUILD +test-if-else + BUILD +test-for-in + BUILD +test-copy-glob + BUILD +test-expand-args + BUILD +test-build-args + BUILD +test-pass-args + BUILD +test-copy-target-args + +test-files: RUN echo hello > my-file DO --pass-args +RUN_EARTHLY_ARGS --earthfile=simple.earth --target=+mypipeline --output_contains="I was run" RUN if ! grep "SSB3YXMgcnVuCg" earthly.output >/dev/null; then echo "base64 encoded RUN echo command is missing from output" && exit 1; fi @@ -28,7 +32,7 @@ test-auto-skip: DO --pass-args +RUN_EARTHLY_ARGS --earthfile=simple.earth --target=+mypipeline --output_contains="I was run" RUN if ! grep "SSB3YXMgcnVuCg" earthly.output >/dev/null; then echo "base64 encoded RUN echo command is missing from output" && exit 1; fi -test-auto-skip-with-subdir: +test-with-subdir: COPY subdir.earth Earthfile RUN mkdir subdir COPY subdir/test.earth subdir/Earthfile @@ -40,43 +44,77 @@ test-auto-skip-with-subdir: DO --pass-args +RUN_EARTHLY_ARGS --target=+allpipe --output_contains="ba1f2511fc30423bdbb183fe33f3dd0f" DO --pass-args +RUN_EARTHLY_ARGS --target=+allpipe --output_does_not_contain="ba1f2511fc30423bdbb183fe33f3dd0f" --output_contains="target .* has already been run; exiting" -test-auto-skip-requires-project: +test-requires-project: RUN echo hello > my-file DO --pass-args +RUN_EARTHLY_ARGS --earthfile=no-project.earth --target=+no-project --output_contains="I was run" RUN if ! grep "PROJECT command missing" earthly.output >/dev/null; then echo "no warning displayed for missing PROJECT keyword" && exit 1; fi -test-auto-skip-wait: +test-wait: DO --pass-args +RUN_EARTHLY_ARGS --earthfile=wait.earth --target=+test --output_contains="not skipped" - DO --pass-args +RUN_EARTHLY_ARGS --earthfile=wait.earth --target=+test --output_contains="3deeaceb7e263a72114ebc2da62d9fecc87506f3" + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=wait.earth --target=+test --output_contains="target .* has already been run; exiting" -test-auto-skip-if-else: +test-if-else: DO --pass-args +RUN_EARTHLY_ARGS --earthfile=if-else.earth --target=+test --output_contains="condition ok" - DO --pass-args +RUN_EARTHLY_ARGS --earthfile=if-else.earth --target=+test --output_contains="26ac9a5c6e4893049c218da18decfba3caff1746" + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=if-else.earth --target=+test --output_contains="target .* has already been run; exiting" -test-auto-skip-for-in: +test-for-in: DO --pass-args +RUN_EARTHLY_ARGS --earthfile=for.earth --target=+test --output_contains="hello 3" - DO --pass-args +RUN_EARTHLY_ARGS --earthfile=for.earth --target=+test --output_contains="ac33a31a36f496ed219293301bb44513a3a52d28" + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=for.earth --target=+test --output_contains="target .* has already been run; exiting" -test-auto-skip-copy-glob: +test-copy-glob: DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --should_fail=true --target=+globstar --output_contains="globstar (\*\*) not supported" COPY --dir glob . DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+dir --output_contains="glob/subdir/hello.txt" - DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+dir --output_contains="7dc88536e40a00684aeeb5ef3d6621d1e867aab2" + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+dir --output_contains="target .* has already been run; exiting" DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob --output_contains="glob/subdir/hello.txt" - DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob --output_contains="9a45ff884f09e576caf91fed3d3a209b109c68e4" + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob --output_contains="target .* has already been run; exiting" DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob-mid-path --output_contains="glob/subdir/hello.txt" - DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob-mid-path --output_contains="cec86ff354a3064adda819695eb91db965bf0b3e" + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob-mid-path --output_contains="target .* has already been run; exiting" DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob-dir --output_contains="glob/subdir/hello.txt" - DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob-dir --output_contains="111c0b03e1c36e9e55740cade50719e4c6e4e0ec" + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob-dir --output_contains="target .* has already been run; exiting" RUN echo data > glob/subdir/new.txt DO --pass-args +RUN_EARTHLY_ARGS --earthfile=copy-glob.earth --target=+glob-dir --output_contains="glob/subdir/new.txt" +test-expand-args: + COPY glob/hello.txt . + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=expand-args.earth --target=+basic --output_contains="COPY hello.txt" + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=expand-args.earth --target=+basic --output_contains="target .* has already been run; exiting" + + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=expand-args.earth --target=+dynamic-build --output_contains="dynamic target ok" + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=expand-args.earth --target=+dynamic-build --output_contains="target .* has already been run; exiting" + + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=expand-args.earth --target=+dynamic-arg --output_contains="hello bar" + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=expand-args.earth --target=+dynamic-arg --output_contains="target .* has already been run; exiting" + +test-build-args: + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=expand-args.earth --target=+build-args --output_contains="hello 3 4" + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=expand-args.earth --target=+build-args --output_contains="target .* has already been run; exiting" + + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=expand-args.earth --target=+build-args-2 --post_command="--foo=5 --bar=6" --output_contains="hello 5 6" + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=expand-args.earth --target=+build-args-2 --post_command="--foo=5 --bar=6" --output_contains="target .* has already been run; exiting" + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=expand-args.earth --target=+build-args-2 --post_command="--foo=5 --bar=7" --output_does_not_contain="target .* has already been run; exiting" + +test-pass-args: + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=expand-args.earth --target=+pass-args --post_command="--foo=3 --bar=4" --output_contains="hello 3 4" + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=expand-args.earth --target=+pass-args --post_command="--foo=3 --bar=4" --output_contains="target .* has already been run; exiting" + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=expand-args.earth --target=+pass-args --post_command="--foo=3 --bar=5" --output_does_not_contain="target .* has already been run; exiting" + +test-copy-target-args: + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=expand-args.earth --target=+copy-target-args --output_contains="+copy-target-args | hello" + DO --pass-args +RUN_EARTHLY_ARGS --earthfile=expand-args.earth --target=+copy-target-args --output_contains="target .* has already been run; exiting" + + RUN sed -i s/--foo=hello/--foo=changed/g Earthfile + + DO --pass-args +RUN_EARTHLY_ARGS --target=+copy-target-args --output_contains="+copy-target-args | changed" + DO --pass-args +RUN_EARTHLY_ARGS --target=+copy-target-args --output_contains="target .* has already been run; exiting" + + RUN_EARTHLY_ARGS: COMMAND ARG earthfile @@ -84,10 +122,12 @@ RUN_EARTHLY_ARGS: ARG should_fail=false ARG output_contains ARG output_does_not_contain + ARG post_command DO --pass-args tests+RUN_EARTHLY \ --earthfile=$earthfile \ --target=$target \ --should_fail=$should_fail \ --output_contains=$output_contains \ --output_does_not_contain=$output_does_not_contain \ - --extra_args="--auto-skip --auto-skip-db-path=test.db" + --extra_args="--auto-skip --auto-skip-db-path=test.db $extra_args" \ + --post_command=$post_command diff --git a/tests/autoskip/expand-args.earth b/tests/autoskip/expand-args.earth new file mode 100644 index 00000000..1cddc1b3 --- /dev/null +++ b/tests/autoskip/expand-args.earth @@ -0,0 +1,60 @@ +VERSION --pass-args 0.7 + +PROJECT earthly-technologies/core + +FROM alpine + +foo: + RUN echo "dynamic target ok" + +basic: + ARG ext=txt + COPY hello.$ext /tmp + RUN ls -l /tmp + +dynamic-build: + ARG target=+foo + BUILD $target + +dynamic-arg: + ARG foo=bar + ARG baz=$foo + RUN echo "hello $baz" + +other-target: + ARG foo=1 + ARG bar=2 + RUN echo "hello $foo $bar" + +build-args: + BUILD +other-target --foo=3 --bar=4 + +build-args-2: + ARG foo + ARG bar + BUILD +other-target --foo=$foo --bar=$bar + +pass-args-target: + ARG foo=1 + ARG bar=2 + RUN echo "hello $foo $bar" + +pass-args: + BUILD --pass-args +pass-args-target --baz=1 + +target-2: + ARG foo + ARG bar + RUN echo "hello $foo $bar" + +build-args-flag: + BUILD +target-2 --build-flags + +copy-target: + ARG foo + RUN echo $foo > x + SAVE ARTIFACT x + +copy-target-args: + COPY (+copy-target/x --foo=hello) . + RUN cat x diff --git a/util/flagutil/parse.go b/util/flagutil/parse.go index 724016ff..55e053a9 100644 --- a/util/flagutil/parse.go +++ b/util/flagutil/parse.go @@ -1,10 +1,14 @@ package flagutil import ( + "context" "fmt" "os" "strings" + "github.com/earthly/earthly/ast/commandflag" + "github.com/earthly/earthly/ast/spec" + "github.com/earthly/earthly/util/stringutil" "github.com/pkg/errors" "github.com/jessevdk/go-flags" @@ -27,6 +31,16 @@ func ParseArgs(command string, data interface{}, args []string) ([]string, error ) } +func ParseArgsCleaned(cmdName string, opts interface{}, args []string) ([]string, error) { + processed := stringutil.ProcessParamsAndQuotes(args) + return ParseArgs(cmdName, opts, processed) +} + +func ParseArgsWithValueModifierCleaned(cmdName string, opts interface{}, args []string, argumentModFunc ArgumentModFunc) ([]string, error) { + processed := stringutil.ProcessParamsAndQuotes(args) + return ParseArgsWithValueModifier(cmdName, opts, processed, argumentModFunc) +} + // ParseArgsWithValueModifier parses flags and args from a command string; it accepts an optional argumentModFunc // which is called before each flag value is parsed, and allows one to change the value. // if the flag value @@ -72,3 +86,113 @@ func SplitFlagString(value cli.StringSlice) []string { return r == ' ' || r == ',' }) } + +var ErrInvalidSyntax = errors.New("invalid syntax") +var ErrRequiredArgHasDefault = errors.New("required ARG cannot have a default value") +var ErrGlobalArgNotInBase = errors.New("global ARG can only be set in the base target") + +// ParseArgArgs parses the ARG command's arguments +// and returns the argOpts, key, value (or nil if missing), or error +func ParseArgArgs(ctx context.Context, cmd spec.Command, isBaseTarget bool, explicitGlobalFeature bool) (commandflag.ArgOpts, string, *string, error) { + var opts commandflag.ArgOpts + args, err := ParseArgsCleaned("ARG", &opts, GetArgsCopy(cmd)) + if err != nil { + return commandflag.ArgOpts{}, "", nil, err + } + if opts.Global { + // since the global flag is part of the struct, we need to manually return parsing error if it's used while the feature flag is off + if !explicitGlobalFeature { + return commandflag.ArgOpts{}, "", nil, errors.New("unknown flag --global") + } + // global flag can only bet set on base targets + if !isBaseTarget { + return commandflag.ArgOpts{}, "", nil, ErrGlobalArgNotInBase + } + } else if !explicitGlobalFeature { + // if the feature flag is off, all base target args are considered global + opts.Global = isBaseTarget + } + switch len(args) { + case 3: + if args[1] != "=" { + return commandflag.ArgOpts{}, "", nil, ErrInvalidSyntax + } + if opts.Required { + return commandflag.ArgOpts{}, "", nil, ErrRequiredArgHasDefault + } + return opts, args[0], &args[2], nil + case 1: + return opts, args[0], nil, nil + default: + return commandflag.ArgOpts{}, "", nil, ErrInvalidSyntax + } +} + +func GetArgsCopy(cmd spec.Command) []string { + argsCopy := make([]string, len(cmd.Args)) + copy(argsCopy, cmd.Args) + return argsCopy +} + +func IsInParamsForm(str string) bool { + return (strings.HasPrefix(str, "\"(") && strings.HasSuffix(str, "\")")) || + (strings.HasPrefix(str, "(") && strings.HasSuffix(str, ")")) +} + +// parseParams turns "(+target --flag=something)" into "+target" and []string{"--flag=something"}, +// or "\"(+target --flag=something)\"" into "+target" and []string{"--flag=something"} +func ParseParams(str string) (string, []string, error) { + if !IsInParamsForm(str) { + return "", nil, errors.New("params atom not in ( ... )") + } + if strings.HasPrefix(str, "\"(") { + str = str[2 : len(str)-2] // remove \"( and )\" + } else { + str = str[1 : len(str)-1] // remove ( and ) + } + var parts []string + var part []rune + nextEscaped := false + inQuotes := false + for _, char := range str { + switch char { + case '"': + if !nextEscaped { + inQuotes = !inQuotes + } + nextEscaped = false + case '\\': + nextEscaped = true + case ' ', '\t', '\n': + if !inQuotes && !nextEscaped { + if len(part) > 0 { + parts = append(parts, string(part)) + part = []rune{} + nextEscaped = false + continue + } else { + nextEscaped = false + continue + } + } + nextEscaped = false + default: + nextEscaped = false + } + part = append(part, char) + } + if nextEscaped { + return "", nil, errors.New("unterminated escape sequence") + } + if inQuotes { + return "", nil, errors.New("no ending quotes") + } + if len(part) > 0 { + parts = append(parts, string(part)) + } + + if len(parts) < 1 { + return "", nil, errors.New("invalid empty params") + } + return parts[0], parts[1:], nil +} diff --git a/util/flagutil/parse_test.go b/util/flagutil/parse_test.go index 2a5db490..2b2cc5c3 100644 --- a/util/flagutil/parse_test.go +++ b/util/flagutil/parse_test.go @@ -1,9 +1,11 @@ package flagutil import ( - "github.com/urfave/cli/v2" "reflect" "testing" + + "github.com/stretchr/testify/assert" + "github.com/urfave/cli/v2" ) func TestSplitFlagString(t *testing.T) { @@ -45,3 +47,48 @@ func TestSplitFlagString(t *testing.T) { }) } } + +func TestParseParams(t *testing.T) { + var tests = []struct { + in string + first string + args []string + }{ + {"(+target/art --flag=something)", "+target/art", []string{"--flag=something"}}, + {"(+target/art --flag=something\"\")", "+target/art", []string{"--flag=something\"\""}}, + {"( \n +target/art \t \n --flag=something\t )", "+target/art", []string{"--flag=something"}}, + {"(+target/art --flag=something\\ --another=something)", "+target/art", []string{"--flag=something\\ --another=something"}}, + {"(+target/art --flag=something --another=something)", "+target/art", []string{"--flag=something", "--another=something"}}, + {"(+target/art --flag=\"something in quotes\")", "+target/art", []string{"--flag=\"something in quotes\""}}, + {"(+target/art --flag=\\\"something --not=in-quotes\\\")", "+target/art", []string{"--flag=\\\"something", "--not=in-quotes\\\""}}, + {"(+target/art --flag=look-ma-a-\\))", "+target/art", []string{"--flag=look-ma-a-\\)"}}, + } + + for _, tt := range tests { + t.Run(tt.in, func(t *testing.T) { + actualFirst, actualArgs, err := ParseParams(tt.in) + assert.NoError(t, err) + assert.Equal(t, tt.first, actualFirst) + assert.Equal(t, tt.args, actualArgs) + }) + + } +} + +func TestNegativeParseParams(t *testing.T) { + var tests = []struct { + in string + }{ + {"+target/art --flag=something)"}, + {"(+target/art --flag=something"}, + {"(+target/art --flag=\"something)"}, + {"(+target/art --flag=something\\)"}, + {"()"}, + {"( \t\n )"}, + } + + for _, tt := range tests { + _, _, err := ParseParams(tt.in) + assert.Error(t, err) + } +} diff --git a/variables/builtin.go b/variables/builtin.go index e68eadc5..e5135ba3 100644 --- a/variables/builtin.go +++ b/variables/builtin.go @@ -34,11 +34,15 @@ func BuiltinArgs(target domain.Target, platr *platutil.Resolver, gitMeta *gituti ret.Add(arg.EarthlyTargetName, target.Target) setTargetTag(ret, target, gitMeta) - SetPlatformArgs(ret, platr) - setUserPlatformArgs(ret, platr) - if ftrs.NewPlatform { - setNativePlatformArgs(ret, platr) + + if platr != nil { + SetPlatformArgs(ret, platr) + setUserPlatformArgs(ret, platr) + if ftrs.NewPlatform { + setNativePlatformArgs(ret, platr) + } } + if ftrs.WaitBlock { ret.Add(arg.EarthlyPush, fmt.Sprintf("%t", push)) } diff --git a/variables/collection.go b/variables/collection.go index f2182481..cfc9883b 100644 --- a/variables/collection.go +++ b/variables/collection.go @@ -481,5 +481,6 @@ func (c *Collection) effective() *Scope { if c.effectiveCache == nil { c.effectiveCache = CombineScopes(c.vars(), c.overriding(), c.builtin, c.args(), c.envs, c.globals()) } + return c.effectiveCache }