From 4447d9338740483bcd2b8dbda1de4daa19237dce Mon Sep 17 00:00:00 2001 From: Srevin Saju Date: Thu, 22 Jun 2023 00:57:11 +0530 Subject: [PATCH 1/6] feat: add docker ports support --- pkg/blocks/data/git.go | 1 + pkg/ci/stage_containers_hcl.go | 42 ++++++++++++++++++++++++++ pkg/ci/stage_run.go | 54 ++++++++++++++++++++++------------ pkg/ci/stage_schema.go | 13 ++++++-- 4 files changed, 90 insertions(+), 20 deletions(-) create mode 100644 pkg/ci/stage_containers_hcl.go diff --git a/pkg/blocks/data/git.go b/pkg/blocks/data/git.go index 5d8b4f5..aecf30e 100644 --- a/pkg/blocks/data/git.go +++ b/pkg/blocks/data/git.go @@ -304,6 +304,7 @@ func (e *GitProvider) Attributes(ctx context.Context) map[string]cty.Value { fmt.Println(e.cfg.repo) var s storage.Storer var authMethod transport.AuthMethod + if e.cfg.auth.password != "" { authMethod = &http.BasicAuth{ Username: e.cfg.auth.username, diff --git a/pkg/ci/stage_containers_hcl.go b/pkg/ci/stage_containers_hcl.go new file mode 100644 index 0000000..93e4f63 --- /dev/null +++ b/pkg/ci/stage_containers_hcl.go @@ -0,0 +1,42 @@ +package ci + +import ( + "github.com/docker/go-connections/nat" + "github.com/hashicorp/hcl/v2" + "github.com/zclconf/go-cty/cty" +) + +// Nat returns a map of exposed ports and a map of port bindings +// after parsing the HCL configuration from StageContainerPorts. +func (s StageContainerPorts) Nat(evalCtx *hcl.EvalContext) (map[nat.Port]struct{}, map[nat.Port][]nat.PortBinding, hcl.Diagnostics) { + var hclDiags hcl.Diagnostics + var rawPortSpecs []string + for _, port := range s { + p, d := port.Port.Value(evalCtx) + hclDiags = hclDiags.Extend(d) + if d.HasErrors() { + continue + } + if p.Type() != cty.String { + hclDiags = hclDiags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid port specification", + Detail: "Port specification must be a string", + Subject: port.Port.Range().Ptr(), + }) + continue + } + + rawPortSpecs = append(rawPortSpecs, p.AsString()) + } + + exposedPorts, bindings, err := nat.ParsePortSpecs(rawPortSpecs) + if err != nil { + hclDiags = hclDiags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid port specification", + Detail: err.Error(), + }) + } + return exposedPorts, bindings, hclDiags +} diff --git a/pkg/ci/stage_run.go b/pkg/ci/stage_run.go index d97854d..e9351f3 100644 --- a/pkg/ci/stage_run.go +++ b/pkg/ci/stage_run.go @@ -265,7 +265,7 @@ func (s *Stage) Run(ctx context.Context) diag.Diagnostics { } } - oldParam, ok := evalCtx.Variables["param"] + oldParam, ok := evalCtx.Variables[ParamBlock] if ok { oldParamMap := oldParam.AsValueMap() for k, v := range oldParamMap { @@ -274,17 +274,16 @@ func (s *Stage) Run(ctx context.Context) diag.Diagnostics { } evalCtx = evalCtx.NewChild() evalCtx.Variables = map[string]cty.Value{ - "this": cty.ObjectVal(map[string]cty.Value{ + ThisBlock: cty.ObjectVal(map[string]cty.Value{ "name": cty.StringVal(s.Name), "id": cty.StringVal(s.Id), }), - "param": cty.ObjectVal(paramsGo), + ParamBlock: cty.ObjectVal(paramsGo), } script, d := s.Script.Value(evalCtx) if d.HasErrors() && isDryRun { script = cty.StringVal(ui.Italic(ui.Yellow("(will be evaluated later)"))) - } else { hclDiags = hclDiags.Extend(d) } @@ -334,7 +333,6 @@ func (s *Stage) Run(ctx context.Context) diag.Diagnostics { if shell == "" { shell = "bash" } - runCommand := shell // emptyCommands - specifies if both args and scripts were unset @@ -384,19 +382,33 @@ func (s *Stage) Run(ctx context.Context) diag.Diagnostics { cmd.Dir = dir if s.Container == nil { - s.process = cmd - logger.Trace("running command:", cmd.String()) - if !isDryRun { err = cmd.Run() } else { fmt.Println(cmd.String()) } - } else { logger := logger.WithField("🐳", "") + + imageRaw, d := s.Container.Image.Value(evalCtx) + if d.HasErrors() { + hclDiags = hclDiags.Extend(d) + } else if imageRaw.Type() != cty.String { + hclDiags = hclDiags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "image must be a string", + Detail: fmt.Sprintf("the provided image, was not recognized as a valid string. received image='''%s'''", imageRaw), + Subject: s.Container.Image.Range().Ptr(), + EvalContext: evalCtx, + }) + } + if hclDiags.HasErrors() { + return diags.ExtendHCLDiagnostics(hclDiags, hclDgWriter, s.Identifier()) + } + image := imageRaw.AsString() + cli, err := dockerClient.NewClientWithOpts(dockerClient.FromEnv, dockerClient.WithAPIVersionNegotiation()) if err != nil { return diags.Append(diag.Diagnostic{ @@ -408,11 +420,11 @@ func (s *Stage) Run(ctx context.Context) diag.Diagnostics { } defer cli.Close() // check if image exists - logger.Debugf("checking if image %s exists", s.Container.Image) - _, _, err = cli.ImageInspectWithRaw(ctx, s.Container.Image) + logger.Debugf("checking if image %s exists", image) + _, _, err = cli.ImageInspectWithRaw(ctx, image) if err != nil { - logger.Infof("image %s does not exist, pulling...", s.Container.Image) - reader, err := cli.ImagePull(ctx, s.Container.Image, types.ImagePullOptions{}) + logger.Infof("image %s does not exist, pulling...", image) + reader, err := cli.ImagePull(ctx, image, types.ImagePullOptions{}) if err != nil { return diags.Append(diag.Diagnostic{ Severity: diag.SeverityError, @@ -453,8 +465,14 @@ func (s *Stage) Run(ctx context.Context) diag.Diagnostics { } if !isDryRun { + exposedPorts, bindings, d := s.Container.Ports.Nat(evalCtx) + if d.HasErrors() { + hclDiags = hclDiags.Extend(d) + return diags.ExtendHCLDiagnostics(hclDiags, hclDgWriter, s.Identifier()) + } + resp, err := cli.ContainerCreate(ctx, &dockerContainer.Config{ - Image: s.Container.Image, + Image: image, Cmd: containerArgs, WorkingDir: "/workspace", Tty: true, @@ -464,9 +482,11 @@ func (s *Stage) Run(ctx context.Context) diag.Diagnostics { OpenStdin: s.Container.Stdin, StdinOnce: s.Container.Stdin, Env: envStrings, + ExposedPorts: exposedPorts, // User: s.Container.User, }, &dockerContainer.HostConfig{ - Binds: binds, + Binds: binds, + PortBindings: bindings, }, nil, nil, "") if err != nil { return diags.Append(diag.Diagnostic{ @@ -534,11 +554,9 @@ func (s *Stage) Run(ctx context.Context) diag.Diagnostics { return diags } } else { - fmt.Println(ui.Blue("docker:run.image"), ui.Green(s.Container.Image)) - + fmt.Println(ui.Blue("docker:run.image"), ui.Green(image)) fmt.Println(ui.Blue("docker:run.workdir"), ui.Green("/workspace")) fmt.Println(ui.Blue("docker:run.volume"), ui.Green(cmd.Dir+":/workspace")) - fmt.Println(ui.Blue("docker:run.env"), ui.Green(strings.Join(envStrings, " "))) fmt.Println(ui.Blue("docker:run.stdin"), ui.Green(s.Container.Stdin)) fmt.Println(ui.Blue("docker:run.args"), ui.Green(strings.Join(containerArgs, " "))) diff --git a/pkg/ci/stage_schema.go b/pkg/ci/stage_schema.go index 28b4126..66579d4 100644 --- a/pkg/ci/stage_schema.go +++ b/pkg/ci/stage_schema.go @@ -15,10 +15,19 @@ type StageContainerVolume struct { type StageContainerVolumes []StageContainerVolume +type StageContainerPort struct { + Hostname hcl.Expression `hcl:"host,optional" json:"host"` + ContainerPort hcl.Expression `hcl:"container_port" json:"container_port"` + Port hcl.Expression `hcl:"port" json:"port"` +} + +type StageContainerPorts []StageContainerPort + type StageContainer struct { - Image string `hcl:"image" json:"image"` + Image hcl.Expression `hcl:"image" json:"image"` Volumes StageContainerVolumes `hcl:"volume,block" json:"volumes"` - Ports []string `hcl:"ports,optional" json:"ports"` + Ports StageContainerPorts `hcl:"ports,optional" json:"ports"` + Host hcl.Expression `hcl:"host,optional" json:"host"` Stdin bool `hcl:"stdin,optional" json:"stdin"` } From 281b72aef437f7b23f99232d1d1f4e2c2e2426de Mon Sep 17 00:00:00 2001 From: Srevin Saju Date: Thu, 22 Jun 2023 02:30:42 +0530 Subject: [PATCH 2/6] refactor: move to hcl.Diagnostics instead of custom diagnostics to reduce complexity --- cmd/togomak/main.go | 2 +- pkg/blocks/data/env.go | 54 +++++----- pkg/blocks/data/git.go | 101 ++++++++---------- pkg/blocks/data/prompt.go | 50 +++++---- pkg/blocks/data/provider.go | 7 +- pkg/c/context.go | 2 + pkg/ci/data.go | 31 +++--- pkg/ci/data_hcl.go | 6 +- pkg/ci/data_retry.go | 10 +- pkg/ci/data_run.go | 64 +++++------ pkg/ci/locals.go | 30 +++--- pkg/ci/locals_run.go | 35 ++---- pkg/ci/macro.go | 16 +-- pkg/ci/macro_run.go | 22 ++-- pkg/ci/runnable.go | 109 ++++++++++--------- pkg/ci/stage.go | 11 +- pkg/ci/stage_run.go | 205 ++++++++++++++++-------------------- pkg/core/togomak.go | 1 + pkg/diag/diagnostics.go | 152 -------------------------- pkg/graph/depgraph.go | 21 ++-- pkg/orchestra/context.go | 10 +- pkg/orchestra/format.go | 8 +- pkg/orchestra/interrupt.go | 38 +++---- pkg/orchestra/orchestra.go | 57 ++++++---- 24 files changed, 426 insertions(+), 616 deletions(-) create mode 100644 pkg/core/togomak.go delete mode 100644 pkg/diag/diagnostics.go diff --git a/cmd/togomak/main.go b/cmd/togomak/main.go index a9b503f..d6e1995 100644 --- a/cmd/togomak/main.go +++ b/cmd/togomak/main.go @@ -208,7 +208,7 @@ func newConfigFromCliContext(ctx *cli.Context) orchestra.Config { func run(ctx *cli.Context) error { cfg := newConfigFromCliContext(ctx) - orchestra.Orchestra(cfg) + os.Exit(orchestra.Orchestra(cfg)) return nil } diff --git a/pkg/blocks/data/env.go b/pkg/blocks/data/env.go index 25d5014..8c86848 100644 --- a/pkg/blocks/data/env.go +++ b/pkg/blocks/data/env.go @@ -2,14 +2,17 @@ package data import ( "context" - "fmt" "github.com/hashicorp/hcl/v2" "github.com/srevinsaju/togomak/v1/pkg/c" - "github.com/srevinsaju/togomak/v1/pkg/diag" "github.com/zclconf/go-cty/cty" "os" ) +const ( + EnvProviderAttrKey = "key" + EnvProviderAttrDefault = "default" +) + type EnvProvider struct { initialized bool Key hcl.Expression `hcl:"key" json:"key"` @@ -45,11 +48,11 @@ func (e *EnvProvider) New() Provider { } } -func (e *EnvProvider) Attributes(ctx context.Context) map[string]cty.Value { +func (e *EnvProvider) Attributes(ctx context.Context) (map[string]cty.Value, hcl.Diagnostics) { return map[string]cty.Value{ - "key": cty.StringVal(e.keyParsed), - "default": cty.StringVal(e.def), - } + EnvProviderAttrKey: cty.StringVal(e.keyParsed), + EnvProviderAttrDefault: cty.StringVal(e.def), + }, nil } func (e *EnvProvider) Url() string { @@ -60,11 +63,11 @@ func (e *EnvProvider) Schema() *hcl.BodySchema { schema := &hcl.BodySchema{ Attributes: []hcl.AttributeSchema{ { - Name: "key", + Name: EnvProviderAttrKey, Required: true, }, { - Name: "default", + Name: EnvProviderAttrDefault, Required: false, }, }, @@ -73,48 +76,41 @@ func (e *EnvProvider) Schema() *hcl.BodySchema { } -func (e *EnvProvider) DecodeBody(body hcl.Body) diag.Diagnostics { +func (e *EnvProvider) DecodeBody(body hcl.Body) hcl.Diagnostics { if !e.initialized { panic("provider not initialized") } - var diags diag.Diagnostics - hclDiagWriter := e.ctx.Value(c.TogomakContextHclDiagWriter).(hcl.DiagnosticWriter) + var diags hcl.Diagnostics hclContext := e.ctx.Value(c.TogomakContextHclEval).(*hcl.EvalContext) schema := e.Schema() - content, hclDiags := body.Content(schema) - if hclDiags.HasErrors() { - source := fmt.Sprintf("data.%s:decodeBody", e.Name()) - diags = diags.NewHclWriteDiagnosticsError(source, hclDiagWriter.WriteDiagnostics(hclDiags)) - } + content, d := body.Content(schema) + diags = diags.Extend(d) + attr := content.Attributes["key"] var key cty.Value - key, hclDiags = attr.Expr.Value(hclContext) - if hclDiags.HasErrors() { - source := fmt.Sprintf("data.%s:decodeBody", e.Name()) - diags = diags.NewHclWriteDiagnosticsError(source, hclDiagWriter.WriteDiagnostics(hclDiags)) - } + key, d = attr.Expr.Value(hclContext) + diags = diags.Extend(d) + e.keyParsed = key.AsString() attr = content.Attributes["default"] - key, hclDiags = attr.Expr.Value(hclContext) - if hclDiags.HasErrors() { - source := fmt.Sprintf("data.%s:decodeBody", e.Name()) - diags = diags.NewHclWriteDiagnosticsError(source, hclDiagWriter.WriteDiagnostics(hclDiags)) - } + key, d = attr.Expr.Value(hclContext) + diags = diags.Extend(d) + e.def = key.AsString() return diags } -func (e *EnvProvider) Value(ctx context.Context, id string) string { +func (e *EnvProvider) Value(ctx context.Context, id string) (string, hcl.Diagnostics) { if !e.initialized { panic("provider not initialized") } v, exists := os.LookupEnv(e.keyParsed) if exists { - return v + return v, nil } - return e.def + return e.def, nil } diff --git a/pkg/blocks/data/git.go b/pkg/blocks/data/git.go index aecf30e..12ee8ca 100644 --- a/pkg/blocks/data/git.go +++ b/pkg/blocks/data/git.go @@ -14,7 +14,6 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/sirupsen/logrus" "github.com/srevinsaju/togomak/v1/pkg/c" - "github.com/srevinsaju/togomak/v1/pkg/diag" "github.com/srevinsaju/togomak/v1/pkg/ui" "github.com/zclconf/go-cty/cty" "io" @@ -93,89 +92,86 @@ func (e *GitProvider) Url() string { return "embedded::togomak.srev.in/providers/data/git" } -func (e *GitProvider) DecodeBody(body hcl.Body) diag.Diagnostics { +func (e *GitProvider) DecodeBody(body hcl.Body) hcl.Diagnostics { if !e.initialized { panic("provider not initialized") } - var diags diag.Diagnostics - hclDiagWriter := e.ctx.Value(c.TogomakContextHclDiagWriter).(hcl.DiagnosticWriter) + var diags hcl.Diagnostics hclContext := e.ctx.Value(c.TogomakContextHclEval).(*hcl.EvalContext) schema := e.Schema() - content, hclDiags := body.Content(schema) - if hclDiags.HasErrors() { - diags = diags.NewHclWriteDiagnosticsError(e.Identifier(), hclDiagWriter.WriteDiagnostics(hclDiags)) - } + content, d := body.Content(schema) + diags = diags.Extend(d) repo, d := content.Attributes[GitBlockArgumentUrl].Expr.Value(hclContext) - hclDiags = append(hclDiags, d...) + diags = diags.Extend(d) tagAttr, ok := content.Attributes[GitBlockArgumentTag] tag := cty.StringVal("") if ok { tag, d = tagAttr.Expr.Value(hclContext) - hclDiags = append(hclDiags, d...) + diags = diags.Extend(d) } branchAttr, ok := content.Attributes[GitBlockArgumentBranch] branch := cty.StringVal("") if ok { branch, d = branchAttr.Expr.Value(hclContext) - hclDiags = append(hclDiags, d...) + diags = diags.Extend(d) } commitAttr, ok := content.Attributes[GitBlockArgumentCommit] commit := cty.StringVal("") if ok { commit, d = commitAttr.Expr.Value(hclContext) - hclDiags = append(hclDiags, d...) + diags = diags.Extend(d) } destinationAttr, ok := content.Attributes[GitBlockArgumentDestination] destination := cty.StringVal("") if ok { destination, d = destinationAttr.Expr.Value(hclContext) - hclDiags = append(hclDiags, d...) + diags = diags.Extend(d) } depthAttr, ok := content.Attributes[GitBlockArgumentDepth] depth := cty.NumberIntVal(0) if ok { depth, d = depthAttr.Expr.Value(hclContext) - hclDiags = append(hclDiags, d...) + diags = diags.Extend(d) } caBundleAttr, ok := content.Attributes[GitBlockArgumentCaBundle] caBundle := cty.StringVal("") if ok { caBundle, d = caBundleAttr.Expr.Value(hclContext) - hclDiags = append(hclDiags, d...) + diags = diags.Extend(d) } filesAttr, ok := content.Attributes[GitBlockArgumentFiles] files := cty.ListValEmpty(cty.String) if ok { files, d = filesAttr.Expr.Value(hclContext) - hclDiags = append(hclDiags, d...) + diags = diags.Extend(d) } authBlock := content.Blocks.OfType(GitBlockArgumentAuth) var authConfig gitProviderAuthConfig if len(authBlock) == 1 { auth, d := content.Blocks.OfType(GitBlockArgumentAuth)[0].Body.Content(GitProviderAuthSchema()) - hclDiags = append(hclDiags, d...) + diags = diags.Extend(d) authUsername, d := auth.Attributes["username"].Expr.Value(hclContext) - hclDiags = append(hclDiags, d...) + diags = diags.Extend(d) authPassword, d := auth.Attributes["password"].Expr.Value(hclContext) - hclDiags = append(hclDiags, d...) + diags = diags.Extend(d) authSshPassword, d := auth.Attributes["ssh_password"].Expr.Value(hclContext) - hclDiags = append(hclDiags, d...) + diags = diags.Extend(d) authSshPrivateKey, d := auth.Attributes["ssh_private_key"].Expr.Value(hclContext) - hclDiags = append(hclDiags, d...) + diags = diags.Extend(d) authConfig = gitProviderAuthConfig{ username: authUsername.AsString(), @@ -185,10 +181,6 @@ func (e *GitProvider) DecodeBody(body hcl.Body) diag.Diagnostics { } } - if hclDiags.HasErrors() { - diags = diags.NewHclWriteDiagnosticsError(e.Identifier(), hclDiagWriter.WriteDiagnostics(hclDiags)) - } - depthInt, _ := depth.AsBigFloat().Int64() var f []string for _, file := range files.AsValueSlice() { @@ -207,7 +199,7 @@ func (e *GitProvider) DecodeBody(body hcl.Body) diag.Diagnostics { files: f, } - return nil + return diags } func (e *GitProvider) New() Provider { @@ -283,16 +275,16 @@ func (e *GitProvider) Initialized() bool { return e.initialized } -func (e *GitProvider) Value(ctx context.Context, id string) string { +func (e *GitProvider) Value(ctx context.Context, id string) (string, hcl.Diagnostics) { if !e.initialized { panic("provider not initialized") } - return "" + return "", nil } -func (e *GitProvider) Attributes(ctx context.Context) map[string]cty.Value { +func (e *GitProvider) Attributes(ctx context.Context) (map[string]cty.Value, hcl.Diagnostics) { logger := ctx.Value(c.TogomakContextLogger).(*logrus.Logger).WithField("provider", e.Name()) - var diags diag.Diagnostics + var diags hcl.Diagnostics if !e.initialized { panic("provider not initialized") } @@ -301,7 +293,6 @@ func (e *GitProvider) Attributes(ctx context.Context) map[string]cty.Value { // clone git repo // git clone - fmt.Println(e.cfg.repo) var s storage.Storer var authMethod transport.AuthMethod @@ -313,12 +304,12 @@ func (e *GitProvider) Attributes(ctx context.Context) map[string]cty.Value { } else if e.cfg.auth.isSsh { publicKeys, err := ssh.NewPublicKeys(e.cfg.auth.username, []byte(e.cfg.auth.sshPrivateKey), e.cfg.auth.sshPassword) if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, Summary: "ssh key error", Detail: err.Error(), }) - return nil + return nil, diags } authMethod = publicKeys } @@ -358,26 +349,22 @@ func (e *GitProvider) Attributes(ctx context.Context) map[string]cty.Value { cloneComplete <- true if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, Summary: "git clone failed", Detail: err.Error(), - Source: e.Identifier(), }) - diags.Write(logger.WriterLevel(logrus.ErrorLevel)) - return nil + return nil, diags } w, err := repo.Worktree() if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, Summary: "checkout failed", Detail: err.Error(), - Source: e.Identifier(), }) - diags.Write(logger.WriterLevel(logrus.ErrorLevel)) - return nil + return nil, diags } commitIter, err := repo.Log(&git.LogOptions{ @@ -387,11 +374,10 @@ func (e *GitProvider) Attributes(ctx context.Context) map[string]cty.Value { count := 0 lastTag := "" if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityWarning, + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, Summary: "git log failed", Detail: err.Error(), - Source: e.Identifier(), }) } else { for commit, err := commitIter.Next(); err != nil; { @@ -407,21 +393,19 @@ func (e *GitProvider) Attributes(ctx context.Context) map[string]cty.Value { for _, f := range e.cfg.files { _, err := w.Filesystem.Stat(f) if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, Summary: "git file search failed", Detail: err.Error(), - Source: e.Identifier(), }) continue } file, err := w.Filesystem.Open(f) if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, Summary: "git file open failed", Detail: err.Error(), - Source: e.Identifier(), }) continue } @@ -429,11 +413,10 @@ func (e *GitProvider) Attributes(ctx context.Context) map[string]cty.Value { data, err := io.ReadAll(file) if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, Summary: "git file read failed", Detail: err.Error(), - Source: e.Identifier(), }) continue } @@ -460,8 +443,8 @@ func (e *GitProvider) Attributes(ctx context.Context) map[string]cty.Value { sha := cty.StringVal("") if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityWarning, + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, Summary: "git head failed", Detail: err.Error(), }) @@ -489,5 +472,5 @@ func (e *GitProvider) Attributes(ctx context.Context) map[string]cty.Value { attrs[GitBlockAttrCommitsSinceLastTag] = commitsSinceLastTag // get the commit - return attrs + return attrs, diags } diff --git a/pkg/blocks/data/prompt.go b/pkg/blocks/data/prompt.go index d16216a..e569517 100644 --- a/pkg/blocks/data/prompt.go +++ b/pkg/blocks/data/prompt.go @@ -7,7 +7,6 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/sirupsen/logrus" "github.com/srevinsaju/togomak/v1/pkg/c" - "github.com/srevinsaju/togomak/v1/pkg/diag" "github.com/srevinsaju/togomak/v1/pkg/meta" "github.com/zclconf/go-cty/cty" "os" @@ -40,35 +39,28 @@ func (e *PromptProvider) Url() string { return "embedded::togomak.srev.in/providers/data/prompt" } -func (e *PromptProvider) DecodeBody(body hcl.Body) diag.Diagnostics { +func (e *PromptProvider) DecodeBody(body hcl.Body) hcl.Diagnostics { if !e.initialized { panic("provider not initialized") } - var diags diag.Diagnostics - hclDiagWriter := e.ctx.Value(c.TogomakContextHclDiagWriter).(hcl.DiagnosticWriter) + var diags hcl.Diagnostics + hclContext := e.ctx.Value(c.TogomakContextHclEval).(*hcl.EvalContext) schema := e.Schema() - content, hclDiags := body.Content(schema) - if hclDiags.HasErrors() { - source := fmt.Sprintf("data.%s:decodeBody", e.Name()) - diags = diags.NewHclWriteDiagnosticsError(source, hclDiagWriter.WriteDiagnostics(hclDiags)) - } + content, d := body.Content(schema) + diags = append(diags, d...) + attr := content.Attributes["prompt"] var key cty.Value - key, hclDiags = attr.Expr.Value(hclContext) - if hclDiags.HasErrors() { - source := fmt.Sprintf("data.%s:decodeBody", e.Name()) - diags = diags.NewHclWriteDiagnosticsError(source, hclDiagWriter.WriteDiagnostics(hclDiags)) - } + key, d = attr.Expr.Value(hclContext) + diags = append(diags, d...) + e.promptParsed = key.AsString() attr = content.Attributes["default"] - key, hclDiags = attr.Expr.Value(hclContext) - if hclDiags.HasErrors() { - source := fmt.Sprintf("data.%s:decodeBody", e.Name()) - diags = diags.NewHclWriteDiagnosticsError(source, hclDiagWriter.WriteDiagnostics(hclDiags)) - } + key, d = attr.Expr.Value(hclContext) + diags = append(diags, d...) e.def = key.AsString() return diags @@ -96,18 +88,19 @@ func (e *PromptProvider) Schema() *hcl.BodySchema { } } -func (e *PromptProvider) Attributes(ctx context.Context) map[string]cty.Value { +func (e *PromptProvider) Attributes(ctx context.Context) (map[string]cty.Value, hcl.Diagnostics) { return map[string]cty.Value{ "prompt": cty.StringVal(e.promptParsed), "default": cty.StringVal(e.def), - } + }, nil } func (e *PromptProvider) Initialized() bool { return e.initialized } -func (e *PromptProvider) Value(ctx context.Context, id string) string { +func (e *PromptProvider) Value(ctx context.Context, id string) (string, hcl.Diagnostics) { + var diags hcl.Diagnostics if !e.initialized { panic("provider not initialized") } @@ -120,11 +113,11 @@ func (e *PromptProvider) Value(ctx context.Context, id string) string { envExists, ok := os.LookupEnv(envVarName) if ok { logger.Debug("environment variable found, using that") - return envExists + return envExists, nil } if unattended { logger.Warn("--unattended/--ci mode enabled, falling back to default") - return e.def + return e.def, nil } prompt := e.promptParsed @@ -143,8 +136,13 @@ func (e *PromptProvider) Value(ctx context.Context, id string) string { err := survey.AskOne(&input, &resp) if err != nil { logger.Warn("unable to get value from prompt: ", err) - return e.def + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagWarning, + Summary: "unable to get value from prompt", + Detail: err.Error(), + }) + return e.def, diags } - return resp + return resp, diags } diff --git a/pkg/blocks/data/provider.go b/pkg/blocks/data/provider.go index e93b2a6..f5d23dd 100644 --- a/pkg/blocks/data/provider.go +++ b/pkg/blocks/data/provider.go @@ -3,7 +3,6 @@ package data import ( "context" "github.com/hashicorp/hcl/v2" - "github.com/srevinsaju/togomak/v1/pkg/diag" "github.com/zclconf/go-cty/cty" ) @@ -16,9 +15,9 @@ type Provider interface { New() Provider SetContext(context context.Context) - DecodeBody(body hcl.Body) diag.Diagnostics - Value(ctx context.Context, id string) string - Attributes(ctx context.Context) map[string]cty.Value + DecodeBody(body hcl.Body) hcl.Diagnostics + Value(ctx context.Context, id string) (string, hcl.Diagnostics) + Attributes(ctx context.Context) (map[string]cty.Value, hcl.Diagnostics) } func Variables(e Provider, body hcl.Body) []hcl.Traversal { diff --git a/pkg/c/context.go b/pkg/c/context.go index 594b603..f2179dd 100644 --- a/pkg/c/context.go +++ b/pkg/c/context.go @@ -30,4 +30,6 @@ const ( TogomakContextMutexStages = "mutex_stages" TogomakContextMutexData = "mutex_data" TogomakContextMutexMacro = "mutex_macro" + + Togomak = "togomak" ) diff --git a/pkg/ci/data.go b/pkg/ci/data.go index 773c02a..7374d02 100644 --- a/pkg/ci/data.go +++ b/pkg/ci/data.go @@ -1,48 +1,53 @@ package ci import ( - "fmt" - "github.com/srevinsaju/togomak/v1/pkg/diag" + "github.com/hashicorp/hcl/v2" ) const DataBlock = "data" -func (d Data) Description() string { +func (s Data) Description() string { return "" } -func (d Data) Identifier() string { - return d.Id +func (s Data) Identifier() string { + return s.Id } -func (d Datas) ById(provider string, id string) (*Data, error) { +func (d Datas) ById(provider string, id string) (*Data, hcl.Diagnostics) { for _, data := range d { if data.Id == id && data.Provider == provider { return &data, nil } } - return nil, fmt.Errorf("data block with id=%s and provider=%s not found", id, provider) + return nil, hcl.Diagnostics{ + { + Severity: hcl.DiagError, + Summary: "Data not found", + Detail: "Data with id " + id + " not found", + }, + } } -func (d Data) Type() string { +func (s Data) Type() string { return DataBlock } -func (d Data) Set(k any, v any) { +func (s Data) Set(k any, v any) { } -func (d Data) Get(k any) any { +func (s Data) Get(k any) any { return nil } -func (d Data) IsDaemon() bool { +func (s Data) IsDaemon() bool { return false } -func (d Data) Terminate() diag.Diagnostics { +func (s Data) Terminate() hcl.Diagnostics { return nil } -func (d Data) Kill() diag.Diagnostics { +func (s Data) Kill() hcl.Diagnostics { return nil } diff --git a/pkg/ci/data_hcl.go b/pkg/ci/data_hcl.go index fe9df11..69c72cd 100644 --- a/pkg/ci/data_hcl.go +++ b/pkg/ci/data_hcl.go @@ -5,15 +5,15 @@ import ( dataBlock "github.com/srevinsaju/togomak/v1/pkg/blocks/data" ) -func (d Data) Variables() []hcl.Traversal { +func (s Data) Variables() []hcl.Traversal { var traversal []hcl.Traversal - provider := dataBlock.DefaultProviders.Get(d.Provider) + provider := dataBlock.DefaultProviders.Get(s.Provider) // TODO: this will panic, if the provider is not found if provider == nil { panic("provider not found") } provide := provider.New() - traversal = append(traversal, dataBlock.Variables(provide, d.Body)...) + traversal = append(traversal, dataBlock.Variables(provide, s.Body)...) return traversal } diff --git a/pkg/ci/data_retry.go b/pkg/ci/data_retry.go index 9fc2cd1..305371a 100644 --- a/pkg/ci/data_retry.go +++ b/pkg/ci/data_retry.go @@ -1,21 +1,21 @@ package ci -func (d Data) CanRetry() bool { +func (s Data) CanRetry() bool { return false } -func (d Data) MaxRetries() int { +func (s Data) MaxRetries() int { return 0 } -func (d Data) MinRetryBackoff() int { +func (s Data) MinRetryBackoff() int { return 0 } -func (d Data) MaxRetryBackoff() int { +func (s Data) MaxRetryBackoff() int { return 0 } -func (d Data) RetryExponentialBackoff() bool { +func (s Data) RetryExponentialBackoff() bool { return false } diff --git a/pkg/ci/data_run.go b/pkg/ci/data_run.go index fbf323e..3c8bb1b 100644 --- a/pkg/ci/data_run.go +++ b/pkg/ci/data_run.go @@ -7,24 +7,26 @@ import ( "github.com/sirupsen/logrus" dataBlock "github.com/srevinsaju/togomak/v1/pkg/blocks/data" "github.com/srevinsaju/togomak/v1/pkg/c" - "github.com/srevinsaju/togomak/v1/pkg/diag" "github.com/zclconf/go-cty/cty" "sync" ) -func (d Data) Prepare(ctx context.Context, skip bool, overridden bool) diag.Diagnostics { +const ( + DataAttrValue = "value" +) + +func (s Data) Prepare(ctx context.Context, skip bool, overridden bool) hcl.Diagnostics { return nil // no-op } -func (d Data) Run(ctx context.Context) diag.Diagnostics { +func (s Data) Run(ctx context.Context) hcl.Diagnostics { // _ := ctx.Value(TogomakContextHclDiagWriter).(hcl.DiagnosticWriter) - logger := ctx.Value(c.TogomakContextLogger).(*logrus.Logger).WithField(DataBlock, d.Id) - logger.Debugf("running %s.%s.%s", DataBlock, d.Provider, d.Id) + logger := ctx.Value(c.TogomakContextLogger).(*logrus.Logger).WithField(DataBlock, s.Id) + logger.Debugf("running %s.%s.%s", DataBlock, s.Provider, s.Id) hclContext := ctx.Value(c.TogomakContextHclEval).(*hcl.EvalContext) - hcDiagWriter := ctx.Value(c.TogomakContextHclDiagWriter).(hcl.DiagnosticWriter) muData := ctx.Value(c.TogomakContextMutexData).(*sync.Mutex) - var hcDiags hcl.Diagnostics - var diags diag.Diagnostics + var diags hcl.Diagnostics + var d hcl.Diagnostics // region: mutating the data map // TODO: move it to a dedicated helper function @@ -34,26 +36,28 @@ func (d Data) Run(ctx context.Context) diag.Diagnostics { var value string var attr map[string]cty.Value for _, pr := range dataBlock.DefaultProviders { - if pr.Name() == d.Provider { + if pr.Name() == s.Provider { validProvider = true provide := pr.New() provide.SetContext(ctx) - diags = diags.Extend(provide.DecodeBody(d.Body)) - value = provide.Value(ctx, d.Id) - attr = provide.Attributes(ctx) + diags = diags.Extend(provide.DecodeBody(s.Body)) + value, d = provide.Value(ctx, s.Id) + diags = diags.Extend(d) + attr, d = provide.Attributes(ctx) + diags = diags.Extend(d) break } } if !validProvider || diags.HasErrors() { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityInvalid, - Summary: fmt.Sprintf("invalid provider %s", d.Provider), + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("invalid provider %s", s.Provider), Detail: fmt.Sprintf("built-in providers are %v", dataBlock.DefaultProviders), }) return diags } m := make(map[string]cty.Value) - m["value"] = cty.StringVal(value) + m[DataAttrValue] = cty.StringVal(value) for k, v := range attr { m[k] = v } @@ -66,44 +70,28 @@ func (d Data) Run(ctx context.Context) diag.Diagnostics { } else { dataMutated = data.AsValueMap() } - provider := dataMutated[d.Provider] + provider := dataMutated[s.Provider] var providerMutated map[string]cty.Value if provider.IsNull() { providerMutated = make(map[string]cty.Value) } else { providerMutated = provider.AsValueMap() } - providerMutated[d.Id] = cty.ObjectVal(m) - dataMutated[d.Provider] = cty.ObjectVal(providerMutated) + providerMutated[s.Id] = cty.ObjectVal(m) + dataMutated[s.Provider] = cty.ObjectVal(providerMutated) hclContext.Variables[DataBlock] = cty.ObjectVal(dataMutated) muData.Unlock() // endregion - if hcDiags.HasErrors() { - err := hcDiagWriter.WriteDiagnostics(hcDiags) - if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, - Summary: "failed to write HCL diagnostics", - Detail: err.Error(), - }) - } - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, - Summary: "failed to evaluate data block", - Detail: hcDiags.Error(), - }) - } - if diags.HasErrors() { return diags } - v := d.Variables() - logger.Debug(fmt.Sprintf("data.%s variables: %v", d.Id, v)) + v := s.Variables() + logger.Debug(fmt.Sprintf("%s variables: %v", s.Identifier(), v)) return nil } -func (d Data) CanRun(ctx context.Context) (bool, diag.Diagnostics) { +func (s Data) CanRun(ctx context.Context) (bool, hcl.Diagnostics) { return true, nil } diff --git a/pkg/ci/locals.go b/pkg/ci/locals.go index ab7b49d..8788e30 100644 --- a/pkg/ci/locals.go +++ b/pkg/ci/locals.go @@ -2,7 +2,7 @@ package ci import ( "fmt" - "github.com/srevinsaju/togomak/v1/pkg/diag" + "github.com/hashicorp/hcl/v2" ) const LocalsBlock = "locals" @@ -20,16 +20,16 @@ func (l Locals) Type() string { return LocalsBlock } -func (l Locals) Expand() ([]*Local, diag.Diagnostics) { +func (l Locals) Expand() ([]*Local, hcl.Diagnostics) { var locals []*Local - var diags diag.Diagnostics + var diags hcl.Diagnostics attr, err := l.Body.JustAttributes() if err != nil { - return nil, diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, + return nil, diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, Summary: "Failed to decode locals", Detail: err.Error(), - Source: l.Identifier(), + Subject: l.Body.MissingItemRange().Ptr(), }) } @@ -58,11 +58,11 @@ func (*Local) IsDaemon() bool { return false } -func (*Local) Terminate() diag.Diagnostics { +func (*Local) Terminate() hcl.Diagnostics { return nil } -func (*Local) Kill() diag.Diagnostics { +func (*Local) Kill() hcl.Diagnostics { return nil } @@ -73,8 +73,8 @@ func (*Local) Get(k any) any { return nil } -func (l LocalsGroup) Expand() ([]*Local, diag.Diagnostics) { - var diags diag.Diagnostics +func (l LocalsGroup) Expand() ([]*Local, hcl.Diagnostics) { + var diags hcl.Diagnostics var locals []*Local for _, lo := range l { ll, dd := lo.Expand() @@ -84,11 +84,17 @@ func (l LocalsGroup) Expand() ([]*Local, diag.Diagnostics) { return locals, diags } -func (l LocalGroup) ById(id string) (*Local, error) { +func (l LocalGroup) ById(id string) (*Local, hcl.Diagnostics) { for _, lo := range l { if lo.Key == id { return lo, nil } } - return nil, fmt.Errorf("local variable with id %s not found", id) + return nil, hcl.Diagnostics{ + { + Severity: hcl.DiagError, + Summary: "Local not found", + Detail: "Local with id " + id + " not found", + }, + } } diff --git a/pkg/ci/locals_run.go b/pkg/ci/locals_run.go index 4da8415..fb32473 100644 --- a/pkg/ci/locals_run.go +++ b/pkg/ci/locals_run.go @@ -5,20 +5,18 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/sirupsen/logrus" "github.com/srevinsaju/togomak/v1/pkg/c" - "github.com/srevinsaju/togomak/v1/pkg/diag" "github.com/zclconf/go-cty/cty" "sync" ) -func (l *Local) Run(ctx context.Context) diag.Diagnostics { +func (l *Local) Run(ctx context.Context) hcl.Diagnostics { logger := ctx.Value(c.TogomakContextLogger).(*logrus.Logger).WithField(LocalBlock, l.Key) logger.Debugf("running %s.%s", LocalBlock, l.Key) hclContext := ctx.Value(c.TogomakContextHclEval).(*hcl.EvalContext) - hcDiagWriter := ctx.Value(c.TogomakContextHclDiagWriter).(hcl.DiagnosticWriter) + muLocals := ctx.Value(c.TogomakContextMutexLocals).(*sync.Mutex) - var hcDiags hcl.Diagnostics - var diags diag.Diagnostics + var diags hcl.Diagnostics // region: mutating the data map // TODO: move it to a dedicated helper function @@ -31,38 +29,19 @@ func (l *Local) Run(ctx context.Context) diag.Diagnostics { localMutated = locals.AsValueMap() } v, d := l.Value.Value(hclContext) - hcDiags = hcDiags.Extend(d) + diags = diags.Extend(d) localMutated[l.Key] = v hclContext.Variables[LocalBlock] = cty.ObjectVal(localMutated) muLocals.Unlock() // endregion - if hcDiags.HasErrors() { - err := hcDiagWriter.WriteDiagnostics(hcDiags) - if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, - Summary: "failed to write HCL diagnostics", - Detail: err.Error(), - }) - } - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, - Summary: "failed to evaluate data block", - Detail: hcDiags.Error(), - }) - } - - if diags.HasErrors() { - return diags - } - return nil + return diags } -func (l *Local) CanRun(ctx context.Context) (bool, diag.Diagnostics) { +func (l *Local) CanRun(ctx context.Context) (bool, hcl.Diagnostics) { return true, nil } -func (l *Local) Prepare(ctx context.Context, skip bool, overridden bool) diag.Diagnostics { +func (l *Local) Prepare(ctx context.Context, skip bool, overridden bool) hcl.Diagnostics { return nil } diff --git a/pkg/ci/macro.go b/pkg/ci/macro.go index 0cd74ec..f0cb182 100644 --- a/pkg/ci/macro.go +++ b/pkg/ci/macro.go @@ -1,9 +1,7 @@ package ci import ( - "fmt" "github.com/hashicorp/hcl/v2" - "github.com/srevinsaju/togomak/v1/pkg/diag" ) const MacroBlock = "macro" @@ -17,13 +15,19 @@ func (m *Macro) Identifier() string { return m.Id } -func (m Macros) ById(id string) (*Macro, error) { +func (m Macros) ById(id string) (*Macro, hcl.Diagnostics) { for _, macro := range m { if macro.Id == id { return ¯o, nil } } - return nil, fmt.Errorf("macro block with id %s not found", id) + return nil, hcl.Diagnostics{ + { + Severity: hcl.DiagError, + Summary: "Macro not found", + Detail: "Macro with id " + id + " not found", + }, + } } func (m *Macro) Type() string { @@ -41,11 +45,11 @@ func (m *Macro) IsDaemon() bool { return false } -func (m *Macro) Terminate() diag.Diagnostics { +func (m *Macro) Terminate() hcl.Diagnostics { return nil } -func (m *Macro) Kill() diag.Diagnostics { +func (m *Macro) Kill() hcl.Diagnostics { return nil } diff --git a/pkg/ci/macro_run.go b/pkg/ci/macro_run.go index d99ae59..285ab09 100644 --- a/pkg/ci/macro_run.go +++ b/pkg/ci/macro_run.go @@ -5,7 +5,6 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/sirupsen/logrus" "github.com/srevinsaju/togomak/v1/pkg/c" - "github.com/srevinsaju/togomak/v1/pkg/diag" "github.com/zclconf/go-cty/cty" ) @@ -13,18 +12,16 @@ const ( SourceTypeGit = "git" ) -func (m *Macro) Prepare(ctx context.Context, skip bool, overridden bool) diag.Diagnostics { +func (m *Macro) Prepare(ctx context.Context, skip bool, overridden bool) hcl.Diagnostics { return nil // no-op } -func (m *Macro) Run(ctx context.Context) diag.Diagnostics { +func (m *Macro) Run(ctx context.Context) hcl.Diagnostics { // _ := ctx.Value(TogomakContextHclDiagWriter).(hcl.DiagnosticWriter) logger := ctx.Value(c.TogomakContextLogger).(*logrus.Logger).WithField(DataBlock, m.Id) logger.Tracef("running %s.%s", MacroBlock, m.Id) hclContext := ctx.Value(c.TogomakContextHclEval).(*hcl.EvalContext) - hcDiagWriter := ctx.Value(c.TogomakContextHclDiagWriter).(hcl.DiagnosticWriter) - var hcDiags hcl.Diagnostics - var diags diag.Diagnostics + var diags hcl.Diagnostics // region: mutating the data map // TODO: move it to a dedicated helper function @@ -39,7 +36,7 @@ func (m *Macro) Run(ctx context.Context) diag.Diagnostics { // -> update r.Value accordingly f, d := m.Files.Value(hclContext) if d != nil { - hcDiags = hcDiags.Extend(d) + diags = diags.Extend(d) } macroMutated[m.Id] = cty.ObjectVal(map[string]cty.Value{ "files": f, @@ -47,16 +44,9 @@ func (m *Macro) Run(ctx context.Context) diag.Diagnostics { hclContext.Variables[MacroBlock] = cty.ObjectVal(macroMutated) // endregion - if hcDiags.HasErrors() { - diags = diags.ExtendHCLDiagnostics(hcDiags, hcDiagWriter, m.Identifier()) - } - - if diags.HasErrors() { - return diags - } - return nil + return diags } -func (m *Macro) CanRun(ctx context.Context) (bool, diag.Diagnostics) { +func (m *Macro) CanRun(ctx context.Context) (bool, hcl.Diagnostics) { return true, nil } diff --git a/pkg/ci/runnable.go b/pkg/ci/runnable.go index 0819a55..ffe0771 100644 --- a/pkg/ci/runnable.go +++ b/pkg/ci/runnable.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "github.com/hashicorp/hcl/v2" - "github.com/srevinsaju/togomak/v1/pkg/diag" "strings" "sync" @@ -12,19 +11,9 @@ import ( const ThisBlock = "this" -type Runnable interface { - Description() string - Identifier() string - - Run(ctx context.Context) diag.Diagnostics - CanRun(ctx context.Context) (bool, diag.Diagnostics) - +type Retryable interface { // CanRetry decides if the runnable can be retried CanRetry() bool - - // Prepare is called before the runnable is run - Prepare(ctx context.Context, skip bool, overridden bool) diag.Diagnostics - // MaxRetries returns the maximum number of retries that is valid for // this runnable MaxRetries() int @@ -35,24 +24,53 @@ type Runnable interface { // RetryExponentialBackoff returns true if the backoff time should be // exponentially increasing RetryExponentialBackoff() bool +} - // IsDaemon returns true if the runnable is a daemon - IsDaemon() bool - - Variables() []hcl.Traversal - +type Describable interface { + Description() string + Identifier() string Type() string +} - Terminate() diag.Diagnostics - Kill() diag.Diagnostics +type Runnable interface { + // Prepare is called before the runnable is run + Prepare(ctx context.Context, skip bool, overridden bool) hcl.Diagnostics + Run(ctx context.Context) hcl.Diagnostics + CanRun(ctx context.Context) (bool, hcl.Diagnostics) +} + +type Traversable interface { + Variables() []hcl.Traversal +} +type Contextual interface { Set(k any, v any) Get(k any) any } -type Runnables []Runnable +type Killable interface { + Terminate() hcl.Diagnostics + Kill() hcl.Diagnostics +} + +type Daemon interface { + // IsDaemon returns true if the runnable is a daemon + IsDaemon() bool +} + +type Block interface { + Retryable + Describable + Contextual + Traversable + Runnable + Killable + Daemon +} + +type Blocks []Block -func (r Runnables) Variables() []hcl.Traversal { +func (r Blocks) Variables() []hcl.Traversal { var traversal []hcl.Traversal for _, runnable := range r { traversal = append(traversal, runnable.Variables()...) @@ -60,14 +78,14 @@ func (r Runnables) Variables() []hcl.Traversal { return traversal } -func (r Runnables) Run(ctx context.Context) diag.Diagnostics { +func (r Blocks) Run(ctx context.Context) hcl.Diagnostics { // run all runnables in parallel, collect errors and return // create a channel to receive errors var wg sync.WaitGroup errChan := make(chan error, len(r)) for _, runnable := range r { wg.Add(1) - go func(runnable Runnable) { + go func(runnable Block) { defer wg.Done() errChan <- runnable.Run(ctx) }(runnable) @@ -78,17 +96,14 @@ func (r Runnables) Run(ctx context.Context) diag.Diagnostics { return nil } -func Resolve(ctx context.Context, pipe *Pipeline, id string) (Runnable, diag.Diagnostics) { - summaryErr := "Resolution failed" - var diags diag.Diagnostics +func Resolve(ctx context.Context, pipe *Pipeline, id string) (Block, hcl.Diagnostics) { + var diags hcl.Diagnostics blocks := strings.Split(id, ".") if len(blocks) != 2 && len(blocks) != 3 { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, Summary: "Invalid identifier", Detail: fmt.Sprintf("Expected a valid identifier, got '%s'", id), - - Source: "resolve", }) } if diags.HasErrors() { @@ -97,30 +112,26 @@ func Resolve(ctx context.Context, pipe *Pipeline, id string) (Runnable, diag.Dia switch blocks[0] { case "provider": - return nil, diags.Append(diag.NewNotImplementedError("provider")) + return nil, diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Unsupported identifier", + Detail: fmt.Sprintf("Expected a valid identifier, got %s", id), + }) case StageBlock: - stage, err := pipe.Stages.ById(blocks[1]) - if err != nil { - diags = diags.Append(diag.NewError("resolve", summaryErr, err.Error())) - } + stage, d := pipe.Stages.ById(blocks[1]) + diags = diags.Extend(d) return stage, diags case DataBlock: - data, err := pipe.Data.ById(blocks[1], blocks[2]) - if err != nil { - diags = diags.Append(diag.NewError("resolve", summaryErr, err.Error())) - } + data, d := pipe.Data.ById(blocks[1], blocks[2]) + diags = diags.Extend(d) return data, diags case MacroBlock: - macro, err := pipe.Macros.ById(blocks[1]) - if err != nil { - diags = diags.Append(diag.NewError("resolve", summaryErr, err.Error())) - } + macro, d := pipe.Macros.ById(blocks[1]) + diags = diags.Extend(d) return macro, diags case LocalBlock: - local, err := pipe.Local.ById(blocks[1]) - if err != nil { - diags = diags.Append(diag.NewError("resolve", summaryErr, err.Error())) - } + local, d := pipe.Local.ById(blocks[1]) + diags = diags.Extend(d) return local, diags case LocalsBlock: panic("locals block cannot be resolved") @@ -131,8 +142,8 @@ func Resolve(ctx context.Context, pipe *Pipeline, id string) (Runnable, diag.Dia return nil, nil } - return nil, diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, + return nil, diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, Summary: "Unsupported identifier", Detail: fmt.Sprintf("Expected a valid identifier, got %s", id), }) diff --git a/pkg/ci/stage.go b/pkg/ci/stage.go index 738d53a..69363e0 100644 --- a/pkg/ci/stage.go +++ b/pkg/ci/stage.go @@ -3,6 +3,7 @@ package ci import ( "context" "fmt" + "github.com/hashicorp/hcl/v2" ) const StageContextChildStatuses = "child_statuses" @@ -39,11 +40,17 @@ func (s *Stage) IsDaemon() bool { return s.Daemon != nil && s.Daemon.Enabled } -func (s Stages) ById(id string) (*Stage, error) { +func (s Stages) ById(id string) (*Stage, hcl.Diagnostics) { for _, stage := range s { if stage.Id == id { return &stage, nil } } - return nil, fmt.Errorf("stage with id %s not found", id) + return nil, hcl.Diagnostics{ + { + Severity: hcl.DiagError, + Summary: "Stage not found", + Detail: fmt.Sprintf("Stage with id %s not found", id), + }, + } } diff --git a/pkg/ci/stage_run.go b/pkg/ci/stage_run.go index e9351f3..18acc56 100644 --- a/pkg/ci/stage_run.go +++ b/pkg/ci/stage_run.go @@ -12,7 +12,6 @@ import ( "github.com/imdario/mergo" "github.com/sirupsen/logrus" "github.com/srevinsaju/togomak/v1/pkg/c" - "github.com/srevinsaju/togomak/v1/pkg/diag" "github.com/srevinsaju/togomak/v1/pkg/meta" "github.com/srevinsaju/togomak/v1/pkg/ui" "github.com/zclconf/go-cty/cty" @@ -30,7 +29,7 @@ const TogomakParamEnvVarPrefix = "TOGOMAK__param__" var TogomakParamEnvVarRegexExpression = fmt.Sprintf("%s([a-zA-Z0-9_]+)", TogomakParamEnvVarPrefix) var TogomakParamEnvVarRegex = regexp.MustCompile(TogomakParamEnvVarRegexExpression) -func (s *Stage) Prepare(ctx context.Context, skip bool, overridden bool) diag.Diagnostics { +func (s *Stage) Prepare(ctx context.Context, skip bool, overridden bool) hcl.Diagnostics { logger := ctx.Value(c.TogomakContextLogger).(*logrus.Logger) // show some user-friendly output on the details of the stage about to be run @@ -47,6 +46,7 @@ func (s *Stage) Prepare(ctx context.Context, skip bool, overridden bool) diag.Di return nil } +// expandMacros expands the macro in the stage, if any. func (s *Stage) expandMacros(ctx context.Context) (*Stage, hcl.Diagnostics) { if s.Use == nil { @@ -62,42 +62,42 @@ func (s *Stage) expandMacros(ctx context.Context) (*Stage, hcl.Diagnostics) { unattended := ctx.Value(c.TogomakContextUnattended).(bool) logger.Debugf("running %s.%s", s.Identifier(), MacroBlock) - var hclDiags hcl.Diagnostics + var diags hcl.Diagnostics var err error v := s.Use.Macro.Variables() if v == nil || len(v) == 0 { // this stage does not use a macro - return s, hclDiags + return s, diags } if len(v) != 1 { - hclDiags = hclDiags.Append(&hcl.Diagnostic{ + diags = diags.Append(&hcl.Diagnostic{ Severity: hcl.DiagError, Summary: "invalid macro", Detail: fmt.Sprintf("%s can only use a single macro", s.Identifier()), EvalContext: hclContext, Subject: v[0].SourceRange().Ptr(), }) - return s, hclDiags + return s, diags } variable := v[0] if variable.RootName() != MacroBlock { - hclDiags = hclDiags.Append(&hcl.Diagnostic{ + diags = diags.Append(&hcl.Diagnostic{ Severity: hcl.DiagError, Summary: "invalid macro", Detail: fmt.Sprintf("%s uses an invalid macro, got '%s'", s.Identifier(), variable.RootName()), EvalContext: hclContext, Subject: v[0].SourceRange().Ptr(), }) - return s, hclDiags + return s, diags } macroName := variable[1].(hcl.TraverseAttr).Name logger.Debugf("stage.%s uses macro.%s", s.Id, macroName) macroRunnable, d := Resolve(ctx, pipe, fmt.Sprintf("macro.%s", macroName)) if d.HasErrors() { - d.Fatal(logger.WriterLevel(logrus.ErrorLevel)) + return nil, diags.Extend(d) } macro := macroRunnable.(*Macro) @@ -127,14 +127,14 @@ func (s *Stage) expandMacros(ctx context.Context) (*Stage, hcl.Diagnostics) { } else { f, d := macro.Files.Value(hclContext) if d.HasErrors() { - return s, hclDiags.Extend(d) + return s, diags.Extend(d) } if !f.IsNull() { files := f.AsValueMap() logger.Debugf("using %d files from %s", len(files), macro.Identifier()) err = os.MkdirAll(filepath.Join(tmpDir, s.Id), 0755) if err != nil { - return s, hclDiags.Append(&hcl.Diagnostic{ + return s, diags.Append(&hcl.Diagnostic{ Severity: hcl.DiagError, Summary: "failed to create temporary directory", Detail: fmt.Sprintf("failed to create temporary directory for stage %s", s.Id), @@ -156,7 +156,7 @@ func (s *Stage) expandMacros(ctx context.Context) (*Stage, hcl.Diagnostics) { fpath := filepath.Join(tmpDir, s.Id, fName) logger.Debugf("writing %s to %s", fName, fpath) if fContent.IsNull() { - return s, hclDiags.Append(&hcl.Diagnostic{ + return s, diags.Append(&hcl.Diagnostic{ Severity: hcl.DiagError, Summary: "invalid macro", Detail: fmt.Sprintf("%s uses a macro with an invalid file %s", s.Identifier(), fName), @@ -167,7 +167,7 @@ func (s *Stage) expandMacros(ctx context.Context) (*Stage, hcl.Diagnostics) { err = os.WriteFile(fpath, []byte(fContent.AsString()), 0644) if err != nil { // TODO: move to diagnostics - return s, hclDiags.Append(&hcl.Diagnostic{ + return s, diags.Append(&hcl.Diagnostic{ Severity: hcl.DiagError, Summary: "invalid macro", Detail: fmt.Sprintf("%s uses a macro with an invalid file %s", s.Identifier(), fName), @@ -182,14 +182,14 @@ func (s *Stage) expandMacros(ctx context.Context) (*Stage, hcl.Diagnostics) { } } if defaultExecutionPath == "" { - hclDiags = hclDiags.Append(&hcl.Diagnostic{ + diags = diags.Append(&hcl.Diagnostic{ Severity: hcl.DiagError, Summary: "invalid macro", Detail: fmt.Sprintf("%s uses a macro without a default execution file. include a file named togomak.hcl to avoid this error", s.Identifier()), EvalContext: hclContext, Subject: variable.SourceRange().Ptr(), }) - return s, hclDiags + return s, diags } executable, err := os.Executable() @@ -237,15 +237,13 @@ func (s *Stage) expandMacros(ctx context.Context) (*Stage, hcl.Diagnostics) { } -func (s *Stage) Run(ctx context.Context) diag.Diagnostics { - hclDgWriter := ctx.Value(c.TogomakContextHclDiagWriter).(hcl.DiagnosticWriter) +func (s *Stage) Run(ctx context.Context) hcl.Diagnostics { logger := ctx.Value(c.TogomakContextLogger).(*logrus.Logger).WithField(StageBlock, s.Id) cwd := ctx.Value(c.TogomakContextCwd).(string) tmpDir := ctx.Value(c.TogomakContextTempDir).(string) logger.Debugf("running %s.%s", StageBlock, s.Id) isDryRun := ctx.Value(c.TogomakContextPipelineDryRun).(bool) - var diags diag.Diagnostics var hclDiags hcl.Diagnostics var err error evalCtx := ctx.Value(c.TogomakContextHclEval).(*hcl.EvalContext) @@ -300,9 +298,8 @@ func (s *Stage) Run(ctx context.Context) diag.Diagnostics { environment[env.Name] = v } - diags = diags.ExtendHCLDiagnostics(hclDiags, hclDgWriter, s.Identifier()) - if diags.HasErrors() { - return diags + if hclDiags.HasErrors() { + return hclDiags } envStrings := make([]string, len(environment)) @@ -350,12 +347,14 @@ func (s *Stage) Run(ctx context.Context) diag.Diagnostics { } } else if s.Container == nil { // if the container is not null, we may rely on internal args or entrypoint scripts - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, - Summary: "script or args must be a string", - Detail: fmt.Sprintf("the provided script or args, was not recognized as a valid string. received script='''%s''', args='''%s'''", script, args), + return hclDiags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "No commands specified", + Detail: "Either script or args must be specified", + Subject: s.Script.Range().Ptr(), + EvalContext: evalCtx, }) - return diags + } else { emptyCommands = true } @@ -405,17 +404,18 @@ func (s *Stage) Run(ctx context.Context) diag.Diagnostics { }) } if hclDiags.HasErrors() { - return diags.ExtendHCLDiagnostics(hclDiags, hclDgWriter, s.Identifier()) + return hclDiags } image := imageRaw.AsString() cli, err := dockerClient.NewClientWithOpts(dockerClient.FromEnv, dockerClient.WithAPIVersionNegotiation()) if err != nil { - return diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, - Summary: "could not create docker client", - Detail: err.Error(), - Source: s.Identifier(), + return hclDiags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "could not create docker client", + Detail: err.Error(), + Subject: s.Container.Image.Range().Ptr(), + EvalContext: evalCtx, }) } defer cli.Close() @@ -426,11 +426,12 @@ func (s *Stage) Run(ctx context.Context) diag.Diagnostics { logger.Infof("image %s does not exist, pulling...", image) reader, err := cli.ImagePull(ctx, image, types.ImagePullOptions{}) if err != nil { - return diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, - Summary: "could not pull image", - Detail: err.Error(), - Source: s.Identifier(), + return hclDiags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "could not pull image", + Detail: err.Error(), + Subject: s.Container.Image.Range().Ptr(), + EvalContext: evalCtx, }) } @@ -459,16 +460,15 @@ func (s *Stage) Run(ctx context.Context) diag.Diagnostics { } binds = append(binds, fmt.Sprintf("%s:%s", source.AsString(), dest.AsString())) } - diags = diags.ExtendHCLDiagnostics(hclDiags, hclDgWriter, s.Identifier()) - if diags.HasErrors() { - return diags + if hclDiags.HasErrors() { + return hclDiags } if !isDryRun { exposedPorts, bindings, d := s.Container.Ports.Nat(evalCtx) - if d.HasErrors() { - hclDiags = hclDiags.Extend(d) - return diags.ExtendHCLDiagnostics(hclDiags, hclDgWriter, s.Identifier()) + hclDiags = hclDiags.Extend(d) + if hclDiags.HasErrors() { + return hclDiags } resp, err := cli.ContainerCreate(ctx, &dockerContainer.Config{ @@ -489,20 +489,21 @@ func (s *Stage) Run(ctx context.Context) diag.Diagnostics { PortBindings: bindings, }, nil, nil, "") if err != nil { - return diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, - Summary: "could not create container", - Detail: err.Error(), - Source: s.Identifier(), + return hclDiags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "could not create container", + Detail: err.Error(), + Subject: s.Container.Image.Range().Ptr(), + EvalContext: evalCtx, }) } if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil { - return diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, + return hclDiags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, Summary: "could not start container", Detail: err.Error(), - Source: s.Identifier(), + Subject: s.Container.Image.Range().Ptr(), }) } s.ContainerId = resp.ID @@ -517,13 +518,12 @@ func (s *Stage) Run(ctx context.Context) diag.Diagnostics { Follow: true, }) if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, - Summary: "failed to get container logs", + return hclDiags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "could not get container logs", Detail: err.Error(), + Subject: s.Container.Image.Range().Ptr(), }) - diags.Write(logger.WriterLevel(logrus.ErrorLevel)) - return diags } defer responseBody.Close() @@ -534,24 +534,26 @@ func (s *Stage) Run(ctx context.Context) diag.Diagnostics { } if err != nil && err != io.EOF { if err == context.Canceled { - return diags + return hclDiags } - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, + hclDiags = hclDiags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, Summary: "failed to copy container logs", Detail: err.Error(), + Subject: s.Container.Image.Range().Ptr(), }) } err = cli.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{ RemoveVolumes: true, }) if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityWarning, - Summary: "failed to remove docker container", + hclDiags = hclDiags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "failed to remove container", Detail: err.Error(), + Subject: s.Container.Image.Range().Ptr(), }) - return diags + return hclDiags } } else { fmt.Println(ui.Blue("docker:run.image"), ui.Green(image)) @@ -566,31 +568,26 @@ func (s *Stage) Run(ctx context.Context) diag.Diagnostics { } if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, - Summary: "failed to run command", + hclDiags = hclDiags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("failed to run command (%s)", s.Identifier()), Detail: err.Error(), - Source: s.Identifier(), }) - return diags } - return diags + return hclDiags } -func (s *Stage) CanRun(ctx context.Context) (bool, diag.Diagnostics) { +func (s *Stage) CanRun(ctx context.Context) (bool, hcl.Diagnostics) { logger := ctx.Value(c.TogomakContextLogger).(*logrus.Logger).WithField("stage", s.Id) logger.Debugf("checking if stage.%s can run", s.Id) - - var diags diag.Diagnostics evalCtx := ctx.Value(c.TogomakContextHclEval).(*hcl.EvalContext) - hclWriter := ctx.Value(c.TogomakContextHclDiagWriter).(hcl.DiagnosticWriter) - var hclDiags hcl.Diagnostics + var diags hcl.Diagnostics paramsGo := map[string]cty.Value{} if s.Use != nil && s.Use.Parameters != nil { parameters, d := s.Use.Parameters.Value(evalCtx) - hclDiags = hclDiags.Extend(d) + diags = diags.Extend(d) if !parameters.IsNull() { for k, v := range parameters.AsValueMap() { paramsGo[k] = v @@ -606,26 +603,11 @@ func (s *Stage) CanRun(ctx context.Context) (bool, diag.Diagnostics) { }), "param": cty.ObjectVal(paramsGo), } - v, hclDiags := s.Condition.Value(evalCtx) - if hclDiags.HasErrors() { - err := hclWriter.WriteDiagnostics(hclDiags) - if err != nil { - diags = append(diags, diag.Diagnostic{ - Severity: diag.SeverityError, - Summary: "failed to write HCL diagnostics", - Detail: err.Error(), - }) - } - - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, - Summary: "failed to evaluate condition", - Detail: hclDiags.Error(), - Source: "stage_condition_check", - }) - - return false, diags + v, d := s.Condition.Value(evalCtx) + if d.HasErrors() { + return false, diags.Extend(d) } + if v.Equals(cty.False).True() { // this stage has been explicitly evaluated to false // we will not run this @@ -639,36 +621,36 @@ func dockerContainerSourceFmt(containerId string) string { return fmt.Sprintf("docker: container=%s", containerId) } -func (s *Stage) Terminate() diag.Diagnostics { - var diags diag.Diagnostics +func (s *Stage) Terminate() hcl.Diagnostics { + var diags hcl.Diagnostics if s.Container != nil && s.ContainerId != "" { ctx := context.Background() cli, err := dockerClient.NewClientWithOpts(dockerClient.FromEnv, dockerClient.WithAPIVersionNegotiation()) if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, Summary: "failed to create docker client", - Detail: err.Error(), - Source: dockerContainerSourceFmt(s.ContainerId), + Detail: fmt.Sprintf("%s: %s", dockerContainerSourceFmt(s.ContainerId), err.Error()), }) } err = cli.ContainerStop(ctx, s.ContainerId, dockerContainer.StopOptions{}) if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, Summary: "failed to stop container", - Detail: err.Error(), - Source: dockerContainerSourceFmt(s.ContainerId), + Detail: fmt.Sprintf("%s: %s", dockerContainerSourceFmt(s.ContainerId), err.Error()), }) } - } else if s.process != nil { + } else if s.process != nil && s.process.Process != nil { + if s.process.ProcessState.Exited() { + return diags + } err := s.process.Process.Signal(syscall.SIGTERM) if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, Summary: "failed to terminate process", - Source: s.Identifier(), Detail: err.Error(), }) } @@ -677,14 +659,13 @@ func (s *Stage) Terminate() diag.Diagnostics { return diags } -func (s *Stage) Kill() diag.Diagnostics { +func (s *Stage) Kill() hcl.Diagnostics { diags := s.Terminate() if s.process != nil && !s.process.ProcessState.Exited() { err := s.process.Process.Kill() if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, - Source: s.Identifier(), + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, Summary: "couldn't kill stage", Detail: err.Error(), }) diff --git a/pkg/core/togomak.go b/pkg/core/togomak.go new file mode 100644 index 0000000..9a8bc95 --- /dev/null +++ b/pkg/core/togomak.go @@ -0,0 +1 @@ +package core diff --git a/pkg/diag/diagnostics.go b/pkg/diag/diagnostics.go deleted file mode 100644 index 6219f1b..0000000 --- a/pkg/diag/diagnostics.go +++ /dev/null @@ -1,152 +0,0 @@ -package diag - -import ( - "fmt" - "github.com/hashicorp/hcl/v2" - "github.com/srevinsaju/togomak/v1/pkg/ui" - "io" - "os" -) - -const ( - SeverityError = "error" - SeverityWarning = "warning" - SeverityInvalid = "invalid" -) - -type Diagnostic struct { - Severity string - Summary string - Detail string - Source string -} - -func NewDiagnostic(severity, summary, detail, source string) Diagnostic { - return Diagnostic{ - Severity: severity, - Summary: summary, - Detail: detail, - Source: source, - } -} - -type Diagnostics []Diagnostic - -func (d Diagnostics) Len() int { - return len(d) -} - -func (d Diagnostics) Extend(diags Diagnostics) Diagnostics { - return append(d, diags...) -} - -func (d Diagnostics) ExtendHCLDiagnostics(hclDiags hcl.Diagnostics, hclDgWriter hcl.DiagnosticWriter, source string) Diagnostics { - if hclDiags.HasErrors() { - err := hclDgWriter.WriteDiagnostics(hclDiags) - if err != nil { - d = d.Append(Diagnostic{ - Severity: SeverityError, - Summary: "failed to write HCL diagnostics", - Detail: err.Error(), - }) - } - d = d.Append(Diagnostic{ - Severity: SeverityError, - Summary: "failed to evaluate HCL", - Detail: hclDiags.Error(), - Source: source, - }) - } - - return d -} - -func (d Diagnostics) Append(diag Diagnostic) Diagnostics { - return append(d, diag) -} - -func (d Diagnostics) HasErrors() bool { - for _, diag := range d { - if diag.Severity == "error" { - return true - } - } - return false -} -func (d Diagnostic) Error() string { - message := fmt.Sprintf("%s: %s, (%s)", d.Severity, d.Summary, d.Detail) - return message -} - -func (d Diagnostics) Error() string { - count := len(d) - switch { - case count == 0: - return "no diagnostics" - case count == 1: - return d[0].Error() - default: - return fmt.Sprintf("%s, and %d other diagnostic(s)", d[0].Error(), count-1) - } -} - -func (d Diagnostics) HasWarnings() bool { - for _, diag := range d { - if diag.Severity == SeverityWarning { - return true - } - } - return false -} - -func NewNotImplementedError(source string) Diagnostic { - return NewError(source, "Runtime Error", "not implemented") -} - -func NewError(source, summary, message string) Diagnostic { - return NewDiagnostic(SeverityError, summary, message, source) -} - -func (d Diagnostics) NewHclWriteDiagnosticsError(source string, err error) Diagnostics { - diags := d - if err != nil { - diags = diags.Append(Diagnostic{ - Severity: SeverityError, - Summary: "failed to write diagnostics", - Detail: err.Error(), - Source: source, - }) - } - diags = diags.Append(Diagnostic{ - Severity: SeverityError, - Summary: "failed to decode body", - Detail: err.Error(), - Source: source, - }) - return diags -} - -func (d Diagnostics) Write(writer io.Writer) { - for i, diag := range d { - - diagHeader := ui.Red(fmt.Sprintf("Error %d", i+1)) - if diag.Severity == SeverityWarning { - diagHeader = ui.Yellow(fmt.Sprintf("Warning %d", i+1)) - } - _, err := fmt.Fprintf(writer, "%s: %s\n\t%s\n\tsource: %s\n\n", - diagHeader, - ui.Bold(diag.Summary), - diag.Detail, - ui.Grey(diag.Source), - ) - if err != nil { - panic(err) - } - } - writer.Write([]byte("\n")) -} - -func (d Diagnostics) Fatal(writer io.Writer) { - d.Write(writer) - os.Exit(1) -} diff --git a/pkg/graph/depgraph.go b/pkg/graph/depgraph.go index 8b45ce4..caf70e0 100644 --- a/pkg/graph/depgraph.go +++ b/pkg/graph/depgraph.go @@ -6,14 +6,13 @@ import ( "github.com/kendru/darwin/go/depgraph" "github.com/sirupsen/logrus" "github.com/srevinsaju/togomak/v1/pkg/ci" - "github.com/srevinsaju/togomak/v1/pkg/diag" "github.com/srevinsaju/togomak/v1/pkg/meta" "github.com/srevinsaju/togomak/v1/pkg/x" ) -func TopoSort(ctx context.Context, pipe *ci.Pipeline) (*depgraph.Graph, diag.Diagnostics) { +func TopoSort(ctx context.Context, pipe *ci.Pipeline) (*depgraph.Graph, hcl.Diagnostics) { g := depgraph.New() - var diags diag.Diagnostics + var diags hcl.Diagnostics logger := ctx.Value("logger").(*logrus.Logger).WithField("component", "graph") for _, local := range pipe.Local { @@ -64,8 +63,8 @@ func TopoSort(ctx context.Context, pipe *ci.Pipeline) (*depgraph.Graph, diag.Dia err = g.DependOn(child, parent) if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, Summary: "Invalid dependency", Detail: err.Error(), }) @@ -125,8 +124,8 @@ func TopoSort(ctx context.Context, pipe *ci.Pipeline) (*depgraph.Graph, diag.Dia err = g.DependOn(child, parent) if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, Summary: "Invalid dependency", Detail: err.Error(), }) @@ -184,8 +183,8 @@ func TopoSort(ctx context.Context, pipe *ci.Pipeline) (*depgraph.Graph, diag.Dia err = g.DependOn(child, parent) if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, Summary: "Invalid dependency", Detail: err.Error(), }) @@ -238,8 +237,8 @@ func TopoSort(ctx context.Context, pipe *ci.Pipeline) (*depgraph.Graph, diag.Dia err = g.DependOn(child, parent) if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, Summary: "Invalid dependency", Detail: err.Error(), }) diff --git a/pkg/orchestra/context.go b/pkg/orchestra/context.go index 51c3b09..5b1d4e5 100644 --- a/pkg/orchestra/context.go +++ b/pkg/orchestra/context.go @@ -23,7 +23,7 @@ import ( ) type Togomak struct { - logger *logrus.Logger + Logger *logrus.Logger pipelineId string cfg Config cwd string @@ -33,10 +33,6 @@ type Togomak struct { hclDiagWriter hcl.DiagnosticWriter } -func (t Togomak) Logger() *logrus.Logger { - return t.logger -} - func (t Togomak) Parser() *hclparse.Parser { return t.parser } @@ -240,12 +236,12 @@ func NewContextWithTogomak(cfg Config) (Togomak, context.Context) { } parser := hclparse.NewParser() - diagnosticTextWriter := hcl.NewDiagnosticTextWriter(logger.Writer(), parser.Files(), 0, true) + diagnosticTextWriter := hcl.NewDiagnosticTextWriter(os.Stdout, parser.Files(), 0, true) ctx = context.WithValue(ctx, c.TogomakContextHclDiagWriter, diagnosticTextWriter) ctx = context.WithValue(ctx, c.TogomakContextHclEval, hclContext) t := Togomak{ - logger: logger, + Logger: logger, pipelineId: pipelineId, cfg: cfg, cwd: cwd, diff --git a/pkg/orchestra/format.go b/pkg/orchestra/format.go index 2b53070..cf7c413 100644 --- a/pkg/orchestra/format.go +++ b/pkg/orchestra/format.go @@ -17,19 +17,19 @@ func Format(cfg Config, check bool, recursive bool) error { if recursive { matches, err := doublestar.Glob("**/*.hcl") for _, path := range matches { - t.logger.Tracef("Found %s", path) + t.Logger.Tracef("Found %s", path) data, err := os.ReadFile(path) if err != nil { return err } outSrc := hclwrite.Format(data) if !bytes.Equal(outSrc, data) { - t.logger.Tracef("%s needs formatting", path) + t.Logger.Tracef("%s needs formatting", path) toFormat = append(toFormat, path) } } if err != nil { - t.logger.Fatalf("Error while globbing for **/*.hcl: %s", err) + t.Logger.Fatalf("Error while globbing for **/*.hcl: %s", err) } } else { fn := pipeline.ConfigFilePath(ctx) @@ -39,7 +39,7 @@ func Format(cfg Config, check bool, recursive bool) error { } outSrc := hclwrite.Format(data) if !bytes.Equal(outSrc, data) { - t.logger.Tracef("%s needs formatting", fn) + t.Logger.Tracef("%s needs formatting", fn) toFormat = append(toFormat, fn) } } diff --git a/pkg/orchestra/interrupt.go b/pkg/orchestra/interrupt.go index f028106..d9abcfa 100644 --- a/pkg/orchestra/interrupt.go +++ b/pkg/orchestra/interrupt.go @@ -2,20 +2,20 @@ package orchestra import ( "context" + "github.com/hashicorp/hcl/v2" "github.com/sirupsen/logrus" "github.com/srevinsaju/togomak/v1/pkg/c" "github.com/srevinsaju/togomak/v1/pkg/ci" - "github.com/srevinsaju/togomak/v1/pkg/diag" "os" "os/signal" "time" ) -func InterruptHandler(ctx context.Context, cancel context.CancelFunc, ch chan os.Signal, runnables *ci.Runnables) { +func InterruptHandler(ctx context.Context, cancel context.CancelFunc, ch chan os.Signal, runnables *ci.Blocks) { logger := ctx.Value(c.TogomakContextLogger).(*logrus.Logger) select { case <-ch: - var diags diag.Diagnostics + var diags hcl.Diagnostics logger.Warn("received interrupt signal, cancelling the pipeline") logger.Warn("stopping running operations...") logger.Warn("press CTRL+C again to force quit") @@ -25,15 +25,12 @@ func InterruptHandler(ctx context.Context, cancel context.CancelFunc, ch chan os go func() { <-ch logger.Warn("Two interrupts received. Exiting immediately.") - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, Summary: "Force quit", Detail: "data loss may have occurred", - Source: "orchestra", }) - diags.Fatal(logger.WriterLevel(logrus.ErrorLevel)) - Finale(ctx, logrus.FatalLevel) - os.Exit(1) + os.Exit(fatal(ctx)) return }() for _, runnable := range *runnables { @@ -42,8 +39,10 @@ func InterruptHandler(ctx context.Context, cancel context.CancelFunc, ch chan os diags = diags.Extend(d) } - if diags.HasErrors() || diags.HasWarnings() { - diags.Write(logger.WriterLevel(logrus.ErrorLevel)) + if diags.HasErrors() { + writer := hcl.NewDiagnosticTextWriter(os.Stderr, nil, 78, true) + _ = writer.WriteDiagnostics(diags) + os.Exit(fatal(ctx)) } cancel() case <-ctx.Done(): @@ -52,11 +51,11 @@ func InterruptHandler(ctx context.Context, cancel context.CancelFunc, ch chan os } } -func KillHandler(ctx context.Context, cancel context.CancelFunc, ch chan os.Signal, runnables *ci.Runnables) { +func KillHandler(ctx context.Context, cancel context.CancelFunc, ch chan os.Signal, runnables *ci.Blocks) { logger := ctx.Value(c.TogomakContextLogger).(*logrus.Logger) select { case <-ch: - var diags diag.Diagnostics + var diags hcl.Diagnostics logger.Warn("received kill signal, killing all subprocesses") logger.Warn("stopping running operations...") @@ -67,15 +66,16 @@ func KillHandler(ctx context.Context, cancel context.CancelFunc, ch chan os.Sign } cancel() - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, Summary: "Force quit", Detail: "data loss may have occurred", - Source: "orchestra", }) - diags.Fatal(logger.WriterLevel(logrus.ErrorLevel)) - Finale(ctx, logrus.FatalLevel) - os.Exit(1) + if diags.HasErrors() { + writer := hcl.NewDiagnosticTextWriter(os.Stderr, nil, 78, true) + _ = writer.WriteDiagnostics(diags) + } + os.Exit(fatal(ctx)) case <-ctx.Done(): logger.Infof("took %s to complete the pipeline", time.Since(ctx.Value(c.TogomakContextBootTime).(time.Time))) return diff --git a/pkg/orchestra/orchestra.go b/pkg/orchestra/orchestra.go index 140b451..48cd667 100644 --- a/pkg/orchestra/orchestra.go +++ b/pkg/orchestra/orchestra.go @@ -4,11 +4,11 @@ import ( "context" "fmt" "github.com/hashicorp/go-envparse" + "github.com/hashicorp/hcl/v2" "github.com/kendru/darwin/go/depgraph" "github.com/sirupsen/logrus" "github.com/srevinsaju/togomak/v1/pkg/c" "github.com/srevinsaju/togomak/v1/pkg/ci" - "github.com/srevinsaju/togomak/v1/pkg/diag" "github.com/srevinsaju/togomak/v1/pkg/graph" "github.com/srevinsaju/togomak/v1/pkg/meta" "github.com/srevinsaju/togomak/v1/pkg/pipeline" @@ -82,11 +82,14 @@ func Chdir(cfg Config, logger *logrus.Logger) string { } -func Orchestra(cfg Config) { - var diags diag.Diagnostics +func Orchestra(cfg Config) int { + var diags hcl.Diagnostics t, ctx := NewContextWithTogomak(cfg) ctx, cancel := context.WithCancel(ctx) - logger := t.logger + logger := t.Logger + + defer cancel() + defer diagnostics(&t, &diags) // region: external parameters paramsGo := make(map[string]cty.Value) @@ -126,14 +129,16 @@ func Orchestra(cfg Config) { } err := os.WriteFile(pipelineFilePath, pipelineData, 0644) if err != nil { - logger.Fatal(err) + return fatal(ctx) } /// we will first expand all local blocks locals, d := pipe.Locals.Expand() + diags = diags.Extend(d) if d.HasErrors() { - d.Fatal(logger.WriterLevel(logrus.ErrorLevel)) + return fatal(ctx) } + pipe.Local = locals // store the pipe in the context @@ -146,9 +151,10 @@ func Orchestra(cfg Config) { // we will now generate a dependency graph from the pipeline // this will be used to generate the pipeline var depGraph *depgraph.Graph - depGraph, diags = graph.TopoSort(ctx, pipe) + depGraph, d = graph.TopoSort(ctx, pipe) + diags = diags.Extend(d) if diags.HasErrors() { - diags.Fatal(logger.WriterLevel(logrus.ErrorLevel)) + return fatal(ctx) } // --> run the pipeline @@ -158,7 +164,7 @@ func Orchestra(cfg Config) { var wg sync.WaitGroup var daemonWg sync.WaitGroup var hasDaemons bool - var runnables ci.Runnables + var runnables ci.Blocks // region: interrupt handler chInterrupt := make(chan os.Signal, 1) @@ -180,14 +186,14 @@ func Orchestra(cfg Config) { if err == nil { e, err := envparse.Parse(envFile) if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.SeverityError, + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, Summary: "could not parse TOGOMAK_ENV file", Detail: err.Error(), }) break } - envFile.Close() + x.Must(envFile.Close()) ee := make(map[string]cty.Value) for k, v := range e { ee[k] = cty.StringVal(v) @@ -199,7 +205,7 @@ func Orchestra(cfg Config) { for _, runnableId := range layer { - var runnable ci.Runnable + var runnable ci.Block var ok bool if runnableId == meta.RootStage { @@ -270,7 +276,7 @@ func Orchestra(cfg Config) { d := runnable.Prepare(ctx, !ok, overridden) if d.HasErrors() { - diags.Write(logger.WriterLevel(logrus.ErrorLevel)) + diags = diags.Extend(d) break } @@ -351,7 +357,6 @@ func Orchestra(cfg Config) { wg.Wait() if diags.HasErrors() { - diags.Write(logger.WriterLevel(logrus.ErrorLevel)) if hasDaemons && !cfg.Pipeline.DryRun && !cfg.Unattended { logger.Info("pipeline failed, waiting for daemons to shut down") logger.Info("hit Ctrl+C to force stop them") @@ -369,12 +374,10 @@ func Orchestra(cfg Config) { } daemonWg.Wait() - - if diags.HasErrors() || diags.HasWarnings() { - diags.Fatal(logger.WriterLevel(logrus.ErrorLevel)) + if diags.HasErrors() { + return fatal(ctx) } - Finale(ctx, logrus.InfoLevel) - + return ok(ctx) } func Finale(ctx context.Context, logLevel logrus.Level) { @@ -382,3 +385,17 @@ func Finale(ctx context.Context, logLevel logrus.Level) { bootTime := ctx.Value(c.TogomakContextBootTime).(time.Time) logger.Log(logLevel, ui.Grey(fmt.Sprintf("took %s", time.Since(bootTime).Round(time.Millisecond)))) } + +func fatal(ctx context.Context) int { + Finale(ctx, logrus.ErrorLevel) + return 1 +} + +func ok(ctx context.Context) int { + Finale(ctx, logrus.InfoLevel) + return 0 +} + +func diagnostics(t *Togomak, diags *hcl.Diagnostics) { + x.Must(t.hclDiagWriter.WriteDiagnostics(*diags)) +} From ac8003f50bdb2bb332c3dab616e3781ecc57b0a5 Mon Sep 17 00:00:00 2001 From: Srevin Saju Date: Thu, 22 Jun 2023 16:20:27 +0530 Subject: [PATCH 3/6] fix: wip go-git changes' --- go.mod | 32 ++++++- go.sum | 128 ++++++++++++++++++++++++-- pkg/blocks/data/git.go | 198 ++++++++++++++++++++++++++++++++++++++--- pkg/c/context.go | 1 + pkg/ci/stage_run.go | 3 +- 5 files changed, 338 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index f3ad85e..400082e 100644 --- a/go.mod +++ b/go.mod @@ -8,9 +8,9 @@ require ( github.com/alessio/shellescape v1.4.1 github.com/bcicen/jstream v1.0.1 github.com/bmatcuk/doublestar v1.1.5 - github.com/bmatcuk/doublestar/v4 v4.6.0 github.com/creack/pty v1.1.18 github.com/docker/docker v24.0.2+incompatible + github.com/docker/go-connections v0.4.0 github.com/fatih/color v1.15.0 github.com/go-git/go-billy/v5 v5.4.1 github.com/go-git/go-git/v5 v5.7.0 @@ -20,6 +20,7 @@ require ( github.com/hashicorp/hcl/v2 v2.17.0 github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec github.com/imdario/mergo v0.3.16 + github.com/jdxcode/netrc v0.0.0-20221124155335-4616370d1a84 github.com/kendru/darwin/go/depgraph v0.0.0-20221105232959-877d6a81060c github.com/mattn/go-isatty v0.0.17 github.com/mitchellh/go-homedir v1.1.0 @@ -35,47 +36,72 @@ require ( ) require ( + code.gitea.io/gitea v1.19.3 // indirect + gitea.com/go-chi/cache v0.2.0 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 // indirect github.com/acomagu/bufpipe v1.0.4 // indirect github.com/agext/levenshtein v1.2.3 // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect + github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cloudflare/circl v1.3.3 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/djherbis/buffer v1.2.0 // indirect + github.com/djherbis/nio/v3 v3.0.1 // indirect github.com/docker/distribution v2.8.2+incompatible // indirect - github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect + github.com/go-enry/go-enry/v2 v2.8.3 // indirect + github.com/go-enry/go-oniguruma v1.2.1 // indirect + github.com/go-fed/httpsig v1.1.1-0.20201223112313-55836744818e // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/gobwas/glob v0.2.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v4 v4.4.3 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.5.9 // indirect + github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b // indirect + github.com/hashicorp/go-version v1.6.0 // indirect + github.com/hashicorp/golang-lru v0.6.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-sqlite3 v1.14.16 // indirect github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/moby/term v0.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc3 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/redis/go-redis/v9 v9.0.3 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/sergi/go-diff v1.2.0 // indirect + github.com/sergi/go-diff v1.3.1 // indirect github.com/skeema/knownhosts v1.1.1 // indirect + github.com/syndtr/goleveldb v1.0.0 // indirect + github.com/unknwon/com v1.0.1 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect + github.com/yuin/goldmark v1.5.3 // indirect golang.org/x/mod v0.9.0 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/sys v0.8.0 // indirect golang.org/x/term v0.8.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.7.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.4.0 // indirect diff --git a/go.sum b/go.sum index 075e9d1..ac18948 100644 --- a/go.sum +++ b/go.sum @@ -35,7 +35,11 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +code.gitea.io/gitea v1.19.3 h1:6RS02zVp/RZZyyIVnEQekVwM0rJOWmAXeejUHHibp/o= +code.gitea.io/gitea v1.19.3/go.mod h1:mM4ZdUm2xiQ2VXyBo6cHCig9AWRKM8JxMJBX6irYtiE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +gitea.com/go-chi/cache v0.2.0 h1:E0npuTfDW6CT1yD8NMDVc1SK6IeRjfmRL2zlEsCEd7w= +gitea.com/go-chi/cache v0.2.0/go.mod h1:iQlVK2aKTZ/rE9UcHyz9pQWGvdP9i1eI2spOpzgCrtE= github.com/AlecAivazis/survey/v2 v2.3.6 h1:NvTuVHISgTHEHeBFqt6BHOe4Ny/NwGZr7w+F8S9ziyw= github.com/AlecAivazis/survey/v2 v2.3.6/go.mod h1:4AuI9b7RjAR+G7v9+C4YSlX/YL3K3cWNXgWXOhllqvI= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= @@ -62,10 +66,15 @@ github.com/bcicen/jstream v1.0.1 h1:BXY7Cu4rdmc0rhyTVyT3UkxAiX3bnLpKLas9btbH5ck= github.com/bcicen/jstream v1.0.1/go.mod h1:9ielPxqFry7Y4Tg3j4BfjPocfJ3TbsRtXOAYXYmRuAQ= github.com/bmatcuk/doublestar v1.1.5 h1:2bNwBOmhyFEFcoB3tGvTD5xanq+4kyOZlB8wFYbMjkk= github.com/bmatcuk/doublestar v1.1.5/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= -github.com/bmatcuk/doublestar/v4 v4.6.0 h1:HTuxyug8GyFbRkrffIpzNCSK4luc0TY3wzXvzIZhEXc= -github.com/bmatcuk/doublestar/v4 v4.6.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= +github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b h1:L/QXpzIa3pOvUGt1D1lA5KjYhPBAN/3iWdP7xeFS9F0= +github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= +github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao= +github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -82,9 +91,17 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76/go.mod h1:vYwsqCOLxGiisLwp9rITslkFNpZD5rz43tf41QFkTWY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/djherbis/buffer v1.1.0/go.mod h1:VwN8VdFkMY0DCALdY8o00d3IZ6Amz/UNVMWcSaJT44o= +github.com/djherbis/buffer v1.2.0 h1:PH5Dd2ss0C7CRRhQCZ2u7MssF+No9ide8Ye71nPHcrQ= +github.com/djherbis/buffer v1.2.0/go.mod h1:fjnebbZjCUpPinBRD+TDwXSOeNQ7fPQWLfGQqiAiUyE= +github.com/djherbis/nio/v3 v3.0.1 h1:6wxhnuppteMa6RHA4L81Dq7ThkZH8SwnDzXDYy95vB4= +github.com/djherbis/nio/v3 v3.0.1/go.mod h1:Ng4h80pbZFMla1yKzm61cF0tqqilXZYrogmWgZxOcmg= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v24.0.2+incompatible h1:eATx+oLz9WdNVkQrr0qjQ8HvRJ4bOOxfzEo8R+dA3cg= @@ -93,6 +110,9 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 h1:RIB4cRk+lBqKK3Oy0r2gRX4ui7tuhiZq2SuTtTCi0/0= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= @@ -104,7 +124,16 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= +github.com/go-enry/go-enry/v2 v2.8.3 h1:BwvNrN58JqBJhyyVdZSl5QD3xoxEEGYUrRyPh31FGhw= +github.com/go-enry/go-enry/v2 v2.8.3/go.mod h1:GVzIiAytiS5uT/QiuakK7TF1u4xDab87Y8V5EJRpsIQ= +github.com/go-enry/go-oniguruma v1.2.1 h1:k8aAMuJfMrqm/56SG2lV9Cfti6tC4x8673aHCcBk+eo= +github.com/go-enry/go-oniguruma v1.2.1/go.mod h1:bWDhYP+S6xZQgiRL7wlTScFYBe023B6ilRZbCAD5Hf4= +github.com/go-fed/httpsig v1.1.1-0.20201223112313-55836744818e h1:oRq/fiirun5HqlEWMLIcDmLpIELlG4iGbd0s8iqgPi8= +github.com/go-fed/httpsig v1.1.1-0.20201223112313-55836744818e/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4= @@ -115,9 +144,15 @@ github.com/go-git/go-git/v5 v5.7.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhc github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= +github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -145,6 +180,10 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -159,6 +198,7 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -172,6 +212,8 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b h1:8htHrh2bw9c7Idkb7YNac+ZpTqLMjRpI+FWu51ltaQc= +github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= @@ -179,24 +221,39 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg= +github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/hashicorp/go-envparse v0.1.0 h1:bE++6bhIsNCPLvgDZkYqo3nA+/PFI51pkrHdmPSDFPY= github.com/hashicorp/go-envparse v0.1.0/go.mod h1:OHheN1GoygLlAkTlXLXvAdnXdZxy8JUweQ1rAXx1xnc= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4= +github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl/v2 v2.17.0 h1:z1XvSUyXd1HP10U4lrLg5e0JMVz6CPaJvAgxM0KNZVY= github.com/hashicorp/hcl/v2 v2.17.0/go.mod h1:gJyW2PTShkJqQBKpAmPO3yxMxIuoXkOF2TpqXzrQyx4= github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jdxcode/netrc v0.0.0-20221124155335-4616370d1a84 h1:2uT3aivO7NVpUPGcQX7RbHijHMyWix/yCnIrCWc+5co= +github.com/jdxcode/netrc v0.0.0-20221124155335-4616370d1a84/go.mod h1:Zi/ZFkEqFHTm7qkjyNJjaWH4LQA9LQhGJyF0lTYGpxw= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kendru/darwin/go/depgraph v0.0.0-20221105232959-877d6a81060c h1:eKb4PqwAMhlqwXw0W3atpKaYaPGlXE/Fwh+xpCEYaPk= @@ -209,12 +266,14 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lunny/log v0.0.0-20160921050905-7887c61bf0de/go.mod h1:3q8WtuPQsoRbatJuy3nvq/hRSvuBJrHHr+ybPPiNvHQ= +github.com/lunny/nodb v0.0.0-20160621015157-fc1ef06ad4af/go.mod h1:Cqz6pqow14VObJ7peltM+2n3PWOz7yTrfUuGbVFkzN0= github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -224,6 +283,9 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= +github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -234,13 +296,28 @@ github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vyg github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc3 h1:fzg1mXZFj8YdPeNkRXMg+zb88BFV0Ys52cJydRwBkb8= github.com/opencontainers/image-spec v1.1.0-rc3/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -249,16 +326,27 @@ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qR github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/redis/go-redis/v9 v9.0.3 h1:+7mmR26M0IvyLxGZUHxu4GiBkJkVDid0Un+j4ScYu4k= +github.com/redis/go-redis/v9 v9.0.3/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= -github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw= +github.com/siddontang/go-snappy v0.0.0-20140704025258-d8f7bb82a96d/go.mod h1:vq0tzqLRu6TS7Id0wMo2N5QzJoKedVeovOpHjnykSzY= +github.com/siddontang/ledisdb v0.0.0-20190202134119-8ceb77e66a92/go.mod h1:mF1DpOSOUiJRMR+FDqaqu3EBqrybQtrDDszLUZ6oxPg= +github.com/siddontang/rdb v0.0.0-20150307021120-fc89ed2e418d/go.mod h1:AMEsy7v5z92TR1JKMkLLoaOQk++LVnOKL3ScbJ8GNGA= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/skeema/knownhosts v1.1.1 h1:MTk78x9FPgDFVFkDLTrsnnfCJl7g1C/nnKvePgrIngE= github.com/skeema/knownhosts v1.1.1/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 h1:Jpy1PXuP99tXNrhbq2BaPz9B+jNAvH1JPQQpG/9GCXY= +github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= +github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= +github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= github.com/srevinsaju/logrus v1.10.2-0.20230616201049-4e5a7636a82f h1:fqDRPqbyG4oiL4x03rD5xxnOPTuKScc1Azs3e6xVFb4= @@ -267,6 +355,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -275,6 +364,11 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/unknwon/com v0.0.0-20190804042917-757f69c95f3e/go.mod h1:tOOxU81rwgoCLoOVVPHb6T/wt8HZygqH5id+GNnlCXM= +github.com/unknwon/com v1.0.1 h1:3d1LTxD+Lnf3soQiD4Cp/0BRB+Rsa/+RTvz8GMMzIXs= +github.com/unknwon/com v1.0.1/go.mod h1:tOOxU81rwgoCLoOVVPHb6T/wt8HZygqH5id+GNnlCXM= github.com/urfave/cli/v2 v2.25.5 h1:d0NIAyhh5shGscroL7ek/Ya9QYQE0KNabJgiUinIQkc= github.com/urfave/cli/v2 v2.25.5/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= @@ -286,6 +380,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.5.3 h1:3HUJmBFbQW9fhQOzMgseU134xfi6hU+mjWywx5Ty+/M= +github.com/yuin/goldmark v1.5.3/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zclconf/go-cty v1.13.0 h1:It5dfKTTZHe9aeppbNOda3mN7Ag7sg6QkBNm6TkyFa0= github.com/zclconf/go-cty v1.13.0/go.mod h1:YKQzy/7pZ7iq2jNFzy5go57xdxdWoLLpaEp4u238AE0= github.com/zclconf/go-cty-yaml v1.0.3 h1:og/eOQ7lvA/WWhHGFETVWNduJM7Rjsv2RRpx1sdFMLc= @@ -347,6 +443,7 @@ golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -405,6 +502,7 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -414,7 +512,9 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190730183949-1393eb018365/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -476,13 +576,13 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -625,13 +725,23 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/ini.v1 v1.44.2/go.mod h1:M3Cogqpuv0QCi3ExAY5V4uOt4qb/R3xZubo9m8lK5wg= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/blocks/data/git.go b/pkg/blocks/data/git.go index 12ee8ca..6e2b410 100644 --- a/pkg/blocks/data/git.go +++ b/pkg/blocks/data/git.go @@ -1,22 +1,31 @@ package data import ( + "bytes" + "code.gitea.io/gitea/modules/git" "context" + "errors" "fmt" "github.com/go-git/go-billy/v5/memfs" - "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/go-git/go-git/v5/plumbing/transport/ssh" "github.com/go-git/go-git/v5/storage" "github.com/go-git/go-git/v5/storage/memory" "github.com/hashicorp/hcl/v2" + "github.com/jdxcode/netrc" "github.com/sirupsen/logrus" "github.com/srevinsaju/togomak/v1/pkg/c" "github.com/srevinsaju/togomak/v1/pkg/ui" "github.com/zclconf/go-cty/cty" "io" + "net/url" + "os" + "os/user" + "path/filepath" + "strings" ) type gitProviderAuthConfig struct { @@ -178,6 +187,7 @@ func (e *GitProvider) DecodeBody(body hcl.Body) hcl.Diagnostics { password: authPassword.AsString(), sshPassword: authSshPassword.AsString(), sshPrivateKey: authSshPrivateKey.AsString(), + isSsh: authSshPassword.AsString() != "" || authSshPrivateKey.AsString() != "", } } @@ -295,6 +305,7 @@ func (e *GitProvider) Attributes(ctx context.Context) (map[string]cty.Value, hcl // git clone var s storage.Storer var authMethod transport.AuthMethod + repoUrl := e.cfg.repo if e.cfg.auth.password != "" { authMethod = &http.BasicAuth{ @@ -312,15 +323,49 @@ func (e *GitProvider) Attributes(ctx context.Context) (map[string]cty.Value, hcl return nil, diags } authMethod = publicKeys + } else { + // fallback to inferring it from the environment + u, err := url.Parse(repoUrl) + if err != nil { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "url parse error", + Detail: err.Error(), + }) + return nil, diags + } + if u.Scheme == "http" || u.Scheme == "https" { + usr, err := user.Current() + if err != nil { + panic(err) + } + homeDir := usr.HomeDir + gitCredentialsFilePath := filepath.Join(homeDir, ".git-credentials") + gitCredentialsFilePathDir := filepath.Join(homeDir, ".git", "credentials") + netrcFilePath := filepath.Join(homeDir, ".netrc") + if _, err := os.Stat(gitCredentialsFilePath); err == nil { + // TODO: we will replace this will the implicit auth method + // once go-git implements it + authMethod = parseGitCredentialsFile(logger, gitCredentialsFilePath, u) + } else if _, err := os.Stat(gitCredentialsFilePathDir); err == nil { + authMethod = parseGitCredentialsFile(logger, gitCredentialsFilePathDir, u) + } else if _, err := os.Stat(netrcFilePath); err == nil { + authMethod = parseNetrcFile(logger, netrcFilePath, u) + } + } else if u.Scheme == "ssh" || u.Scheme == "git" || u.Scheme == "" { + authMethod = nil + } + } cloneOptions := &git.CloneOptions{ - Tags: git.AllTags, - Depth: e.cfg.depth, - CABundle: e.cfg.caBundle, - Auth: authMethod, - URL: e.cfg.repo, - Progress: nil, + Tags: git.AllTags, + Depth: e.cfg.depth, + CABundle: e.cfg.caBundle, + RecurseSubmodules: git.DefaultSubmoduleRecursionDepth, + Auth: authMethod, + URL: repoUrl, + Progress: nil, } var repo *git.Repository var err error @@ -380,11 +425,45 @@ func (e *GitProvider) Attributes(ctx context.Context) (map[string]cty.Value, hcl Detail: err.Error(), }) } else { - for commit, err := commitIter.Next(); err != nil; { - if commit.Type() == plumbing.TagObject { - lastTag = commit.Hash.String() + latestTag, latestTagCommit, err := GetLatestTagFromRepository(repo) + if err != nil { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "git tags failed", + Detail: err.Error(), + }) + return nil, diags + } + if latestTag != nil { + lastTag = latestTag.Name().Short() + } + + logger.Debugf("latest tag is %s", lastTag) + logger.Debugf("iterating over commits...") + foundTagError := errors.New("found tag") + + if latestTagCommit != nil { + commitIterErr := commitIter.ForEach(func(commit *object.Commit) error { + ref, err := repo.Reference(latestTag.Name(), true) + if err != nil { + return err + } + + commitRef, err := repo.CommitObject(ref.Hash()) + if err != nil { + return err + } + + fmt.Println("checking commits", commit.Hash.String(), "==", latestTagCommit.Hash.String(), ref.Hash().String(), commitRef.Hash.String()) + if latestTagCommit.Hash == commit.Hash { + return foundTagError + } + count++ + return nil + }) + if foundTagError != commitIterErr && commitIterErr != nil { + logger.Warn(commitIterErr) } - count++ } } commitsSinceLastTag := cty.NumberIntVal(int64(count)) @@ -474,3 +553,100 @@ func (e *GitProvider) Attributes(ctx context.Context) (map[string]cty.Value, hcl // get the commit return attrs, diags } + +func parseNetrcFile(log *logrus.Entry, path string, u *url.URL) transport.AuthMethod { + logger := log.WithField("auth", "netrc") + n, err := netrc.Parse(path) + if err != nil { + logger.Warn("failed to parse netrc file: ", err) + return nil + } + username := n.Machine(u.Host).Get("login") + password := n.Machine(u.Host).Get("password") + authMethod := &http.BasicAuth{ + Username: username, + Password: password, + } + return authMethod +} + +func parseGitCredentialsFile(log *logrus.Entry, path string, u *url.URL) transport.AuthMethod { + logger := log.WithField("auth", "git-credentials") + data, err := os.ReadFile(path) + if err != nil { + logger.Warn("failed to read git-credentials file: ", err) + return nil + } + + data = bytes.TrimSpace(data) + lines := bytes.Split(data, []byte("\n")) + for _, lineRaw := range lines { + lineRaw = bytes.TrimSpace(lineRaw) + line := string(lineRaw) + if strings.HasPrefix(line, "#") { + logger.Trace("skipping comment: ", line) + continue + } + credentialUrl, err := url.Parse(line) + if err != nil { + logger.Warn("failed to parse git-credentials line: ", err) + continue + } + + if credentialUrl.Host == u.Host && credentialUrl.Scheme == u.Scheme && strings.HasPrefix(u.Path, credentialUrl.Path) { + username := credentialUrl.User.Username() + password, ok := credentialUrl.User.Password() + if !ok { + logger.Warn("failed to retrieve password from git-credentials file, falling back to '': ", line) + } + authMethod := &http.BasicAuth{ + Username: username, + Password: password, + } + return authMethod + } + } + return nil + +} + +// GetLatestTagFromRepository returns the latest tag from a git repository +// https://github.com/src-d/go-git/issues/1030#issuecomment-443679681 +func GetLatestTagFromRepository(repository *git.Repository) (*plumbing.Reference, *object.Commit, error) { + tagRefs, err := repository.Tags() + if err != nil { + return nil, nil, err + } + var commit *object.Commit + var latestTagCommit *object.Commit + var latestTagName *plumbing.Reference + err = tagRefs.ForEach(func(tagRef *plumbing.Reference) error { + revision := plumbing.Revision(tagRef.Name().String()) + tagCommitHash, err := repository.ResolveRevision(revision) + if err != nil { + return err + } + + commit, err = repository.CommitObject(*tagCommitHash) + if err != nil { + return err + } + + if latestTagCommit == nil { + latestTagCommit = commit + latestTagName = tagRef + } + + if commit.Committer.When.After(latestTagCommit.Committer.When) { + latestTagCommit = commit + latestTagName = tagRef + } + + return nil + }) + if err != nil { + return nil, commit, err + } + + return latestTagName, commit, nil +} diff --git a/pkg/c/context.go b/pkg/c/context.go index f2179dd..dbc4bdb 100644 --- a/pkg/c/context.go +++ b/pkg/c/context.go @@ -21,6 +21,7 @@ const ( TogomakContextCi = "ci" TogomakContextUnattended = "unattended" + TogomakContextVerbosity = "verbosity" TogomakContextLogger = "logger" TogomakContextHclDiagWriter = "hcl_diag_writer" diff --git a/pkg/ci/stage_run.go b/pkg/ci/stage_run.go index 18acc56..26dd903 100644 --- a/pkg/ci/stage_run.go +++ b/pkg/ci/stage_run.go @@ -57,6 +57,7 @@ func (s *Stage) expandMacros(ctx context.Context) (*Stage, hcl.Diagnostics) { logger := ctx.Value(c.TogomakContextLogger).(*logrus.Logger).WithField(StageBlock, s.Id).WithField(MacroBlock, true) pipe := ctx.Value(c.TogomakContextPipeline).(*Pipeline) cwd := ctx.Value(c.TogomakContextCwd).(string) + tmpDir := ctx.Value(c.TogomakContextTempDir).(string) ci := ctx.Value(c.TogomakContextCi).(bool) unattended := ctx.Value(c.TogomakContextUnattended).(bool) @@ -211,7 +212,7 @@ func (s *Stage) expandMacros(ctx context.Context) (*Stage, hcl.Diagnostics) { args = append(args, cty.StringVal("--unattended")) } childStatuses := s.Get(StageContextChildStatuses).([]string) - fmt.Println(childStatuses) + logger.Trace("child statuses: ", childStatuses) if childStatuses != nil { var ctyChildStatuses []cty.Value for _, childStatus := range childStatuses { From 44ee141d0d12645af44210d058c72234da3ab80d Mon Sep 17 00:00:00 2001 From: Srevin Saju Date: Thu, 22 Jun 2023 17:34:36 +0530 Subject: [PATCH 4/6] fix: move to gitea's shell based go library --- go.mod | 10 +- go.sum | 3 - pkg/blocks/data/env.go | 2 +- pkg/blocks/data/git.go | 480 +++++++++++++----------------------- pkg/blocks/data/prompt.go | 2 +- pkg/blocks/data/provider.go | 2 +- pkg/ci/data_run.go | 2 +- 7 files changed, 178 insertions(+), 323 deletions(-) diff --git a/go.mod b/go.mod index 400082e..961ddaf 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/srevinsaju/togomak/v1 go 1.20 require ( + code.gitea.io/gitea v1.19.3 github.com/AlecAivazis/survey/v2 v2.3.6 github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 github.com/alessio/shellescape v1.4.1 @@ -12,15 +13,12 @@ require ( github.com/docker/docker v24.0.2+incompatible github.com/docker/go-connections v0.4.0 github.com/fatih/color v1.15.0 - github.com/go-git/go-billy/v5 v5.4.1 - github.com/go-git/go-git/v5 v5.7.0 github.com/google/uuid v1.3.0 github.com/hashicorp/go-envparse v0.1.0 github.com/hashicorp/go-uuid v1.0.3 github.com/hashicorp/hcl/v2 v2.17.0 github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec github.com/imdario/mergo v0.3.16 - github.com/jdxcode/netrc v0.0.0-20221124155335-4616370d1a84 github.com/kendru/darwin/go/depgraph v0.0.0-20221105232959-877d6a81060c github.com/mattn/go-isatty v0.0.17 github.com/mitchellh/go-homedir v1.1.0 @@ -36,7 +34,6 @@ require ( ) require ( - code.gitea.io/gitea v1.19.3 // indirect gitea.com/go-chi/cache v0.2.0 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 // indirect @@ -59,6 +56,8 @@ require ( github.com/go-enry/go-oniguruma v1.2.1 // indirect github.com/go-fed/httpsig v1.1.1-0.20201223112313-55836744818e // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.4.1 // indirect + github.com/go-git/go-git/v5 v5.7.0 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.4.3 // indirect @@ -72,7 +71,6 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/kr/pretty v0.3.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-sqlite3 v1.14.16 // indirect github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect @@ -81,6 +79,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/morikuni/aec v1.0.0 // indirect + github.com/nxadm/tail v1.4.8 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc3 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect @@ -99,7 +98,6 @@ require ( golang.org/x/net v0.10.0 // indirect golang.org/x/sys v0.8.0 // indirect golang.org/x/term v0.8.0 // indirect - golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.7.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/go.sum b/go.sum index ac18948..b4358c3 100644 --- a/go.sum +++ b/go.sum @@ -245,8 +245,6 @@ github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/jdxcode/netrc v0.0.0-20221124155335-4616370d1a84 h1:2uT3aivO7NVpUPGcQX7RbHijHMyWix/yCnIrCWc+5co= -github.com/jdxcode/netrc v0.0.0-20221124155335-4616370d1a84/go.mod h1:Zi/ZFkEqFHTm7qkjyNJjaWH4LQA9LQhGJyF0lTYGpxw= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -725,7 +723,6 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= diff --git a/pkg/blocks/data/env.go b/pkg/blocks/data/env.go index 8c86848..7056eaf 100644 --- a/pkg/blocks/data/env.go +++ b/pkg/blocks/data/env.go @@ -48,7 +48,7 @@ func (e *EnvProvider) New() Provider { } } -func (e *EnvProvider) Attributes(ctx context.Context) (map[string]cty.Value, hcl.Diagnostics) { +func (e *EnvProvider) Attributes(ctx context.Context, id string) (map[string]cty.Value, hcl.Diagnostics) { return map[string]cty.Value{ EnvProviderAttrKey: cty.StringVal(e.keyParsed), EnvProviderAttrDefault: cty.StringVal(e.def), diff --git a/pkg/blocks/data/git.go b/pkg/blocks/data/git.go index 6e2b410..cded776 100644 --- a/pkg/blocks/data/git.go +++ b/pkg/blocks/data/git.go @@ -1,31 +1,18 @@ package data import ( - "bytes" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" "context" - "errors" "fmt" - "github.com/go-git/go-billy/v5/memfs" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" - "github.com/go-git/go-git/v5/plumbing/transport" - "github.com/go-git/go-git/v5/plumbing/transport/http" - "github.com/go-git/go-git/v5/plumbing/transport/ssh" - "github.com/go-git/go-git/v5/storage" - "github.com/go-git/go-git/v5/storage/memory" "github.com/hashicorp/hcl/v2" - "github.com/jdxcode/netrc" "github.com/sirupsen/logrus" "github.com/srevinsaju/togomak/v1/pkg/c" "github.com/srevinsaju/togomak/v1/pkg/ui" "github.com/zclconf/go-cty/cty" - "io" "net/url" "os" - "os/user" "path/filepath" - "strings" ) type gitProviderAuthConfig struct { @@ -65,12 +52,10 @@ const ( GitBlockAttrLastTag = "last_tag" GitBlockAttrCommitsSinceLastTag = "commits_since_last_tag" GitBlockAttrSha = "sha" - GitBlockAttrRef = "ref" - GitBlockAttrIsTag = "is_tag" - GitBlockAttrIsBranch = "is_branch" - GitBlockAttrIsNote = "is_note" - GitBlockAttrIsRemote = "is_remote" + GitBlockAttrIsTag = "is_tag" + GitBlockAttrRef = "ref" + GitBlockAttrFiles = "files" ) type GitProvider struct { @@ -212,6 +197,11 @@ func (e *GitProvider) DecodeBody(body hcl.Body) hcl.Diagnostics { return diags } +func init() { + h, _ := os.UserHomeDir() + setting.Git.HomePath = h +} + func (e *GitProvider) New() Provider { return &GitProvider{ initialized: true, @@ -292,7 +282,7 @@ func (e *GitProvider) Value(ctx context.Context, id string) (string, hcl.Diagnos return "", nil } -func (e *GitProvider) Attributes(ctx context.Context) (map[string]cty.Value, hcl.Diagnostics) { +func (e *GitProvider) Attributes(ctx context.Context, id string) (map[string]cty.Value, hcl.Diagnostics) { logger := ctx.Value(c.TogomakContextLogger).(*logrus.Logger).WithField("provider", e.Name()) var diags hcl.Diagnostics if !e.initialized { @@ -301,352 +291,222 @@ func (e *GitProvider) Attributes(ctx context.Context) (map[string]cty.Value, hcl var attrs = make(map[string]cty.Value) - // clone git repo - // git clone - var s storage.Storer - var authMethod transport.AuthMethod - repoUrl := e.cfg.repo + var cloneComplete = make(chan bool) + go e.clonePassiveProgressBar(logger, cloneComplete) + + opts := git.CloneRepoOptions{ + Depth: e.cfg.depth, + Branch: e.cfg.branch, + Bare: false, + // TODO: SkipTLSVerify: e.cfg.skipTLSVerify, + // TODO: make it configurable + Quiet: true, + } - if e.cfg.auth.password != "" { - authMethod = &http.BasicAuth{ - Username: e.cfg.auth.username, - Password: e.cfg.auth.password, - } - } else if e.cfg.auth.isSsh { - publicKeys, err := ssh.NewPublicKeys(e.cfg.auth.username, []byte(e.cfg.auth.sshPrivateKey), e.cfg.auth.sshPassword) - if err != nil { - diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "ssh key error", - Detail: err.Error(), - }) - return nil, diags - } - authMethod = publicKeys - } else { - // fallback to inferring it from the environment - u, err := url.Parse(repoUrl) - if err != nil { - diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "url parse error", - Detail: err.Error(), - }) - return nil, diags - } - if u.Scheme == "http" || u.Scheme == "https" { - usr, err := user.Current() - if err != nil { - panic(err) - } - homeDir := usr.HomeDir - gitCredentialsFilePath := filepath.Join(homeDir, ".git-credentials") - gitCredentialsFilePathDir := filepath.Join(homeDir, ".git", "credentials") - netrcFilePath := filepath.Join(homeDir, ".netrc") - if _, err := os.Stat(gitCredentialsFilePath); err == nil { - // TODO: we will replace this will the implicit auth method - // once go-git implements it - authMethod = parseGitCredentialsFile(logger, gitCredentialsFilePath, u) - } else if _, err := os.Stat(gitCredentialsFilePathDir); err == nil { - authMethod = parseGitCredentialsFile(logger, gitCredentialsFilePathDir, u) - } else if _, err := os.Stat(netrcFilePath); err == nil { - authMethod = parseNetrcFile(logger, netrcFilePath, u) - } - } else if u.Scheme == "ssh" || u.Scheme == "git" || u.Scheme == "" { - authMethod = nil - } + // TODO: implement git submodules + destination, d := e.resolveDestination(ctx, id) + diags = diags.Extend(d) + if diags.HasErrors() { + return nil, diags } - cloneOptions := &git.CloneOptions{ - Tags: git.AllTags, - Depth: e.cfg.depth, - CABundle: e.cfg.caBundle, - RecurseSubmodules: git.DefaultSubmoduleRecursionDepth, - Auth: authMethod, - URL: repoUrl, - Progress: nil, + repoUrl, err := url.Parse(e.cfg.repo) + if err != nil { + return nil, diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "failed to parse git repo url", + Detail: err.Error(), + }) } - var repo *git.Repository - var err error - var cloneComplete = make(chan bool) - go func() { - pb := ui.NewProgressWriter(logger, fmt.Sprintf("pulling git repo %s", e.Identifier())) - for { - select { - case <-cloneComplete: - pb.Close() - return - default: - pb.Write([]byte("1")) - } - } - }() - if e.cfg.destination == "" || e.cfg.destination == "memory" { - logger.Debug("cloning into memory storage") - s = memory.NewStorage() - repo, err = git.CloneContext(ctx, s, memfs.New(), cloneOptions) - } else { - logger.Debugf("cloning to %s", e.cfg.destination) - repo, err = git.PlainCloneContext(ctx, e.cfg.destination, false, cloneOptions) + if e.cfg.auth.username != "" || e.cfg.auth.password != "" { + username := e.cfg.auth.username + if e.cfg.auth.username == "" { + username = "oauth2" + } + repoUrl.User = url.UserPassword(username, e.cfg.auth.password) } - cloneComplete <- true + logger.Debugf("cloning git repo to %s", destination) + err = git.CloneWithArgs(ctx, nil, repoUrl.String(), destination, opts) + cloneComplete <- true if err != nil { - diags = diags.Append(&hcl.Diagnostic{ + return nil, diags.Append(&hcl.Diagnostic{ Severity: hcl.DiagError, - Summary: "git clone failed", + Summary: "failed to clone git repo", Detail: err.Error(), }) - return nil, diags } - - w, err := repo.Worktree() + repo, closer, err := git.RepositoryFromContextOrOpen(ctx, destination) if err != nil { - diags = diags.Append(&hcl.Diagnostic{ + return nil, diags.Append(&hcl.Diagnostic{ Severity: hcl.DiagError, - Summary: "checkout failed", + Summary: "failed to open git repo", Detail: err.Error(), }) - return nil, diags } - commitIter, err := repo.Log(&git.LogOptions{ - Order: git.LogOrderCommitterTime, - }) + gitBranch, err := repo.GetHEADBranch() + if err != nil { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagWarning, + Summary: "failed to get git branch", + Detail: err.Error(), + }) + } + branch := "" + if gitBranch != nil { + branch = gitBranch.Name + } - count := 0 lastTag := "" + tags, err := repo.GetTags(0, 1) if err != nil { diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "git log failed", + Severity: hcl.DiagWarning, + Summary: "failed to get git tags", Detail: err.Error(), }) - } else { - latestTag, latestTagCommit, err := GetLatestTagFromRepository(repo) - if err != nil { - diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "git tags failed", - Detail: err.Error(), - }) - return nil, diags - } - if latestTag != nil { - lastTag = latestTag.Name().Short() - } - - logger.Debugf("latest tag is %s", lastTag) - logger.Debugf("iterating over commits...") - foundTagError := errors.New("found tag") - - if latestTagCommit != nil { - commitIterErr := commitIter.ForEach(func(commit *object.Commit) error { - ref, err := repo.Reference(latestTag.Name(), true) - if err != nil { - return err - } - - commitRef, err := repo.CommitObject(ref.Hash()) - if err != nil { - return err - } - - fmt.Println("checking commits", commit.Hash.String(), "==", latestTagCommit.Hash.String(), ref.Hash().String(), commitRef.Hash.String()) - if latestTagCommit.Hash == commit.Hash { - return foundTagError - } - count++ - return nil - }) - if foundTagError != commitIterErr && commitIterErr != nil { - logger.Warn(commitIterErr) - } - } } - commitsSinceLastTag := cty.NumberIntVal(int64(count)) + noTags := false + if len(tags) == 0 { + noTags = true + } else if len(tags) > 1 { + panic("more than 1 tag returned when only one was supposed to be returned") + } + commitsSinceLastTag := cty.NullVal(cty.Number) + if !noTags { + lastTag = tags[0] - var files = make(map[string]cty.Value) - for _, f := range e.cfg.files { - _, err := w.Filesystem.Stat(f) + commitCount, err := repo.CommitsCountBetween(lastTag, "HEAD") if err != nil { diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "git file search failed", - Detail: err.Error(), - }) - continue - } - file, err := w.Filesystem.Open(f) - if err != nil { - diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "git file open failed", + Severity: hcl.DiagWarning, + Summary: "failed to get commits since last tag", Detail: err.Error(), }) - continue + } else { + commitsSinceLastTag = cty.NumberIntVal(commitCount) } - defer file.Close() - data, err := io.ReadAll(file) - if err != nil { - diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "git file read failed", - Detail: err.Error(), - }) - continue - } - files[f] = cty.StringVal(string(data[:])) } - if files == nil || len(files) == 0 { - attrs[GitBlockArgumentFiles] = cty.MapValEmpty(cty.String) - } else { - attrs[GitBlockArgumentFiles] = cty.MapVal(files) + sha, err := repo.ConvertToSHA1("HEAD") + if err != nil { + return nil, diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "failed to get git sha", + Detail: err.Error(), + }) } - attrs[GitBlockArgumentUrl] = cty.StringVal(e.cfg.repo) - attrs[GitBlockArgumentTag] = cty.StringVal(e.cfg.tag) - - ref, err := repo.Head() - refString := cty.StringVal("") - branch := cty.StringVal("") - tag := cty.StringVal("") - isBranch := cty.False - isTag := cty.False - isRemote := cty.False - isNote := cty.False - sha := cty.StringVal("") + err = closer.Close() + if err != nil { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagWarning, + Summary: "failed to close git repo", + Detail: err.Error(), + }) + } + ref, err := repo.ResolveReference("HEAD") if err != nil { diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "git head failed", + Severity: hcl.DiagWarning, + Summary: "failed to resolve git reference", + Detail: err.Error(), + }) + } + + _, err = repo.GetTagNameBySHA(sha.String()) + isTagResolved := cty.NullVal(cty.Bool) + if git.IsErrNotExist(err) { + isTagResolved = cty.False + } else if err != nil { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagWarning, + Summary: "failed to resolve git tag", Detail: err.Error(), }) + } else { + isTagResolved = cty.True + } + + // read files and store them in the map if whitelisted + files := make(map[string]cty.Value) + for _, file := range e.cfg.files { + f := filepath.Join(destination, file) + if _, err := os.Stat(f); err == nil { + content, err := os.ReadFile(f) + if err != nil { + return nil, diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "failed to read file", + Detail: err.Error(), + }) + } + files[file] = cty.StringVal(string(content)) + } else if !os.IsNotExist(err) { + return nil, diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "failed to read file", + Detail: err.Error(), + }) + } + } + var filesCty cty.Value + if len(files) > 0 { + filesCty = cty.MapVal(files) } else { - refString = cty.StringVal(ref.Name().String()) - branch = cty.StringVal(ref.Name().Short()) - isBranch = cty.BoolVal(ref.Name().IsBranch()) - isTag = cty.BoolVal(ref.Name().IsTag()) - tag = cty.StringVal(ref.Name().Short()) - isRemote = cty.BoolVal(ref.Name().IsRemote()) - isNote = cty.BoolVal(ref.Name().IsNote()) - sha = cty.StringVal(ref.Hash().String()) - } - - attrs[GitBlockArgumentBranch] = branch - attrs[GitBlockArgumentTag] = tag - attrs[GitBlockAttrIsBranch] = isBranch - attrs[GitBlockAttrRef] = refString - attrs[GitBlockAttrIsTag] = isTag - attrs[GitBlockAttrIsRemote] = isRemote - attrs[GitBlockAttrIsNote] = isNote - attrs[GitBlockAttrSha] = sha + filesCty = cty.NullVal(cty.Map(cty.String)) + } + + attrs[GitBlockArgumentBranch] = cty.StringVal(branch) + attrs[GitBlockArgumentTag] = cty.StringVal(e.cfg.tag) + attrs[GitBlockAttrIsTag] = isTagResolved + attrs[GitBlockAttrRef] = cty.StringVal(ref) + attrs[GitBlockArgumentUrl] = cty.StringVal(e.cfg.repo) + attrs[GitBlockAttrSha] = cty.StringVal(sha.String()) attrs[GitBlockAttrLastTag] = cty.StringVal(lastTag) attrs[GitBlockAttrCommitsSinceLastTag] = commitsSinceLastTag + attrs[GitBlockAttrFiles] = filesCty // get the commit return attrs, diags } -func parseNetrcFile(log *logrus.Entry, path string, u *url.URL) transport.AuthMethod { - logger := log.WithField("auth", "netrc") - n, err := netrc.Parse(path) - if err != nil { - logger.Warn("failed to parse netrc file: ", err) - return nil - } - username := n.Machine(u.Host).Get("login") - password := n.Machine(u.Host).Get("password") - authMethod := &http.BasicAuth{ - Username: username, - Password: password, - } - return authMethod -} - -func parseGitCredentialsFile(log *logrus.Entry, path string, u *url.URL) transport.AuthMethod { - logger := log.WithField("auth", "git-credentials") - data, err := os.ReadFile(path) - if err != nil { - logger.Warn("failed to read git-credentials file: ", err) - return nil - } - - data = bytes.TrimSpace(data) - lines := bytes.Split(data, []byte("\n")) - for _, lineRaw := range lines { - lineRaw = bytes.TrimSpace(lineRaw) - line := string(lineRaw) - if strings.HasPrefix(line, "#") { - logger.Trace("skipping comment: ", line) - continue - } - credentialUrl, err := url.Parse(line) - if err != nil { - logger.Warn("failed to parse git-credentials line: ", err) - continue - } - - if credentialUrl.Host == u.Host && credentialUrl.Scheme == u.Scheme && strings.HasPrefix(u.Path, credentialUrl.Path) { - username := credentialUrl.User.Username() - password, ok := credentialUrl.User.Password() - if !ok { - logger.Warn("failed to retrieve password from git-credentials file, falling back to '': ", line) - } - authMethod := &http.BasicAuth{ - Username: username, - Password: password, - } - return authMethod +func (e *GitProvider) clonePassiveProgressBar(logger *logrus.Entry, cloneComplete chan bool) { + pb := ui.NewProgressWriter(logger, fmt.Sprintf("pulling git repo %s", e.Identifier())) + for { + select { + case <-cloneComplete: + pb.Close() + return + default: + pb.Write([]byte("1")) } } - return nil - } -// GetLatestTagFromRepository returns the latest tag from a git repository -// https://github.com/src-d/go-git/issues/1030#issuecomment-443679681 -func GetLatestTagFromRepository(repository *git.Repository) (*plumbing.Reference, *object.Commit, error) { - tagRefs, err := repository.Tags() - if err != nil { - return nil, nil, err - } - var commit *object.Commit - var latestTagCommit *object.Commit - var latestTagName *plumbing.Reference - err = tagRefs.ForEach(func(tagRef *plumbing.Reference) error { - revision := plumbing.Revision(tagRef.Name().String()) - tagCommitHash, err := repository.ResolveRevision(revision) - if err != nil { - return err - } - - commit, err = repository.CommitObject(*tagCommitHash) - if err != nil { - return err - } - - if latestTagCommit == nil { - latestTagCommit = commit - latestTagName = tagRef - } +func (e *GitProvider) resolveDestination(ctx context.Context, id string) (string, hcl.Diagnostics) { + logger := ctx.Value(c.TogomakContextLogger).(*logrus.Logger).WithField("provider", e.Name()) + tmpDir := ctx.Value(c.TogomakContextTempDir).(string) - if commit.Committer.When.After(latestTagCommit.Committer.When) { - latestTagCommit = commit - latestTagName = tagRef + var diags hcl.Diagnostics + destination := e.cfg.destination + if destination == "" || destination == "memory" { + if e.cfg.destination == "memory" { + // we deprecate this mode, warn the users + logger.Warn("git provider destination is set to memory, this mode is deprecated, currently it writes to a temporary directory") + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagWarning, + Summary: "deprecated git destination", + Detail: "git provider destination is set to memory, this mode is deprecated, currently it writes to a temporary directory", + }) } - - return nil - }) - if err != nil { - return nil, commit, err + destination = filepath.Join(tmpDir, e.Identifier(), id) } - - return latestTagName, commit, nil + return destination, diags } diff --git a/pkg/blocks/data/prompt.go b/pkg/blocks/data/prompt.go index e569517..0d349d3 100644 --- a/pkg/blocks/data/prompt.go +++ b/pkg/blocks/data/prompt.go @@ -88,7 +88,7 @@ func (e *PromptProvider) Schema() *hcl.BodySchema { } } -func (e *PromptProvider) Attributes(ctx context.Context) (map[string]cty.Value, hcl.Diagnostics) { +func (e *PromptProvider) Attributes(ctx context.Context, id string) (map[string]cty.Value, hcl.Diagnostics) { return map[string]cty.Value{ "prompt": cty.StringVal(e.promptParsed), "default": cty.StringVal(e.def), diff --git a/pkg/blocks/data/provider.go b/pkg/blocks/data/provider.go index f5d23dd..c421f61 100644 --- a/pkg/blocks/data/provider.go +++ b/pkg/blocks/data/provider.go @@ -17,7 +17,7 @@ type Provider interface { SetContext(context context.Context) DecodeBody(body hcl.Body) hcl.Diagnostics Value(ctx context.Context, id string) (string, hcl.Diagnostics) - Attributes(ctx context.Context) (map[string]cty.Value, hcl.Diagnostics) + Attributes(ctx context.Context, id string) (map[string]cty.Value, hcl.Diagnostics) } func Variables(e Provider, body hcl.Body) []hcl.Traversal { diff --git a/pkg/ci/data_run.go b/pkg/ci/data_run.go index 3c8bb1b..fff6cbe 100644 --- a/pkg/ci/data_run.go +++ b/pkg/ci/data_run.go @@ -43,7 +43,7 @@ func (s Data) Run(ctx context.Context) hcl.Diagnostics { diags = diags.Extend(provide.DecodeBody(s.Body)) value, d = provide.Value(ctx, s.Id) diags = diags.Extend(d) - attr, d = provide.Attributes(ctx) + attr, d = provide.Attributes(ctx, s.Id) diags = diags.Extend(d) break } From 56724d8815bdf4f34886c07730ea36c17d112156 Mon Sep 17 00:00:00 2001 From: Srevin Saju Date: Thu, 22 Jun 2023 21:34:36 +0530 Subject: [PATCH 5/6] feat: add lifecycle rules for daemon stopping --- pkg/ci/daemon.go | 48 ++++++++ pkg/ci/data.go | 2 +- pkg/ci/data_lifecycle.go | 10 ++ pkg/ci/data_run.go | 4 + pkg/ci/data_test.go | 2 +- pkg/ci/locals.go | 2 +- pkg/ci/locals_lifecycle.go | 10 ++ pkg/ci/locals_run.go | 4 + pkg/ci/locals_test.go | 2 +- pkg/ci/macro.go | 2 +- pkg/ci/macro_lifecycle.go | 10 ++ pkg/ci/macro_run.go | 4 + pkg/ci/runnable.go | 37 +++++- pkg/ci/stage_hcl.go | 11 ++ pkg/ci/stage_lifecycle.go | 13 ++ pkg/ci/stage_run.go | 22 +++- pkg/ci/stage_schema.go | 4 +- pkg/graph/depgraph.go | 244 ++++++++----------------------------- pkg/orchestra/context.go | 1 + pkg/orchestra/interrupt.go | 2 +- pkg/orchestra/orchestra.go | 83 ++++++++++++- 21 files changed, 306 insertions(+), 211 deletions(-) create mode 100644 pkg/ci/daemon.go create mode 100644 pkg/ci/data_lifecycle.go create mode 100644 pkg/ci/locals_lifecycle.go create mode 100644 pkg/ci/macro_lifecycle.go create mode 100644 pkg/ci/stage_lifecycle.go diff --git a/pkg/ci/daemon.go b/pkg/ci/daemon.go new file mode 100644 index 0000000..33e9bef --- /dev/null +++ b/pkg/ci/daemon.go @@ -0,0 +1,48 @@ +package ci + +import ( + "context" + "github.com/hashicorp/hcl/v2" + "github.com/srevinsaju/togomak/v1/pkg/c" +) + +type DaemonLifecycle struct { + StopWhenComplete Blocks +} + +type Lifecycle struct { + StopWhenComplete hcl.Expression `hcl:"stop_when_complete,optional" json:"stop_when_complete"` +} + +func (l *Lifecycle) Parse(ctx context.Context) (*DaemonLifecycle, hcl.Diagnostics) { + pipe := ctx.Value(c.TogomakContextPipeline).(*Pipeline) + daemonLifecycle := &DaemonLifecycle{} + var diags hcl.Diagnostics + + variables := l.StopWhenComplete.Variables() + + var runnableString []string + for _, v := range variables { + data, d := ResolveFromTraversal(v) + diags = diags.Extend(d) + if data == "" || diags.HasErrors() { + continue + } + runnableString = append(runnableString, data) + } + var runnables Blocks + for _, runnableId := range runnableString { + runnable, diags := Resolve(ctx, pipe, runnableId) + if diags.HasErrors() { + return nil, diags + } + runnables = append(runnables, runnable) + } + daemonLifecycle.StopWhenComplete = runnables + + return daemonLifecycle, diags +} + +func (l *Lifecycle) Variables() []hcl.Traversal { + return nil +} diff --git a/pkg/ci/data.go b/pkg/ci/data.go index 7374d02..192f3ac 100644 --- a/pkg/ci/data.go +++ b/pkg/ci/data.go @@ -44,7 +44,7 @@ func (s Data) IsDaemon() bool { return false } -func (s Data) Terminate() hcl.Diagnostics { +func (s Data) Terminate(safe bool) hcl.Diagnostics { return nil } diff --git a/pkg/ci/data_lifecycle.go b/pkg/ci/data_lifecycle.go new file mode 100644 index 0000000..a83da1b --- /dev/null +++ b/pkg/ci/data_lifecycle.go @@ -0,0 +1,10 @@ +package ci + +import ( + "context" + "github.com/hashicorp/hcl/v2" +) + +func (d Data) Lifecycle(ctx context.Context) (*DaemonLifecycle, hcl.Diagnostics) { + return nil, nil +} diff --git a/pkg/ci/data_run.go b/pkg/ci/data_run.go index fff6cbe..78fea7e 100644 --- a/pkg/ci/data_run.go +++ b/pkg/ci/data_run.go @@ -95,3 +95,7 @@ func (s Data) Run(ctx context.Context) hcl.Diagnostics { func (s Data) CanRun(ctx context.Context) (bool, hcl.Diagnostics) { return true, nil } + +func (s Data) Terminated() bool { + return true +} diff --git a/pkg/ci/data_test.go b/pkg/ci/data_test.go index 930a12c..89d37ec 100644 --- a/pkg/ci/data_test.go +++ b/pkg/ci/data_test.go @@ -14,7 +14,7 @@ func TestData_Kill(t *testing.T) { func TestData_Terminate(t *testing.T) { data := Data{} - if data.Terminate() != nil { + if data.Terminate(false) != nil { t.Error("Terminate() should do nothing") } } diff --git a/pkg/ci/locals.go b/pkg/ci/locals.go index 8788e30..08d527c 100644 --- a/pkg/ci/locals.go +++ b/pkg/ci/locals.go @@ -58,7 +58,7 @@ func (*Local) IsDaemon() bool { return false } -func (*Local) Terminate() hcl.Diagnostics { +func (*Local) Terminate(safe bool) hcl.Diagnostics { return nil } diff --git a/pkg/ci/locals_lifecycle.go b/pkg/ci/locals_lifecycle.go new file mode 100644 index 0000000..9a10470 --- /dev/null +++ b/pkg/ci/locals_lifecycle.go @@ -0,0 +1,10 @@ +package ci + +import ( + "context" + "github.com/hashicorp/hcl/v2" +) + +func (l *Local) Lifecycle(ctx context.Context) (*DaemonLifecycle, hcl.Diagnostics) { + return nil, nil +} diff --git a/pkg/ci/locals_run.go b/pkg/ci/locals_run.go index fb32473..8c789d4 100644 --- a/pkg/ci/locals_run.go +++ b/pkg/ci/locals_run.go @@ -45,3 +45,7 @@ func (l *Local) CanRun(ctx context.Context) (bool, hcl.Diagnostics) { func (l *Local) Prepare(ctx context.Context, skip bool, overridden bool) hcl.Diagnostics { return nil } + +func (l *Local) Terminated() bool { + return true +} diff --git a/pkg/ci/locals_test.go b/pkg/ci/locals_test.go index 695be67..447bb85 100644 --- a/pkg/ci/locals_test.go +++ b/pkg/ci/locals_test.go @@ -21,7 +21,7 @@ func TestLocal_Kill(t *testing.T) { func TestLocal_Terminate(t *testing.T) { local := Local{} - if local.Terminate() != nil { + if local.Terminate(false) != nil { t.Error("Terminate() should return nil") } } diff --git a/pkg/ci/macro.go b/pkg/ci/macro.go index f0cb182..ede7209 100644 --- a/pkg/ci/macro.go +++ b/pkg/ci/macro.go @@ -45,7 +45,7 @@ func (m *Macro) IsDaemon() bool { return false } -func (m *Macro) Terminate() hcl.Diagnostics { +func (m *Macro) Terminate(safe bool) hcl.Diagnostics { return nil } diff --git a/pkg/ci/macro_lifecycle.go b/pkg/ci/macro_lifecycle.go new file mode 100644 index 0000000..b8d2958 --- /dev/null +++ b/pkg/ci/macro_lifecycle.go @@ -0,0 +1,10 @@ +package ci + +import ( + "context" + "github.com/hashicorp/hcl/v2" +) + +func (m *Macro) Lifecycle(ctx context.Context) (*DaemonLifecycle, hcl.Diagnostics) { + return nil, nil +} diff --git a/pkg/ci/macro_run.go b/pkg/ci/macro_run.go index 285ab09..4fd418b 100644 --- a/pkg/ci/macro_run.go +++ b/pkg/ci/macro_run.go @@ -50,3 +50,7 @@ func (m *Macro) Run(ctx context.Context) hcl.Diagnostics { func (m *Macro) CanRun(ctx context.Context) (bool, hcl.Diagnostics) { return true, nil } + +func (m *Macro) Terminated() bool { + return true +} diff --git a/pkg/ci/runnable.go b/pkg/ci/runnable.go index ffe0771..b559399 100644 --- a/pkg/ci/runnable.go +++ b/pkg/ci/runnable.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/hashicorp/hcl/v2" + "github.com/srevinsaju/togomak/v1/pkg/x" "strings" "sync" @@ -49,13 +50,15 @@ type Contextual interface { } type Killable interface { - Terminate() hcl.Diagnostics + Terminate(safe bool) hcl.Diagnostics Kill() hcl.Diagnostics + Terminated() bool } type Daemon interface { // IsDaemon returns true if the runnable is a daemon IsDaemon() bool + Lifecycle(ctx context.Context) (*DaemonLifecycle, hcl.Diagnostics) } type Block interface { @@ -148,3 +151,35 @@ func Resolve(ctx context.Context, pipe *Pipeline, id string) (Block, hcl.Diagnos Detail: fmt.Sprintf("Expected a valid identifier, got %s", id), }) } + +func ResolveFromTraversal(variable hcl.Traversal) (string, hcl.Diagnostics) { + blockType := variable.RootName() + var parent string + var diags hcl.Diagnostics + switch blockType { + case DataBlock: + // the data block has the provider type as well as the name + provider := variable[1].(hcl.TraverseAttr).Name + name := variable[2].(hcl.TraverseAttr).Name + parent = x.RenderBlock(DataBlock, provider, name) + case StageBlock: + // the stage block has the name + name := variable[1].(hcl.TraverseAttr).Name + parent = x.RenderBlock(StageBlock, name) + case LocalBlock: + // the local block has the name + name := variable[1].(hcl.TraverseAttr).Name + parent = x.RenderBlock(LocalBlock, name) + case MacroBlock: + // the local block has the name + name := variable[1].(hcl.TraverseAttr).Name + parent = x.RenderBlock(MacroBlock, name) + case ParamBlock, ThisBlock, BuilderBlock: + return "", nil + default: + return "", nil + + } + return parent, diags + +} diff --git a/pkg/ci/stage_hcl.go b/pkg/ci/stage_hcl.go index d14b5af..ad367fa 100644 --- a/pkg/ci/stage_hcl.go +++ b/pkg/ci/stage_hcl.go @@ -14,6 +14,14 @@ func (e *StageContainerVolume) Variables() []hcl.Traversal { return traversal } +func (e *StageDaemon) Variables() []hcl.Traversal { + var traversal []hcl.Traversal + if e.Lifecycle != nil { + traversal = append(traversal, e.Lifecycle.Variables()...) + } + return traversal +} + func (e *StageContainerVolumes) Variables() []hcl.Traversal { var traversal []hcl.Traversal for _, volume := range *e { @@ -38,6 +46,9 @@ func (s *Stage) Variables() []hcl.Traversal { if s.Container != nil { traversal = append(traversal, s.Container.Volumes.Variables()...) } + if s.Daemon != nil { + traversal = append(traversal, s.Daemon.Variables()...) + } for _, env := range s.Environment { traversal = append(traversal, env.Variables()...) diff --git a/pkg/ci/stage_lifecycle.go b/pkg/ci/stage_lifecycle.go new file mode 100644 index 0000000..32580c8 --- /dev/null +++ b/pkg/ci/stage_lifecycle.go @@ -0,0 +1,13 @@ +package ci + +import ( + "context" + "github.com/hashicorp/hcl/v2" +) + +func (s *Stage) Lifecycle(ctx context.Context) (*DaemonLifecycle, hcl.Diagnostics) { + if s.Daemon != nil { + return s.Daemon.Lifecycle.Parse(ctx) + } + return nil, nil +} diff --git a/pkg/ci/stage_run.go b/pkg/ci/stage_run.go index 26dd903..672114e 100644 --- a/pkg/ci/stage_run.go +++ b/pkg/ci/stage_run.go @@ -386,6 +386,11 @@ func (s *Stage) Run(ctx context.Context) hcl.Diagnostics { logger.Trace("running command:", cmd.String()) if !isDryRun { err = cmd.Run() + + if err != nil && err.Error() == "signal: terminated" && s.Terminated() { + logger.Warnf("command terminated with signal: %s", cmd.ProcessState.String()) + err = nil + } } else { fmt.Println(cmd.String()) } @@ -622,8 +627,11 @@ func dockerContainerSourceFmt(containerId string) string { return fmt.Sprintf("docker: container=%s", containerId) } -func (s *Stage) Terminate() hcl.Diagnostics { +func (s *Stage) Terminate(safe bool) hcl.Diagnostics { var diags hcl.Diagnostics + if safe { + s.terminated = true + } if s.Container != nil && s.ContainerId != "" { ctx := context.Background() @@ -644,8 +652,10 @@ func (s *Stage) Terminate() hcl.Diagnostics { }) } } else if s.process != nil && s.process.Process != nil { - if s.process.ProcessState.Exited() { - return diags + if s.process.ProcessState != nil { + if s.process.ProcessState.Exited() { + return diags + } } err := s.process.Process.Signal(syscall.SIGTERM) if err != nil { @@ -661,7 +671,7 @@ func (s *Stage) Terminate() hcl.Diagnostics { } func (s *Stage) Kill() hcl.Diagnostics { - diags := s.Terminate() + diags := s.Terminate(false) if s.process != nil && !s.process.ProcessState.Exited() { err := s.process.Process.Kill() if err != nil { @@ -674,3 +684,7 @@ func (s *Stage) Kill() hcl.Diagnostics { } return diags } + +func (s *Stage) Terminated() bool { + return s.terminated +} diff --git a/pkg/ci/stage_schema.go b/pkg/ci/stage_schema.go index 66579d4..3bcfa79 100644 --- a/pkg/ci/stage_schema.go +++ b/pkg/ci/stage_schema.go @@ -50,15 +50,17 @@ type StageUse struct { Macro hcl.Expression `hcl:"macro" json:"macro"` Parameters hcl.Expression `hcl:"parameters,optional" json:"parameters"` } - type StageDaemon struct { Enabled bool `hcl:"enabled" json:"enabled"` Timeout int `hcl:"timeout,optional" json:"timeout"` + + Lifecycle *Lifecycle `hcl:"lifecycle,block" json:"lifecycle"` } type Stage struct { ctx context.Context ctxInitialised bool + terminated bool Id string `hcl:"id,label" json:"id"` Condition hcl.Expression `hcl:"if,optional" json:"if"` diff --git a/pkg/graph/depgraph.go b/pkg/graph/depgraph.go index caf70e0..e9977bf 100644 --- a/pkg/graph/depgraph.go +++ b/pkg/graph/depgraph.go @@ -10,71 +10,57 @@ import ( "github.com/srevinsaju/togomak/v1/pkg/x" ) +func Resolve(ctx context.Context, pipe *ci.Pipeline, g *depgraph.Graph, v []hcl.Traversal, child string) hcl.Diagnostics { + var diags hcl.Diagnostics + + _, d := ci.Resolve(ctx, pipe, child) + diags = diags.Extend(d) + if diags.HasErrors() { + return diags + } + + for _, variable := range v { + parent, d := ci.ResolveFromTraversal(variable) + diags = diags.Extend(d) + if parent == "" { + continue + } + + _, d = ci.Resolve(ctx, pipe, parent) + diags = diags.Extend(d) + err := g.DependOn(child, parent) + + if err != nil { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid dependency", + Detail: err.Error(), + }) + } + + } + return diags +} func TopoSort(ctx context.Context, pipe *ci.Pipeline) (*depgraph.Graph, hcl.Diagnostics) { g := depgraph.New() var diags hcl.Diagnostics logger := ctx.Value("logger").(*logrus.Logger).WithField("component", "graph") for _, local := range pipe.Local { - err := g.DependOn(x.RenderBlock(ci.LocalBlock, local.Key), meta.RootStage) + self := x.RenderBlock(ci.LocalBlock, local.Key) + err := g.DependOn(self, meta.RootStage) if err != nil { panic(err) } v := local.Variables() - for _, variable := range v { - blockType := variable.RootName() - var child string - var parent string - - child = x.RenderBlock(ci.LocalBlock, local.Key) - _, d := ci.Resolve(ctx, pipe, child) - diags = diags.Extend(d) - - switch blockType { - case ci.DataBlock: - // the data block has the provider type as well as the name - provider := variable[1].(hcl.TraverseAttr).Name - name := variable[2].(hcl.TraverseAttr).Name - parent = x.RenderBlock(ci.DataBlock, provider, name) - case ci.StageBlock: - // the stage block has the name - name := variable[1].(hcl.TraverseAttr).Name - parent = x.RenderBlock(ci.StageBlock, name) - case ci.LocalBlock: - // the local block has the name - name := variable[1].(hcl.TraverseAttr).Name - parent = x.RenderBlock(ci.LocalBlock, name) - case ci.ThisBlock: - case ci.MacroBlock: - // the local block has the name - name := variable[1].(hcl.TraverseAttr).Name - parent = x.RenderBlock(ci.MacroBlock, name) - case ci.ParamBlock: - continue - case ci.BuilderBlock: - continue - default: - continue - } - - _, d = ci.Resolve(ctx, pipe, parent) - diags = diags.Extend(d) - err = g.DependOn(child, parent) - - if err != nil { - diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid dependency", - Detail: err.Error(), - }) - } - - } - + d := Resolve(ctx, pipe, g, v, self) + diags = diags.Extend(d) } + for _, macro := range pipe.Macros { - err := g.DependOn(x.RenderBlock(ci.MacroBlock, macro.Id), meta.RootStage) + self := x.RenderBlock(ci.MacroBlock, macro.Id) + err := g.DependOn(self, meta.RootStage) // the addition of the root stage is to ensure that the macro block is always executed // before any stage // this function should succeed always @@ -83,57 +69,12 @@ func TopoSort(ctx context.Context, pipe *ci.Pipeline) (*depgraph.Graph, hcl.Diag } v := macro.Variables() - for _, variable := range v { - blockType := variable.RootName() - var child string - var parent string - child = x.RenderBlock(ci.MacroBlock, macro.Id) - _, d := ci.Resolve(ctx, pipe, child) - diags = diags.Extend(d) - - switch blockType { - case ci.DataBlock: - // the data block has the provider type as well as the name - provider := variable[1].(hcl.TraverseAttr).Name - name := variable[2].(hcl.TraverseAttr).Name - parent = x.RenderBlock(ci.DataBlock, provider, name) - case ci.StageBlock: - // the stage block only has the id which is the second element - name := variable[1].(hcl.TraverseAttr).Name - parent = x.RenderBlock(ci.StageBlock, name) - case ci.LocalBlock: - // the local block has the name - name := variable[1].(hcl.TraverseAttr).Name - parent = x.RenderBlock(ci.LocalBlock, name) - case ci.MacroBlock: - // the local block has the name - name := variable[1].(hcl.TraverseAttr).Name - parent = x.RenderBlock(ci.MacroBlock, name) - case ci.ThisBlock: - case ci.ParamBlock: - continue - - case ci.BuilderBlock: - continue - default: - continue - } - - _, d = ci.Resolve(ctx, pipe, parent) - diags = diags.Extend(d) - err = g.DependOn(child, parent) - - if err != nil { - diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid dependency", - Detail: err.Error(), - }) - } - } + d := Resolve(ctx, pipe, g, v, self) + diags = diags.Extend(d) } for _, data := range pipe.Data { - err := g.DependOn(x.RenderBlock(ci.DataBlock, data.Provider, data.Id), meta.RootStage) + self := x.RenderBlock(ci.DataBlock, data.Provider, data.Id) + err := g.DependOn(self, meta.RootStage) // the addition of the root stage is to ensure that the data block is always executed // before any stage // this function should succeed always @@ -142,108 +83,19 @@ func TopoSort(ctx context.Context, pipe *ci.Pipeline) (*depgraph.Graph, hcl.Diag } v := data.Variables() - for _, variable := range v { - blockType := variable.RootName() - var child string - var parent string - child = x.RenderBlock(ci.DataBlock, data.Provider, data.Id) - _, d := ci.Resolve(ctx, pipe, child) - diags = diags.Extend(d) - - switch blockType { - case ci.DataBlock: - // the data block has the provider type as well as the name - provider := variable[1].(hcl.TraverseAttr).Name - name := variable[2].(hcl.TraverseAttr).Name - parent = x.RenderBlock(ci.DataBlock, provider, name) - case ci.StageBlock: - // the stage block only has the id which is the second element - name := variable[1].(hcl.TraverseAttr).Name - parent = x.RenderBlock(ci.StageBlock, name) - case ci.LocalBlock: - // the local block has the name - name := variable[1].(hcl.TraverseAttr).Name - parent = x.RenderBlock(ci.LocalBlock, name) - case ci.MacroBlock: - // the local block has the name - name := variable[1].(hcl.TraverseAttr).Name - parent = x.RenderBlock(ci.MacroBlock, name) - case ci.ThisBlock: - continue - case ci.ParamBlock: - continue - case ci.BuilderBlock: - continue - default: - continue - } - - _, d = ci.Resolve(ctx, pipe, parent) - diags = diags.Extend(d) - err = g.DependOn(child, parent) - - if err != nil { - diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid dependency", - Detail: err.Error(), - }) - } - } + d := Resolve(ctx, pipe, g, v, self) + diags = diags.Extend(d) } for _, stage := range pipe.Stages { - err := g.DependOn(x.RenderBlock(ci.StageBlock, stage.Id), meta.RootStage) + self := x.RenderBlock(ci.StageBlock, stage.Id) + err := g.DependOn(self, meta.RootStage) if err != nil { panic(err) } v := stage.Variables() - for _, variable := range v { - - blockType := variable.RootName() - var child string - var parent string - child = x.RenderBlock(ci.StageBlock, stage.Id) - _, d := ci.Resolve(ctx, pipe, child) - diags = diags.Extend(d) - - switch blockType { - case ci.DataBlock: - // the data block has the provider type as well as the name - provider := variable[1].(hcl.TraverseAttr).Name - name := variable[2].(hcl.TraverseAttr).Name - parent = x.RenderBlock(ci.DataBlock, provider, name) - case ci.StageBlock: - // the stage block only has the id which is the second element - name := variable[1].(hcl.TraverseAttr).Name - parent = x.RenderBlock(ci.StageBlock, name) - case ci.LocalBlock: - name := variable[1].(hcl.TraverseAttr).Name - parent = x.RenderBlock(ci.LocalBlock, name) - case ci.MacroBlock: - name := variable[1].(hcl.TraverseAttr).Name - parent = x.RenderBlock(ci.MacroBlock, name) - case ci.ThisBlock: - continue - case ci.ParamBlock: - continue - case ci.BuilderBlock: - continue - default: - continue - } - _, d = ci.Resolve(ctx, pipe, parent) - diags = diags.Extend(d) - err = g.DependOn(child, parent) - - if err != nil { - diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid dependency", - Detail: err.Error(), - }) - } - } + d := Resolve(ctx, pipe, g, v, self) + diags = diags.Extend(d) } for i, layer := range g.TopoSortedLayers() { diff --git a/pkg/orchestra/context.go b/pkg/orchestra/context.go index 5b1d4e5..0b68174 100644 --- a/pkg/orchestra/context.go +++ b/pkg/orchestra/context.go @@ -250,6 +250,7 @@ func NewContextWithTogomak(cfg Config) (Togomak, context.Context) { ectx: hclContext, tempDir: tmpDir, } + ctx = context.WithValue(ctx, c.Togomak, &t) return t, ctx } diff --git a/pkg/orchestra/interrupt.go b/pkg/orchestra/interrupt.go index d9abcfa..e57efce 100644 --- a/pkg/orchestra/interrupt.go +++ b/pkg/orchestra/interrupt.go @@ -35,7 +35,7 @@ func InterruptHandler(ctx context.Context, cancel context.CancelFunc, ch chan os }() for _, runnable := range *runnables { logger.Debugf("stopping runnable %s", runnable.Identifier()) - d := runnable.Terminate() + d := runnable.Terminate(false) diags = diags.Extend(d) } diff --git a/pkg/orchestra/orchestra.go b/pkg/orchestra/orchestra.go index 48cd667..8ef89af 100644 --- a/pkg/orchestra/orchestra.go +++ b/pkg/orchestra/orchestra.go @@ -165,6 +165,11 @@ func Orchestra(cfg Config) int { var daemonWg sync.WaitGroup var hasDaemons bool var runnables ci.Blocks + var daemons ci.Blocks + var daemonsMutex sync.Mutex + var completedRunnablesSignal = make(chan ci.Block, 1) + var completedRunnables ci.Blocks + var completedRunnablesMutex sync.Mutex // region: interrupt handler chInterrupt := make(chan os.Signal, 1) @@ -174,6 +179,7 @@ func Orchestra(cfg Config) int { signal.Notify(chKill, os.Kill) go InterruptHandler(ctx, cancel, chInterrupt, &runnables) go KillHandler(ctx, cancel, chKill, &runnables) + go daemonKiller(ctx, completedRunnablesSignal, &daemons) // endregion: interrupt handler for _, layer := range depGraph.TopoSortedLayers() { @@ -288,14 +294,31 @@ func Orchestra(cfg Config) int { if runnable.IsDaemon() { hasDaemons = true daemonWg.Add(1) + + daemonsMutex.Lock() + daemons = append(daemons, runnable) + daemonsMutex.Unlock() } else { wg.Add(1) } go func(runnableId string) { stageDiags := runnable.Run(ctx) + + logger.Tracef("locking completedRunnablesMutex for runnable %s", runnableId) + completedRunnablesMutex.Lock() + completedRunnables = append(completedRunnables, runnable) + completedRunnablesMutex.Unlock() + logger.Tracef("unlocking completedRunnablesMutex for runnable %s", runnableId) + completedRunnablesSignal <- runnable + logger.Tracef("signaling runnable %s", runnableId) + if !stageDiags.HasErrors() { - wg.Done() + if runnable.IsDaemon() { + daemonWg.Done() + } else { + wg.Done() + } return } if !runnable.CanRetry() { @@ -368,9 +391,7 @@ func Orchestra(cfg Config) int { cancel() } break - } - } daemonWg.Wait() @@ -399,3 +420,59 @@ func ok(ctx context.Context) int { func diagnostics(t *Togomak, diags *hcl.Diagnostics) { x.Must(t.hclDiagWriter.WriteDiagnostics(*diags)) } + +func daemonKiller(ctx context.Context, completed chan ci.Block, daemons *ci.Blocks) { + logger := ctx.Value(c.TogomakContextLogger).(*logrus.Logger).WithField("watchdog", "") + var completedRunnables ci.Blocks + var diags hcl.Diagnostics + defer diagnostics(ctx.Value(c.Togomak).(*Togomak), &diags) + logger.Tracef("starting watchdog") + + // execute the following function when we receive any message on the completed channel + for { + c := <-completed + logger.Debugf("received completed runnable, %s", c.Identifier()) + completedRunnables = append(completedRunnables, c) + + daemons := *daemons + for _, daemon := range daemons { + if daemon.Terminated() { + continue + } + logger.Tracef("checking daemon %s", daemon.Identifier()) + lifecycle, d := daemon.Lifecycle(ctx) + if d.HasErrors() { + diags = diags.Extend(d) + d := daemon.Terminate(false) + diags = diags.Extend(d) + return + } + if lifecycle == nil { + continue + } + + allCompleted := true + for _, block := range lifecycle.StopWhenComplete { + logger.Tracef("checking daemon %s, requires block %s to complete", daemon.Identifier(), block.Identifier()) + completed := false + for _, completedBlocks := range completedRunnables { + if block.Identifier() == completedBlocks.Identifier() { + completed = true + break + } + } + if !completed { + allCompleted = false + break + } + } + if allCompleted { + logger.Infof("stopping daemon %s", daemon.Identifier()) + d := daemon.Terminate(true) + if d.HasErrors() { + diags = diags.Extend(d) + } + } + } + } +} From eaa210beb4552dbdf0d02d31de2bf9ec2f6577d3 Mon Sep 17 00:00:00 2001 From: Srevin Saju Date: Thu, 22 Jun 2023 21:56:02 +0530 Subject: [PATCH 6/6] docs: update lifecycle rules --- docs/src/configuration/data_git.md | 18 ++++++++++++------ docs/src/configuration/stage.md | 10 +++++++++- pkg/blocks/data/git.go | 1 + pkg/ci/daemon.go | 4 ++++ 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/docs/src/configuration/data_git.md b/docs/src/configuration/data_git.md index 956a742..1ddc825 100644 --- a/docs/src/configuration/data_git.md +++ b/docs/src/configuration/data_git.md @@ -26,10 +26,10 @@ data "git" "this" { - [`url`](#url) - The URL of the repository to clone - [`tag`](#tag) - The tag to checkout - [`branch`](#branch) - The branch to checkout -- [`destination`](#destination) - The destination directory to clone the repository to, defaults to `"memory"`, which clones into memory +- [`destination`](#destination) - The destination directory to clone the repository to, defaults to `"memory"`, which clones into a temporary directory managed by `togomak` - [`commit`](#commit) - The commit to checkout - [`depth`](#depth) - The depth of the clone -- [`ca_bundle`](#ca_bundle) - The path to a CA bundle file or directory +- [`ca_bundle`](#ca_bundle) - The path to a CA bundle file or directory, (deprecated, does nothing). - [`auth`](#auth) - The authentication credentials to use when cloning the repository. Structure [documented below](#auth) - [`files`](#files) - The files to checkout from the repository. Accepts an array of file paths. @@ -39,10 +39,16 @@ data "git" "this" { - [`sha`](#sha) - The SHA of the commit, defaults to `""` - [`ref`](#ref) - The ref of the commit, in the format `refs/heads/` or `refs/tags/`, defaults to `""` - [`is_tag`](#is-tag) - Whether the ref is a tag, defaults to `false` -- [`is_branch`](#is-branch) - Whether the ref is a branch, defaults to `false` -- [`is_note`](#is-note) - Whether the ref is a note, defaults to `false` -- [`is_remote`](#is-remote) - Whether the ref is remote, defaults to `false` - [`files`](#files) - The files checked out from the repository. Returns a map, with the keys being the file paths and the values being the file contents. - [`branch`](#branch) - The branch checked out from the repository. Returns a string. -- [`tag`](#tag) - The tag checked out from the repository. Returns a string. +- [`destination`](#destination) - The destination where the repository is stored. + +--- + +The `auth` block supports: +- [`username`](#username) - The username to be used. If password is specified, and if the username is empty, it falls back to `oauth2` +- [`password`](#password) - The password to be used for connecting to the private repository. + + + diff --git a/docs/src/configuration/stage.md b/docs/src/configuration/stage.md index 59b7bcd..f6ac818 100644 --- a/docs/src/configuration/stage.md +++ b/docs/src/configuration/stage.md @@ -90,7 +90,10 @@ stage "build" { {{#include ../../../examples/docker/togomak.hcl}} ``` - +## Stage with daemons, with lifecycle rules +```hcl +{{#include ../../../examples/daemon-until/togomak.hcl}} +``` ## Arguments Reference * [`name`](#name) - The name of the stage @@ -127,3 +130,8 @@ The `container` block supports: The `daemon` block supports: * [`enabled`](#enabled) - Whether the stage should be run as a daemon, defaults to `false` * [`timeout`](#timeout) - Time to wait until the stage is terminated, in seconds. Defaults to 0 (no timeout). +* [`lifecycle`](#lifecycle) - Set of rules which decide if the daemon needs to be terminated, or not. Structure [documented below](#lifecycle) +--- + + +* [`stop_when_complete`] - Array of stages. `togomak` waits for all the stages mentioned in the `stop_when_complete` argument, before sending `SIGTERM` to the daemon process diff --git a/pkg/blocks/data/git.go b/pkg/blocks/data/git.go index cded776..92bd7bb 100644 --- a/pkg/blocks/data/git.go +++ b/pkg/blocks/data/git.go @@ -472,6 +472,7 @@ func (e *GitProvider) Attributes(ctx context.Context, id string) (map[string]cty attrs[GitBlockAttrLastTag] = cty.StringVal(lastTag) attrs[GitBlockAttrCommitsSinceLastTag] = commitsSinceLastTag attrs[GitBlockAttrFiles] = filesCty + attrs[GitBlockArgumentDestination] = cty.StringVal(destination) // get the commit return attrs, diags diff --git a/pkg/ci/daemon.go b/pkg/ci/daemon.go index 33e9bef..3b2f3d4 100644 --- a/pkg/ci/daemon.go +++ b/pkg/ci/daemon.go @@ -15,10 +15,14 @@ type Lifecycle struct { } func (l *Lifecycle) Parse(ctx context.Context) (*DaemonLifecycle, hcl.Diagnostics) { + pipe := ctx.Value(c.TogomakContextPipeline).(*Pipeline) daemonLifecycle := &DaemonLifecycle{} var diags hcl.Diagnostics + if l == nil || l.StopWhenComplete == nil { + return daemonLifecycle, diags + } variables := l.StopWhenComplete.Variables() var runnableString []string