From 33d78c437433841b822e279e88516ef24f3f2ced Mon Sep 17 00:00:00 2001 From: Srevin Saju Date: Thu, 21 Sep 2023 18:04:15 +0300 Subject: [PATCH] refactor(conductor): use conductor instead of context Values --- cmd/togomak/main.go | 4 +- pkg/behavior/models.go | 2 + pkg/blocks/data/provider_options.go | 10 +- pkg/ci/data_run.go | 1 + pkg/ci/pipeline_read.go | 4 +- pkg/ci/runnable.go | 4 +- pkg/ci/stage_run.go | 4 +- pkg/conductor/config.go | 4 +- pkg/conductor/hcl.go | 240 +++++++++++++++++++++ pkg/conductor/togomak.go | 78 ++++++- pkg/orchestra/context.go | 309 ---------------------------- pkg/orchestra/format.go | 9 +- pkg/orchestra/imports.go | 2 +- pkg/orchestra/list.go | 31 +-- pkg/orchestra/logging.go | 45 ---- pkg/orchestra/orchestra.go | 24 +-- pkg/orchestra/outputs.go | 9 +- pkg/orchestra/run.go | 2 +- pkg/orchestra/utils.go | 27 --- pkg/parse/parse.go | 4 +- pkg/path/models.go | 11 - pkg/runnable/options.go | 12 +- 22 files changed, 370 insertions(+), 466 deletions(-) create mode 100644 pkg/conductor/hcl.go diff --git a/cmd/togomak/main.go b/cmd/togomak/main.go index d53742d..16e1506 100644 --- a/cmd/togomak/main.go +++ b/cmd/togomak/main.go @@ -188,7 +188,7 @@ func newConfigFromCliContext(ctx *cli.Context) conductor.Config { } cfg := conductor.Config{ - Behavior: behavior.Behavior{ + Behavior: &behavior.Behavior{ Unattended: ctx.Bool("unattended") || ctx.Bool("ci"), Ci: ctx.Bool("ci"), DryRun: ctx.Bool("dry-run"), @@ -200,7 +200,7 @@ func newConfigFromCliContext(ctx *cli.Context) conductor.Config { }, }, - Paths: path.Path{ + Paths: &path.Path{ Pipeline: pipelineFilePath, Cwd: dir, Owd: owd, diff --git a/pkg/behavior/models.go b/pkg/behavior/models.go index 2525e9b..606dede 100644 --- a/pkg/behavior/models.go +++ b/pkg/behavior/models.go @@ -12,6 +12,8 @@ type Child struct { } type Behavior struct { + initialized bool + // Unattended is the flag to indicate whether the program is running in unattended mode Unattended bool diff --git a/pkg/blocks/data/provider_options.go b/pkg/blocks/data/provider_options.go index 317ac01..29d1cd8 100644 --- a/pkg/blocks/data/provider_options.go +++ b/pkg/blocks/data/provider_options.go @@ -6,26 +6,26 @@ import ( ) type ProviderConfig struct { - Paths path.Path + Paths *path.Path - Behavior behavior.Behavior + Behavior *behavior.Behavior } func NewDefaultProviderConfig() *ProviderConfig { return &ProviderConfig{ - Paths: path.NewDefaultPath(), + Paths: nil, } } type ProviderOption func(*ProviderConfig) -func WithPaths(paths path.Path) ProviderOption { +func WithPaths(paths *path.Path) ProviderOption { return func(c *ProviderConfig) { c.Paths = paths } } -func WithBehavior(behavior behavior.Behavior) ProviderOption { +func WithBehavior(behavior *behavior.Behavior) ProviderOption { return func(c *ProviderConfig) { c.Behavior = behavior } diff --git a/pkg/ci/data_run.go b/pkg/ci/data_run.go index 296b6ac..6d33a48 100644 --- a/pkg/ci/data_run.go +++ b/pkg/ci/data_run.go @@ -28,6 +28,7 @@ func (s *Data) Run(ctx context.Context, options ...runnable.Option) (diags hcl.D cfg := runnable.NewConfig(options...) opts := []dataBlock.ProviderOption{ dataBlock.WithPaths(cfg.Paths), + dataBlock.WithBehavior(cfg.Behavior), } // region: mutating the data map diff --git a/pkg/ci/pipeline_read.go b/pkg/ci/pipeline_read.go index ef471a7..ec7e7c5 100644 --- a/pkg/ci/pipeline_read.go +++ b/pkg/ci/pipeline_read.go @@ -18,7 +18,7 @@ import ( // Read reads togomak.hcl from the configuration file directory. A configuration file directory is the one that // contains togomak.hcl, it searches recursively outwards. // DEPRECATED: use ReadDir instead -func Read(paths path.Path, parser *hclparse.Parser) (*Pipeline, hcl.Diagnostics) { +func Read(paths *path.Path, parser *hclparse.Parser) (*Pipeline, hcl.Diagnostics) { ciFile := parse.ConfigFilePath(paths) f, diags := parser.ParseHCLFile(ciFile) @@ -39,7 +39,7 @@ func Read(paths path.Path, parser *hclparse.Parser) (*Pipeline, hcl.Diagnostics) // ReadDir parses an entire directory of *.hcl files and merges them together. This is useful when you want to // split your pipeline into multiple files, without having to import them individually -func ReadDir(paths path.Path, parser *hclparse.Parser) (*Pipeline, hcl.Diagnostics) { +func ReadDir(paths *path.Path, parser *hclparse.Parser) (*Pipeline, hcl.Diagnostics) { dir := parse.ConfigFileDir(paths) return ReadDirFromPath(dir, parser) diff --git a/pkg/ci/runnable.go b/pkg/ci/runnable.go index 218db93..e323601 100644 --- a/pkg/ci/runnable.go +++ b/pkg/ci/runnable.go @@ -88,7 +88,7 @@ func (r Blocks) Variables() []hcl.Traversal { return traversal } -func (r Blocks) Run(ctx context.Context) hcl.Diagnostics { +func (r Blocks) Run(ctx context.Context, opts ...runnable.Option) hcl.Diagnostics { // run all runnables in parallel, collect errors and return // create a channel to receive errors var wg sync.WaitGroup @@ -97,7 +97,7 @@ func (r Blocks) Run(ctx context.Context) hcl.Diagnostics { wg.Add(1) go func(runnable Block) { defer wg.Done() - errChan <- runnable.Run(ctx) + errChan <- runnable.Run(ctx, opts...) }(runnable) } wg.Wait() diff --git a/pkg/ci/stage_run.go b/pkg/ci/stage_run.go index 5bc6abc..a7dd59e 100644 --- a/pkg/ci/stage_run.go +++ b/pkg/ci/stage_run.go @@ -223,7 +223,7 @@ func (s *Stage) expandMacros(ctx context.Context, opts ...runnable.Option) (*Sta if !f.IsNull() { files := f.AsValueMap() logger.Debugf("using %d files from %s", len(files), macro.Identifier()) - err = os.MkdirAll(filepath.Join(cfg.Paths.Cwd, tmpDir, s.Id), 0755) + err = os.MkdirAll(filepath.Join(tmpDir, s.Id), 0755) if err != nil { return s, diags.Append(&hcl.Diagnostic{ Severity: hcl.DiagError, @@ -497,7 +497,7 @@ func (s *Stage) Run(ctx context.Context, options ...runnable.Option) (diags hcl. envStrings = append(envStrings, envParsed) } - togomakEnvExport := fmt.Sprintf("%s=%s", meta.OutputEnvVar, filepath.Join(cfg.Paths.Cwd, tmpDir, meta.OutputEnvFile)) + togomakEnvExport := fmt.Sprintf("%s=%s", meta.OutputEnvVar, filepath.Join(tmpDir, meta.OutputEnvFile)) logger.Tracef("exporting %s", togomakEnvExport) envStrings = append(envStrings, togomakEnvExport) diff --git a/pkg/conductor/config.go b/pkg/conductor/config.go index f25c290..eeb7ca9 100644 --- a/pkg/conductor/config.go +++ b/pkg/conductor/config.go @@ -20,7 +20,7 @@ type Config struct { User string Hostname string - Paths path.Path + Paths *path.Path Interface Interface @@ -28,5 +28,5 @@ type Config struct { Pipeline ConfigPipeline // Behavior is the behavior of the program - Behavior behavior.Behavior + Behavior *behavior.Behavior } diff --git a/pkg/conductor/hcl.go b/pkg/conductor/hcl.go new file mode 100644 index 0000000..af9268e --- /dev/null +++ b/pkg/conductor/hcl.go @@ -0,0 +1,240 @@ +package conductor + +import ( + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/ext/tryfunc" + yaml "github.com/zclconf/go-cty-yaml" + + "github.com/srevinsaju/togomak/v1/pkg/global" + "github.com/srevinsaju/togomak/v1/pkg/meta" + "github.com/srevinsaju/togomak/v1/pkg/third-party/hashicorp/terraform/lang/funcs" + "github.com/srevinsaju/togomak/v1/pkg/ui" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/function" + "github.com/zclconf/go-cty/cty/function/stdlib" + "os" + "os/exec" + "time" +) + +func CreateEvalContext(cfg Config, process Process) *hcl.EvalContext { + // --> set up HCL context + paths := cfg.Paths + behavior := cfg.Behavior + hclContext := &hcl.EvalContext{ + Functions: map[string]function.Function{ + "abs": stdlib.AbsoluteFunc, + "abspath": funcs.AbsPathFunc, + "alltrue": funcs.AllTrueFunc, + "anytrue": funcs.AnyTrueFunc, + "basename": funcs.BasenameFunc, + "base64decode": funcs.Base64DecodeFunc, + "base64encode": funcs.Base64EncodeFunc, + "base64gzip": funcs.Base64GzipFunc, + "base64sha256": funcs.Base64Sha256Func, + "base64sha512": funcs.Base64Sha512Func, + "bcrypt": funcs.BcryptFunc, + "can": tryfunc.CanFunc, + "ceil": stdlib.CeilFunc, + "chomp": stdlib.ChompFunc, + "coalesce": funcs.CoalesceFunc, + "coalescelist": stdlib.CoalesceListFunc, + "compact": stdlib.CompactFunc, + "concat": stdlib.ConcatFunc, + "contains": stdlib.ContainsFunc, + "csvdecode": stdlib.CSVDecodeFunc, + "dirname": funcs.DirnameFunc, + "distinct": stdlib.DistinctFunc, + "element": stdlib.ElementFunc, + "endswith": funcs.EndsWithFunc, + "chunklist": stdlib.ChunklistFunc, + "file": funcs.MakeFileFunc(paths.Cwd, false), + "fileexists": funcs.MakeFileExistsFunc(paths.Cwd), + "fileset": funcs.MakeFileSetFunc(paths.Cwd), + "filebase64": funcs.MakeFileFunc(paths.Cwd, true), + "filebase64sha256": funcs.MakeFileBase64Sha256Func(paths.Cwd), + "filebase64sha512": funcs.MakeFileBase64Sha512Func(paths.Cwd), + "filemd5": funcs.MakeFileMd5Func(paths.Cwd), + "filesha1": funcs.MakeFileSha1Func(paths.Cwd), + "filesha256": funcs.MakeFileSha256Func(paths.Cwd), + "filesha512": funcs.MakeFileSha512Func(paths.Cwd), + "flatten": stdlib.FlattenFunc, + "floor": stdlib.FloorFunc, + "format": stdlib.FormatFunc, + "formatdate": stdlib.FormatDateFunc, + "formatlist": stdlib.FormatListFunc, + "indent": stdlib.IndentFunc, + "index": funcs.IndexFunc, // stdlib.IndexFunc is not compatible + "join": stdlib.JoinFunc, + "jsondecode": stdlib.JSONDecodeFunc, + "jsonencode": stdlib.JSONEncodeFunc, + "keys": stdlib.KeysFunc, + "length": funcs.LengthFunc, + "list": funcs.ListFunc, + "log": stdlib.LogFunc, + "lookup": funcs.LookupFunc, + "lower": stdlib.LowerFunc, + "map": funcs.MapFunc, + "matchkeys": funcs.MatchkeysFunc, + "max": stdlib.MaxFunc, + "md5": funcs.Md5Func, + "merge": stdlib.MergeFunc, + "min": stdlib.MinFunc, + "one": funcs.OneFunc, + "parseint": stdlib.ParseIntFunc, + "pathexpand": funcs.PathExpandFunc, + "pow": stdlib.PowFunc, + "range": stdlib.RangeFunc, + "regex": stdlib.RegexFunc, + "regexall": stdlib.RegexAllFunc, + "replace": funcs.ReplaceFunc, + "reverse": stdlib.ReverseListFunc, + "rsadecrypt": funcs.RsaDecryptFunc, + "sensitive": funcs.SensitiveFunc, + "nonsensitive": funcs.NonsensitiveFunc, + "setintersection": stdlib.SetIntersectionFunc, + "setproduct": stdlib.SetProductFunc, + "setsubtract": stdlib.SetSubtractFunc, + "setunion": stdlib.SetUnionFunc, + "sha1": funcs.Sha1Func, + "sha256": funcs.Sha256Func, + "sha512": funcs.Sha512Func, + "signum": stdlib.SignumFunc, + "slice": stdlib.SliceFunc, + "sort": stdlib.SortFunc, + "split": stdlib.SplitFunc, + "startswith": funcs.StartsWithFunc, + "strcontains": funcs.StrContainsFunc, + "strrev": stdlib.ReverseFunc, + "substr": stdlib.SubstrFunc, + "sum": funcs.SumFunc, + "textdecodebase64": funcs.TextDecodeBase64Func, + "textencodebase64": funcs.TextEncodeBase64Func, + "timestamp": funcs.TimestampFunc, + "timeadd": stdlib.TimeAddFunc, + "timecmp": funcs.TimeCmpFunc, + "title": stdlib.TitleFunc, + "tostring": funcs.MakeToFunc(cty.String), + "tonumber": funcs.MakeToFunc(cty.Number), + "tobool": funcs.MakeToFunc(cty.Bool), + "toset": funcs.MakeToFunc(cty.Set(cty.DynamicPseudoType)), + "tolist": funcs.MakeToFunc(cty.List(cty.DynamicPseudoType)), + "tomap": funcs.MakeToFunc(cty.Map(cty.DynamicPseudoType)), + "transpose": funcs.TransposeFunc, + "trim": stdlib.TrimFunc, + "trimprefix": stdlib.TrimPrefixFunc, + "trimspace": stdlib.TrimSpaceFunc, + "trimsuffix": stdlib.TrimSuffixFunc, + "try": tryfunc.TryFunc, + "upper": stdlib.UpperFunc, + "urlencode": funcs.URLEncodeFunc, + "uuid": funcs.UUIDFunc, + "uuidv5": funcs.UUIDV5Func, + "values": stdlib.ValuesFunc, + "which": function.New(&function.Spec{ + Params: []function.Parameter{ + { + Name: "executable", + AllowDynamicType: true, + Type: cty.String, + }, + }, + Type: function.StaticReturnType(cty.String), + Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { + path, err := exec.LookPath(args[0].AsString()) + if err != nil { + return cty.StringVal(""), err + } + return cty.StringVal(path), nil + }, + Description: "Returns the absolute path to an executable in the current PATH.", + }), + "yamldecode": yaml.YAMLDecodeFunc, + "yamlencode": yaml.YAMLEncodeFunc, + "zipmap": stdlib.ZipmapFunc, + + "ansifmt": ui.AnsiFunc, + "env": function.New(&function.Spec{ + Params: []function.Parameter{ + { + Name: "Key of the environment variable", + AllowDynamicType: true, + Type: cty.String, + }, + }, + VarParam: &function.Parameter{ + Name: "lists", + Description: "One or more lists of strings to join.", + Type: cty.String, + }, + Type: function.StaticReturnType(cty.String), + Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { + v, ok := os.LookupEnv(args[0].AsString()) + if ok { + return cty.StringVal(v), nil + } + def := args[1] + return def, nil + }, + Description: "Returns the value of the environment variable, returns the default value if environment variable is empty, else returns empty string.", + }), + }, + + Variables: map[string]cty.Value{ + "true": cty.True, + "false": cty.False, + "null": cty.NullVal(cty.DynamicPseudoType), + + "owd": cty.StringVal(paths.Owd), + "cwd": cty.StringVal(paths.Cwd), + "hostname": cty.StringVal(cfg.Hostname), + "hostuser": cty.StringVal(cfg.User), + + "pipeline": cty.ObjectVal(map[string]cty.Value{ + "id": cty.StringVal(process.Id.String()), + "path": cty.StringVal(paths.Pipeline), + "tempDir": cty.StringVal(process.TempDir), + }), + + "togomak": cty.ObjectVal(map[string]cty.Value{ + "version": cty.StringVal(meta.AppVersion), + "boot_time": cty.StringVal(time.Now().Format(time.RFC3339)), + "boot_time_unix": cty.NumberIntVal(time.Now().Unix()), + "pipeline_id": cty.StringVal(process.Id.String()), + "ci": cty.BoolVal(behavior.Ci), + "unattended": cty.BoolVal(behavior.Unattended), + }), + + // introduced in v1.5.0 + "ansi": cty.ObjectVal(map[string]cty.Value{ + "bg": cty.ObjectVal(map[string]cty.Value{ + "red": cty.StringVal("\033[41m"), + "green": cty.StringVal("\033[42m"), + "yellow": cty.StringVal("\033[43m"), + "blue": cty.StringVal("\033[44m"), + "purple": cty.StringVal("\033[45m"), + "cyan": cty.StringVal("\033[46m"), + "white": cty.StringVal("\033[47m"), + "grey": cty.StringVal("\033[100m"), + }), + "fg": cty.ObjectVal(map[string]cty.Value{ + "red": cty.StringVal("\033[31m"), + "green": cty.StringVal("\033[32m"), + "yellow": cty.StringVal("\033[33m"), + "blue": cty.StringVal("\033[34m"), + "purple": cty.StringVal("\033[35m"), + "cyan": cty.StringVal("\033[36m"), + "white": cty.StringVal("\033[37m"), + "grey": cty.StringVal("\033[90m"), + "bold": cty.StringVal("\033[1m"), + "italic": cty.StringVal("\033[3m"), + "underline": cty.StringVal("\033[4m"), + }), + + "reset": cty.StringVal("\033[0m"), + }), + }, + } + global.SetHclEvalContext(hclContext) + return hclContext +} diff --git a/pkg/conductor/togomak.go b/pkg/conductor/togomak.go index fb53432..cd1fe9c 100644 --- a/pkg/conductor/togomak.go +++ b/pkg/conductor/togomak.go @@ -2,11 +2,15 @@ package conductor import ( "context" + "github.com/google/uuid" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclparse" "github.com/sirupsen/logrus" + "github.com/srevinsaju/togomak/v1/pkg/global" + "github.com/srevinsaju/togomak/v1/pkg/meta" "github.com/srevinsaju/togomak/v1/pkg/x" "os" + "path/filepath" "time" ) @@ -26,37 +30,107 @@ type Togomak struct { // DiagWriter is the HCL diagnostic writer, it is used to write the diagnostics // to os.Stdout DiagWriter hcl.DiagnosticWriter + + // EvalContext is the HCL evaluation context + EvalContext *hcl.EvalContext } type Process struct { + Id uuid.UUID + Executable string // BootTime is the time when the process was started BootTime time.Time + + // TempDir is the temporary directory created for the process + TempDir string } func NewProcess(cfg Config) Process { e, err := os.Executable() x.Must(err) + pipelineId := uuid.New() + + // create a temporary directory + tempDir, err := os.MkdirTemp("", "togomak") + x.Must(err) + global.SetTempDir(tempDir) + return Process{ + Id: pipelineId, Executable: e, BootTime: time.Now(), + TempDir: tempDir, } } +func Chdir(cfg Config, logger *logrus.Logger) string { + cwd := cfg.Paths.Cwd + if cwd == "" { + cwd = filepath.Dir(cfg.Paths.Pipeline) + if filepath.Base(cwd) == meta.BuildDirPrefix { + cwd = filepath.Dir(cwd) + } + } + err := os.Chdir(cwd) + if err != nil { + logger.Fatal(err) + } + cwd, err = os.Getwd() + x.Must(err) + logger.Debug("changing working directory to ", cwd) + return cwd + +} + func NewTogomak(cfg Config) *Togomak { parser := hclparse.NewParser() + + // TODO: remove this + global.SetHclParser(parser) + diagWriter := hcl.NewDiagnosticTextWriter(os.Stdout, parser.Files(), 0, true) + logger := NewLogger(cfg) + global.SetLogger(logger) + + cfg.Paths.Cwd = Chdir(cfg, logger) + + if !cfg.Behavior.Child.Enabled { + logger.Infof("%s (version=%s)", meta.AppName, meta.AppVersion) + } + + process := NewProcess(cfg) + return &Togomak{ Parser: parser, DiagWriter: diagWriter, Context: context.Background(), - Process: NewProcess(cfg), + Process: process, - Logger: NewLogger(cfg), + Logger: logger, Config: cfg, + + EvalContext: CreateEvalContext(cfg, process), } } + +func (t *Togomak) Destroy() { + t.Logger.Debug("removing temporary directory") + err := os.RemoveAll(t.Process.TempDir) + if err != nil { + t.Logger.Warnf("failed to remove temporary directory: %s", err) + } + + t.Logger.Debug("destroying togomak") + + t.Logger = nil + t.Config = Config{} + t.Context = nil + t.Parser = nil + t.DiagWriter = nil + +} diff --git a/pkg/orchestra/context.go b/pkg/orchestra/context.go index 716008a..22ba1e9 100644 --- a/pkg/orchestra/context.go +++ b/pkg/orchestra/context.go @@ -1,310 +1 @@ package orchestra - -import ( - "context" - "github.com/google/uuid" - "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/ext/tryfunc" - "github.com/hashicorp/hcl/v2/hclparse" - "github.com/sirupsen/logrus" - "github.com/srevinsaju/togomak/v1/pkg/conductor" - "github.com/srevinsaju/togomak/v1/pkg/global" - "github.com/srevinsaju/togomak/v1/pkg/meta" - "github.com/srevinsaju/togomak/v1/pkg/third-party/hashicorp/terraform/lang/funcs" - "github.com/srevinsaju/togomak/v1/pkg/ui" - "github.com/srevinsaju/togomak/v1/pkg/x" - ctyyaml "github.com/zclconf/go-cty-yaml" - "github.com/zclconf/go-cty/cty" - "github.com/zclconf/go-cty/cty/function" - "github.com/zclconf/go-cty/cty/function/stdlib" - "os" - "os/exec" - "path/filepath" - "time" -) - -type Togomak struct { - Logger *logrus.Logger - pipelineId string - cfg conductor.Config - cwd string - ectx *hcl.EvalContext - tempDir string - parser *hclparse.Parser - hclDiagWriter hcl.DiagnosticWriter -} - -func (t Togomak) Parser() *hclparse.Parser { - return t.parser -} - -func createTempDir(cwd string, pipelineId string) string { - // create temporary directory - - tempDir := filepath.Join(meta.BuildDirPrefix, "pipelines", "tmp") - err := os.MkdirAll(tempDir, 0755) - x.Must(err) - tempDir, err = os.MkdirTemp(tempDir, pipelineId) - x.Must(err) - global.SetTempDir(tempDir) - return tempDir -} - -func createHclEvalContext(cwd string, cfg conductor.Config, pipelineId string, tempDir string) *hcl.EvalContext { - // --> set up HCL context - hclContext := &hcl.EvalContext{ - Functions: map[string]function.Function{ - "abs": stdlib.AbsoluteFunc, - "abspath": funcs.AbsPathFunc, - "alltrue": funcs.AllTrueFunc, - "anytrue": funcs.AnyTrueFunc, - "basename": funcs.BasenameFunc, - "base64decode": funcs.Base64DecodeFunc, - "base64encode": funcs.Base64EncodeFunc, - "base64gzip": funcs.Base64GzipFunc, - "base64sha256": funcs.Base64Sha256Func, - "base64sha512": funcs.Base64Sha512Func, - "bcrypt": funcs.BcryptFunc, - "can": tryfunc.CanFunc, - "ceil": stdlib.CeilFunc, - "chomp": stdlib.ChompFunc, - "coalesce": funcs.CoalesceFunc, - "coalescelist": stdlib.CoalesceListFunc, - "compact": stdlib.CompactFunc, - "concat": stdlib.ConcatFunc, - "contains": stdlib.ContainsFunc, - "csvdecode": stdlib.CSVDecodeFunc, - "dirname": funcs.DirnameFunc, - "distinct": stdlib.DistinctFunc, - "element": stdlib.ElementFunc, - "endswith": funcs.EndsWithFunc, - "chunklist": stdlib.ChunklistFunc, - "file": funcs.MakeFileFunc(cwd, false), - "fileexists": funcs.MakeFileExistsFunc(cwd), - "fileset": funcs.MakeFileSetFunc(cwd), - "filebase64": funcs.MakeFileFunc(cwd, true), - "filebase64sha256": funcs.MakeFileBase64Sha256Func(cwd), - "filebase64sha512": funcs.MakeFileBase64Sha512Func(cwd), - "filemd5": funcs.MakeFileMd5Func(cwd), - "filesha1": funcs.MakeFileSha1Func(cwd), - "filesha256": funcs.MakeFileSha256Func(cwd), - "filesha512": funcs.MakeFileSha512Func(cwd), - "flatten": stdlib.FlattenFunc, - "floor": stdlib.FloorFunc, - "format": stdlib.FormatFunc, - "formatdate": stdlib.FormatDateFunc, - "formatlist": stdlib.FormatListFunc, - "indent": stdlib.IndentFunc, - "index": funcs.IndexFunc, // stdlib.IndexFunc is not compatible - "join": stdlib.JoinFunc, - "jsondecode": stdlib.JSONDecodeFunc, - "jsonencode": stdlib.JSONEncodeFunc, - "keys": stdlib.KeysFunc, - "length": funcs.LengthFunc, - "list": funcs.ListFunc, - "log": stdlib.LogFunc, - "lookup": funcs.LookupFunc, - "lower": stdlib.LowerFunc, - "map": funcs.MapFunc, - "matchkeys": funcs.MatchkeysFunc, - "max": stdlib.MaxFunc, - "md5": funcs.Md5Func, - "merge": stdlib.MergeFunc, - "min": stdlib.MinFunc, - "one": funcs.OneFunc, - "parseint": stdlib.ParseIntFunc, - "pathexpand": funcs.PathExpandFunc, - "pow": stdlib.PowFunc, - "range": stdlib.RangeFunc, - "regex": stdlib.RegexFunc, - "regexall": stdlib.RegexAllFunc, - "replace": funcs.ReplaceFunc, - "reverse": stdlib.ReverseListFunc, - "rsadecrypt": funcs.RsaDecryptFunc, - "sensitive": funcs.SensitiveFunc, - "nonsensitive": funcs.NonsensitiveFunc, - "setintersection": stdlib.SetIntersectionFunc, - "setproduct": stdlib.SetProductFunc, - "setsubtract": stdlib.SetSubtractFunc, - "setunion": stdlib.SetUnionFunc, - "sha1": funcs.Sha1Func, - "sha256": funcs.Sha256Func, - "sha512": funcs.Sha512Func, - "signum": stdlib.SignumFunc, - "slice": stdlib.SliceFunc, - "sort": stdlib.SortFunc, - "split": stdlib.SplitFunc, - "startswith": funcs.StartsWithFunc, - "strcontains": funcs.StrContainsFunc, - "strrev": stdlib.ReverseFunc, - "substr": stdlib.SubstrFunc, - "sum": funcs.SumFunc, - "textdecodebase64": funcs.TextDecodeBase64Func, - "textencodebase64": funcs.TextEncodeBase64Func, - "timestamp": funcs.TimestampFunc, - "timeadd": stdlib.TimeAddFunc, - "timecmp": funcs.TimeCmpFunc, - "title": stdlib.TitleFunc, - "tostring": funcs.MakeToFunc(cty.String), - "tonumber": funcs.MakeToFunc(cty.Number), - "tobool": funcs.MakeToFunc(cty.Bool), - "toset": funcs.MakeToFunc(cty.Set(cty.DynamicPseudoType)), - "tolist": funcs.MakeToFunc(cty.List(cty.DynamicPseudoType)), - "tomap": funcs.MakeToFunc(cty.Map(cty.DynamicPseudoType)), - "transpose": funcs.TransposeFunc, - "trim": stdlib.TrimFunc, - "trimprefix": stdlib.TrimPrefixFunc, - "trimspace": stdlib.TrimSpaceFunc, - "trimsuffix": stdlib.TrimSuffixFunc, - "try": tryfunc.TryFunc, - "upper": stdlib.UpperFunc, - "urlencode": funcs.URLEncodeFunc, - "uuid": funcs.UUIDFunc, - "uuidv5": funcs.UUIDV5Func, - "values": stdlib.ValuesFunc, - "which": function.New(&function.Spec{ - Params: []function.Parameter{ - { - Name: "executable", - AllowDynamicType: true, - Type: cty.String, - }, - }, - Type: function.StaticReturnType(cty.String), - Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { - path, err := exec.LookPath(args[0].AsString()) - if err != nil { - return cty.StringVal(""), err - } - return cty.StringVal(path), nil - }, - Description: "Returns the absolute path to an executable in the current PATH.", - }), - "yamldecode": ctyyaml.YAMLDecodeFunc, - "yamlencode": ctyyaml.YAMLEncodeFunc, - "zipmap": stdlib.ZipmapFunc, - - "ansifmt": ui.AnsiFunc, - "env": function.New(&function.Spec{ - Params: []function.Parameter{ - { - Name: "Key of the environment variable", - AllowDynamicType: true, - Type: cty.String, - }, - }, - VarParam: &function.Parameter{ - Name: "lists", - Description: "One or more lists of strings to join.", - Type: cty.String, - }, - Type: function.StaticReturnType(cty.String), - Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { - v, ok := os.LookupEnv(args[0].AsString()) - if ok { - return cty.StringVal(v), nil - } - def := args[1] - return def, nil - }, - Description: "Returns the value of the environment variable, returns the default value if environment variable is empty, else returns empty string.", - }), - }, - - Variables: map[string]cty.Value{ - "true": cty.True, - "false": cty.False, - "null": cty.NullVal(cty.DynamicPseudoType), - - "owd": cty.StringVal(cfg.Paths.Owd), - "cwd": cty.StringVal(cwd), - "hostname": cty.StringVal(cfg.Hostname), - "hostuser": cty.StringVal(cfg.User), - - "pipeline": cty.ObjectVal(map[string]cty.Value{ - "id": cty.StringVal(pipelineId), - "path": cty.StringVal(cfg.Paths.Pipeline), - "tempDir": cty.StringVal(tempDir), - }), - - "togomak": cty.ObjectVal(map[string]cty.Value{ - "version": cty.StringVal(meta.AppVersion), - "boot_time": cty.StringVal(time.Now().Format(time.RFC3339)), - "boot_time_unix": cty.NumberIntVal(time.Now().Unix()), - "pipeline_id": cty.StringVal(pipelineId), - "ci": cty.BoolVal(cfg.Behavior.Ci), - "unattended": cty.BoolVal(cfg.Behavior.Unattended), - }), - - // introduced in v1.5.0 - "ansi": cty.ObjectVal(map[string]cty.Value{ - "bg": cty.ObjectVal(map[string]cty.Value{ - "red": cty.StringVal("\033[41m"), - "green": cty.StringVal("\033[42m"), - "yellow": cty.StringVal("\033[43m"), - "blue": cty.StringVal("\033[44m"), - "purple": cty.StringVal("\033[45m"), - "cyan": cty.StringVal("\033[46m"), - "white": cty.StringVal("\033[47m"), - "grey": cty.StringVal("\033[100m"), - }), - "fg": cty.ObjectVal(map[string]cty.Value{ - "red": cty.StringVal("\033[31m"), - "green": cty.StringVal("\033[32m"), - "yellow": cty.StringVal("\033[33m"), - "blue": cty.StringVal("\033[34m"), - "purple": cty.StringVal("\033[35m"), - "cyan": cty.StringVal("\033[36m"), - "white": cty.StringVal("\033[37m"), - "grey": cty.StringVal("\033[90m"), - "bold": cty.StringVal("\033[1m"), - "italic": cty.StringVal("\033[3m"), - "underline": cty.StringVal("\033[4m"), - }), - - "reset": cty.StringVal("\033[0m"), - }), - }, - } - global.SetHclEvalContext(hclContext) - return hclContext -} - -func NewContextWithTogomak(cfg conductor.Config) (Togomak, context.Context) { - - logger := NewLogger(cfg) - global.SetLogger(logger) - - if !cfg.Behavior.Child.Enabled { - logger.Infof("%s (version=%s)", meta.AppName, meta.AppVersion) - } - - pipelineId := uuid.New().String() - // --> set up the working directory - cwd := Chdir(cfg, logger) - - tempDir := createTempDir(cwd, pipelineId) - - hclContext := createHclEvalContext(cwd, cfg, pipelineId, tempDir) - - parser := hclparse.NewParser() - global.SetHclParser(parser) - - diagnosticTextWriter := hcl.NewDiagnosticTextWriter(os.Stdout, parser.Files(), 0, true) - global.SetHclDiagWriter(diagnosticTextWriter) - - ctx := context.Background() - - t := Togomak{ - Logger: logger, - pipelineId: pipelineId, - cfg: cfg, - cwd: cwd, - hclDiagWriter: diagnosticTextWriter, - parser: parser, - ectx: hclContext, - tempDir: tempDir, - } - - return t, ctx -} diff --git a/pkg/orchestra/format.go b/pkg/orchestra/format.go index a2842ef..13805de 100644 --- a/pkg/orchestra/format.go +++ b/pkg/orchestra/format.go @@ -12,7 +12,6 @@ import ( ) func Format(cfg conductor.Config, check bool, recursive bool) error { - t, _ := NewContextWithTogomak(cfg) togomak := conductor.NewTogomak(cfg) var toFormat []string @@ -20,19 +19,19 @@ func Format(cfg conductor.Config, check bool, recursive bool) error { if recursive { matches, err := doublestar.Glob("**/*.hcl") for _, path := range matches { - t.Logger.Tracef("Found %s", path) + togomak.Logger.Tracef("Found %s", path) data, err := os.ReadFile(path) if err != nil { return err } outSrc := hclwrite.Format(data) if !bytes.Equal(outSrc, data) { - t.Logger.Tracef("%s needs formatting", path) + togomak.Logger.Tracef("%s needs formatting", path) toFormat = append(toFormat, path) } } if err != nil { - t.Logger.Fatalf("Error while globbing for **/*.hcl: %s", err) + togomak.Logger.Fatalf("Error while globbing for **/*.hcl: %s", err) } } else { fDir := parse.ConfigFileDir(togomak.Config.Paths) @@ -55,7 +54,7 @@ func Format(cfg conductor.Config, check bool, recursive bool) error { } outSrc := hclwrite.Format(data) if !bytes.Equal(outSrc, data) { - t.Logger.Tracef("%s needs formatting", fn) + togomak.Logger.Tracef("%s needs formatting", fn) toFormat = append(toFormat, fn) } } diff --git a/pkg/orchestra/imports.go b/pkg/orchestra/imports.go index 6b44419..3a5de5f 100644 --- a/pkg/orchestra/imports.go +++ b/pkg/orchestra/imports.go @@ -8,7 +8,7 @@ import ( "github.com/srevinsaju/togomak/v1/pkg/path" ) -func ExpandImports(ctx context.Context, pipe *ci.Pipeline, parser *hclparse.Parser, paths path.Path) (*ci.Pipeline, hcl.Diagnostics) { +func ExpandImports(ctx context.Context, pipe *ci.Pipeline, parser *hclparse.Parser, paths *path.Path) (*ci.Pipeline, hcl.Diagnostics) { var d hcl.Diagnostics var diags hcl.Diagnostics diff --git a/pkg/orchestra/list.go b/pkg/orchestra/list.go index 1da3756..a0e2b25 100644 --- a/pkg/orchestra/list.go +++ b/pkg/orchestra/list.go @@ -1,46 +1,27 @@ package orchestra import ( - "context" "fmt" - "github.com/google/uuid" "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/hclparse" "github.com/srevinsaju/togomak/v1/pkg/ci" "github.com/srevinsaju/togomak/v1/pkg/conductor" - "github.com/srevinsaju/togomak/v1/pkg/global" - "github.com/srevinsaju/togomak/v1/pkg/meta" "github.com/srevinsaju/togomak/v1/pkg/ui" - "github.com/srevinsaju/togomak/v1/pkg/x" "os" - "path/filepath" ) func List(cfg conductor.Config) error { - logger := NewLogger(cfg) - parser := hclparse.NewParser() - // TODO: move this to a function - // TODO: reduce duplication - pipelineId := uuid.New().String() - tmpDir := filepath.Join(meta.BuildDirPrefix, "pipelines", "tmp") - err := os.MkdirAll(tmpDir, 0755) - x.Must(err) - tmpDir, err = os.MkdirTemp(tmpDir, pipelineId) - x.Must(err) - global.SetTempDir(tmpDir) + togomak := conductor.NewTogomak(cfg) + logger := togomak.Logger + ctx := togomak.Context - // TODO: move this to a function - ctx := context.Background() - cwd := Chdir(cfg, logger) - - dgwriter := hcl.NewDiagnosticTextWriter(os.Stdout, parser.Files(), 0, true) - pipe, hclDiags := ci.Read(cfg.Paths, parser) + dgwriter := hcl.NewDiagnosticTextWriter(os.Stdout, togomak.Parser.Files(), 0, true) + pipe, hclDiags := ci.Read(cfg.Paths, togomak.Parser) if hclDiags.HasErrors() { logger.Fatal(dgwriter.WriteDiagnostics(hclDiags)) } - pipe, d := pipe.ExpandImports(ctx, parser, cwd) + pipe, d := pipe.ExpandImports(ctx, togomak.Parser, togomak.Config.Paths.Cwd) hclDiags = hclDiags.Extend(d) for _, stage := range pipe.Stages { diff --git a/pkg/orchestra/logging.go b/pkg/orchestra/logging.go index 718c6dd..22ba1e9 100644 --- a/pkg/orchestra/logging.go +++ b/pkg/orchestra/logging.go @@ -1,46 +1 @@ package orchestra - -import ( - "github.com/sirupsen/logrus" - "github.com/srevinsaju/togomak/v1/pkg/conductor" - "os" -) - -func NewLogger(cfg conductor.Config) *logrus.Logger { - logger := logrus.New() - logger.SetOutput(os.Stdout) - logger.SetFormatter(&logrus.TextFormatter{ - FullTimestamp: false, - DisableTimestamp: cfg.Behavior.Child.Enabled, - }) - switch cfg.Interface.Verbosity { - case -1: - case 0: - logger.SetLevel(logrus.InfoLevel) - break - case 1: - logger.SetLevel(logrus.DebugLevel) - break - default: - logger.SetLevel(logrus.TraceLevel) - break - } - if cfg.Behavior.Ci { - logger.SetFormatter(&logrus.TextFormatter{ - DisableColors: false, - EnvironmentOverrideColors: false, - ForceColors: true, - ForceQuote: false, - }) - } - if cfg.Behavior.Child.Enabled { - logger.SetFormatter(&logrus.TextFormatter{ - DisableTimestamp: true, - DisableColors: false, - EnvironmentOverrideColors: false, - ForceColors: true, - ForceQuote: false, - }) - } - return logger -} diff --git a/pkg/orchestra/orchestra.go b/pkg/orchestra/orchestra.go index 61e054e..6c6a041 100644 --- a/pkg/orchestra/orchestra.go +++ b/pkg/orchestra/orchestra.go @@ -19,9 +19,9 @@ import ( "sync" ) -func ExpandGlobalParams(t *Togomak, cfg conductor.Config) { +func ExpandGlobalParams(togomak *conductor.Togomak) { paramsGo := make(map[string]cty.Value) - if cfg.Behavior.Child.Enabled { + if togomak.Config.Behavior.Child.Enabled { m := make(map[string]string) for _, e := range os.Environ() { if i := strings.Index(e, "="); i >= 0 { @@ -37,15 +37,13 @@ func ExpandGlobalParams(t *Togomak, cfg conductor.Config) { } } global.EvalContextMutex.Lock() - t.ectx.Variables[ci.ParamBlock] = cty.ObjectVal(paramsGo) + togomak.EvalContext.Variables[ci.ParamBlock] = cty.ObjectVal(paramsGo) global.EvalContextMutex.Unlock() } func Perform(togomak *conductor.Togomak) int { cfg := togomak.Config - - t, ctx := NewContextWithTogomak(cfg) - ctx, cancel := context.WithCancel(ctx) + ctx, cancel := context.WithCancel(togomak.Context) logger := togomak.Logger logger.Debugf("starting watchdogs and signal handlers") @@ -55,23 +53,23 @@ func Perform(togomak *conductor.Togomak) int { defer h.WriteDiagnostics() // region: external parameters - ExpandGlobalParams(&t, cfg) + ExpandGlobalParams(togomak) // endregion // --> parse the config file // we will now read the pipeline from togomak.hcl - pipe, hclDiags := ci.Read(togomak.Config.Paths, t.parser) + pipe, hclDiags := ci.Read(togomak.Config.Paths, togomak.Parser) if hclDiags.HasErrors() { - logger.Fatal(t.hclDiagWriter.WriteDiagnostics(hclDiags)) + logger.Fatal(togomak.DiagWriter.WriteDiagnostics(hclDiags)) } // whitelist all stages if unspecified filterList := cfg.Pipeline.Filtered // write the pipeline to the temporary directory - pipelineFilePath := filepath.Join(t.cwd, t.tempDir, meta.ConfigFileName) + pipelineFilePath := filepath.Join(togomak.Process.TempDir, meta.ConfigFileName) var pipelineData []byte - for _, f := range t.parser.Files() { + for _, f := range togomak.Parser.Files() { pipelineData = append(pipelineData, f.Bytes...) } @@ -114,8 +112,8 @@ func Perform(togomak *conductor.Togomak) int { // endregion: interrupt h opts := []runnable.Option{ - runnable.WithPaths(togomak.Config.Paths), runnable.WithBehavior(togomak.Config.Behavior), + runnable.WithPaths(togomak.Config.Paths), } var diagsMutex sync.Mutex @@ -125,7 +123,7 @@ func Perform(togomak *conductor.Togomak) int { // we parse the TOGOMAK_ENV file at the beginning of every layer // this allows us to have different environments for different layers - d = ExpandOutputs(t, logger) + d = ExpandOutputs(togomak) h.Diags.Extend(d) if h.Diags.HasErrors() { break diff --git a/pkg/orchestra/outputs.go b/pkg/orchestra/outputs.go index 0f39f21..dd89f0d 100644 --- a/pkg/orchestra/outputs.go +++ b/pkg/orchestra/outputs.go @@ -3,8 +3,8 @@ package orchestra import ( "github.com/hashicorp/go-envparse" "github.com/hashicorp/hcl/v2" - "github.com/sirupsen/logrus" "github.com/srevinsaju/togomak/v1/pkg/ci" + "github.com/srevinsaju/togomak/v1/pkg/conductor" "github.com/srevinsaju/togomak/v1/pkg/global" "github.com/srevinsaju/togomak/v1/pkg/meta" "github.com/srevinsaju/togomak/v1/pkg/x" @@ -13,9 +13,10 @@ import ( "path/filepath" ) -func ExpandOutputs(t Togomak, logger *logrus.Logger) hcl.Diagnostics { +func ExpandOutputs(togomak *conductor.Togomak) hcl.Diagnostics { var diags hcl.Diagnostics - togomakEnvFile := filepath.Join(t.cwd, t.tempDir, meta.OutputEnvFile) + logger := togomak.Logger + togomakEnvFile := filepath.Join(togomak.Process.TempDir, meta.OutputEnvFile) logger.Tracef("%s will be stored and exported here: %s", meta.OutputEnvVar, togomakEnvFile) envFile, err := os.OpenFile(togomakEnvFile, os.O_RDONLY|os.O_CREATE, 0644) if err == nil { @@ -34,7 +35,7 @@ func ExpandOutputs(t Togomak, logger *logrus.Logger) hcl.Diagnostics { ee[k] = cty.StringVal(v) } global.EvalContextMutex.Lock() - t.ectx.Variables[ci.OutputBlock] = cty.ObjectVal(ee) + togomak.EvalContext.Variables[ci.OutputBlock] = cty.ObjectVal(ee) global.EvalContextMutex.Unlock() } else { logger.Warnf("could not open %s file, ignoring... :%s", meta.OutputEnvVar, err) diff --git a/pkg/orchestra/run.go b/pkg/orchestra/run.go index 5dfef5c..64a1e26 100644 --- a/pkg/orchestra/run.go +++ b/pkg/orchestra/run.go @@ -13,7 +13,7 @@ import ( ) func RunWithRetries(runnableId string, runnable ci.Block, ctx context.Context, handler *handler.Handler, logger *logrus.Logger, opts ...runnable.Option) { - stageDiags := runnable.Run(ctx) + stageDiags := runnable.Run(ctx, opts...) handler.Tracker.AppendCompleted(runnable) logger.Tracef("signaling runnable %s", runnableId) diff --git a/pkg/orchestra/utils.go b/pkg/orchestra/utils.go index da31915..22ba1e9 100644 --- a/pkg/orchestra/utils.go +++ b/pkg/orchestra/utils.go @@ -1,28 +1 @@ package orchestra - -import ( - "github.com/sirupsen/logrus" - "github.com/srevinsaju/togomak/v1/pkg/conductor" - "github.com/srevinsaju/togomak/v1/pkg/meta" - "github.com/srevinsaju/togomak/v1/pkg/x" - "os" - "path/filepath" -) - -func Chdir(cfg conductor.Config, logger *logrus.Logger) string { - cwd := cfg.Paths.Cwd - if cwd == "" { - cwd = filepath.Dir(cfg.Paths.Pipeline) - if filepath.Base(cwd) == meta.BuildDirPrefix { - cwd = filepath.Dir(cwd) - } - } - err := os.Chdir(cwd) - if err != nil { - logger.Fatal(err) - } - cwd, err = os.Getwd() - x.Must(err) - return cwd - -} diff --git a/pkg/parse/parse.go b/pkg/parse/parse.go index 8d36c2c..b33f9e8 100644 --- a/pkg/parse/parse.go +++ b/pkg/parse/parse.go @@ -9,7 +9,7 @@ import ( // ConfigFilePath returns the path to the configuration file. If the path is not absolute, it is assumed to be // relative to the working directory // DEPRECATED: use configFileDir instead -func ConfigFilePath(paths path.Path) string { +func ConfigFilePath(paths *path.Path) string { pipelineFilePath := paths.Pipeline if pipelineFilePath == "" { pipelineFilePath = meta.ConfigFileName @@ -21,6 +21,6 @@ func ConfigFilePath(paths path.Path) string { return pipelineFilePath } -func ConfigFileDir(paths path.Path) string { +func ConfigFileDir(paths *path.Path) string { return filepath.Dir(ConfigFilePath(paths)) } diff --git a/pkg/path/models.go b/pkg/path/models.go index 67a7d85..dd2143c 100644 --- a/pkg/path/models.go +++ b/pkg/path/models.go @@ -1,7 +1,5 @@ package path -import "github.com/srevinsaju/togomak/v1/pkg/meta" - type Path struct { // Pipeline is the path to the pipeline file Pipeline string @@ -12,12 +10,3 @@ type Path struct { // Cwd is the current working directory Cwd string } - -func NewDefaultPath() Path { - - return Path{ - Pipeline: meta.ConfigFileName, - Owd: ".", - Cwd: ".", - } -} diff --git a/pkg/runnable/options.go b/pkg/runnable/options.go index 9addb32..2288623 100644 --- a/pkg/runnable/options.go +++ b/pkg/runnable/options.go @@ -10,9 +10,9 @@ type Config struct { Parent *ParentConfig Hook bool - Paths path.Path + Paths *path.Path - Behavior behavior.Behavior + Behavior *behavior.Behavior } type ParentConfig struct { @@ -30,7 +30,7 @@ func WithStatus(status StatusType) Option { } } -func WithPaths(paths path.Path) Option { +func WithPaths(paths *path.Path) Option { return func(c *Config) { c.Paths = paths } @@ -42,7 +42,7 @@ func WithParent(parent ParentConfig) Option { } } -func WithBehavior(behavior behavior.Behavior) Option { +func WithBehavior(behavior *behavior.Behavior) Option { return func(c *Config) { c.Behavior = behavior } @@ -53,8 +53,8 @@ func NewDefaultConfig() *Config { Status: &Status{Status: StatusRunning}, Parent: nil, Hook: false, - Paths: path.NewDefaultPath(), - Behavior: behavior.NewDefaultBehavior(), + Paths: nil, + Behavior: nil, } }