feat: add multiwriter to add a file log sink

CHANGELOG.md

@@ -5,7 +5,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [v2.0.0-alpha.11]
+- Add `--logging.local.file` and `--logging.local.file.path` for writing logs to file.
+
+## [v2.0.0-alpha.10]
 - Fix `path.module` incorrectly being populated for togomak modules
 
 ## [v2.0.0-alpha.9]

cmd/togomak/main.go

@@ -9,6 +9,7 @@ import (
 	"github.com/srevinsaju/togomak/v1/internal/ci"
 	"github.com/srevinsaju/togomak/v1/internal/filter"
 	"github.com/srevinsaju/togomak/v1/internal/global"
+	"github.com/srevinsaju/togomak/v1/internal/logging"
 	"github.com/srevinsaju/togomak/v1/internal/meta"
 	"github.com/srevinsaju/togomak/v1/internal/orchestra"
 	"github.com/srevinsaju/togomak/v1/internal/path"
@@ -157,6 +158,29 @@ func main() {
 				"Variables set this way take precedence over variables set in the pipeline file.",
 			Aliases: []string{"variable"},
 		},
+		&cli.BoolFlag{
+			Name:    "logging.remote.google-cloud",
+			Usage:   "Enable remote logging to Google Cloud",
+			EnvVars: []string{"TOGOMAK_LOGGING_REMOTE_GCLOUD"},
+			Value:   false,
+		},
+		&cli.StringFlag{
+			Name:    "logging.remote.google-cloud.project",
+			Usage:   "Google Cloud project ID where logs are ingested",
+			EnvVars: []string{"TOGOMAK_LOGGING_REMOTE_GCLOUD_PROJECT", "GOOGLE_CLOUD_PROJECT"},
+		},
+		&cli.BoolFlag{
+			Name:    "logging.local.file",
+			Usage:   "Enable local logging to a file",
+			EnvVars: []string{"TOGOMAK_LOGGING_LOCAL_FILE"},
+			Value:   false,
+		},
+		&cli.StringFlag{
+			Name:    "logging.local.file.path",
+			Usage:   "Path to the file where logs are written",
+			EnvVars: []string{"TOGOMAK_LOGGING_LOCAL_FILE_PATH"},
+			Value:   "togomak.log",
+		},
 	}
 	cli.VersionFlag = &cli.BoolFlag{
 		Name:    "version",
@@ -261,13 +285,26 @@ func newConfigFromCliContext(ctx *cli.Context) ci.ConductorConfig {
 			DryRun:      ctx.Bool("dry-run"),
 		},
 		Variables: variables,
+
+		Logging: logging.Config{
+			Verbosity:     verboseCount,
+			Child:         ctx.Bool("child"),
+			IsCI:          ctx.Bool("ci"),
+			JSON:          ctx.Bool("json"),
+			CorrelationID: "",
+			Sinks:         logging.ParseSinksFromCLI(ctx),
+		},
 	}
 	return cfg
 }
 
 func run(ctx *cli.Context) error {
 	cfg := newConfigFromCliContext(ctx)
-	logger := ci.NewLogger(cfg)
+	logger, err := logging.New(cfg.Logging)
+	if err != nil {
+		panic(err)
+	}
+
 	global.SetLogger(logger)
 
 	t := ci.NewConductor(cfg)

go.mod

@@ -3,6 +3,7 @@ module github.com/srevinsaju/togomak/v1
 go 1.20
 
 require (
+	cloud.google.com/go/logging v1.7.0
 	code.gitea.io/gitea v1.19.3
 	github.com/AlecAivazis/survey/v2 v2.3.6
 	github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2
@@ -25,6 +26,7 @@ require (
 	github.com/mattn/go-isatty v0.0.17
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/moby/sys/mountinfo v0.6.2
+	github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5
 	github.com/sirupsen/logrus v1.9.2
 	github.com/spf13/afero v1.9.5
 	github.com/stretchr/testify v1.8.2
@@ -33,13 +35,15 @@ require (
 	github.com/zclconf/go-cty-yaml v1.0.3
 	golang.org/x/crypto v0.14.0
 	golang.org/x/text v0.13.0
+	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1
 )
 
 require (
 	cloud.google.com/go v0.110.0 // indirect
 	cloud.google.com/go/compute v1.19.1 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	cloud.google.com/go/iam v0.13.0 // indirect
+	cloud.google.com/go/longrunning v0.4.1 // indirect
 	cloud.google.com/go/storage v1.28.1 // indirect
 	dario.cat/mergo v1.0.0 // indirect
 	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
@@ -57,10 +61,10 @@ require (
 	github.com/ulikunitz/xz v0.5.11 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	golang.org/x/oauth2 v0.7.0 // indirect
+	golang.org/x/sync v0.1.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/api v0.114.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
 	google.golang.org/grpc v1.56.3 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
 )

go.sum

@@ -117,7 +117,10 @@ cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx
 cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI=
 cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8=
 cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08=
+cloud.google.com/go/logging v1.7.0 h1:CJYxlNNNNAMkHp9em/YEXcfJg+rPDg7YfwoRpMU+t5I=
+cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M=
 cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM=
+cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo=
 cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4=
 cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w=
 cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE=
@@ -562,6 +565,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/redis/go-redis/v9 v9.0.3 h1:+7mmR26M0IvyLxGZUHxu4GiBkJkVDid0Un+j4ScYu4k=
 github.com/redis/go-redis/v9 v9.0.3/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk=
+github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo=
+github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=

internal/ci/conductor.go

@@ -8,6 +8,7 @@ import (
 	"github.com/hashicorp/hcl/v2/hclparse"
 	"github.com/sirupsen/logrus"
 	"github.com/srevinsaju/togomak/v1/internal/conductor"
+	"github.com/srevinsaju/togomak/v1/internal/logging"
 	"github.com/srevinsaju/togomak/v1/internal/meta"
 	"github.com/srevinsaju/togomak/v1/internal/x"
 	"os"
@@ -259,7 +260,19 @@ func NewConductor(cfg ConductorConfig, opts ...ConductorOption) *Conductor {
 
 	diagWriter := hcl.NewDiagnosticTextWriter(os.Stdout, parser.Files(), 0, true)
 
-	logger := NewLogger(cfg)
+	process := NewProcess(cfg)
+	// create a new logger derived from conductor configurations
+	logger, err := logging.New(logging.Config{
+		Verbosity:     cfg.Logging.Verbosity,
+		Child:         cfg.Logging.Child,
+		IsCI:          cfg.Logging.IsCI,
+		JSON:          cfg.Logging.JSON,
+		CorrelationID: process.Id.String(),
+		Sinks:         cfg.Logging.Sinks,
+	})
+	if err != nil {
+		panic(err)
+	}
 
 	dir := Chdir(cfg, logger)
 	if dir != cfg.Paths.Cwd {
@@ -269,8 +282,6 @@ func NewConductor(cfg ConductorConfig, opts ...ConductorOption) *Conductor {
 		cfg.Paths.Module = cfg.Paths.Cwd
 	}
 
-	process := NewProcess(cfg)
-
 	c := &Conductor{
 		Parser: &Parser{
 			parser: parser,

internal/ci/conductor_config.go

@@ -2,6 +2,7 @@ package ci
 
 import (
 	"github.com/srevinsaju/togomak/v1/internal/behavior"
+	"github.com/srevinsaju/togomak/v1/internal/logging"
 	"github.com/srevinsaju/togomak/v1/internal/path"
 	"github.com/srevinsaju/togomak/v1/internal/rules"
 )
@@ -33,4 +34,6 @@ type ConductorConfig struct {
 	Behavior *behavior.Behavior
 
 	Variables Variables
+
+	Logging logging.Config
 }

internal/logging/google_cloud.go

@@ -0,0 +1,80 @@
+package logging
+
+import (
+	"context"
+	"github.com/acarl005/stripansi"
+	"github.com/sirupsen/logrus"
+	"github.com/srevinsaju/togomak/v1/internal/meta"
+	"github.com/srevinsaju/togomak/v1/internal/x"
+	"google.golang.org/genproto/googleapis/api/monitoredres"
+	"os"
+)
+import "cloud.google.com/go/logging"
+
+var hostname string
+
+func googleCloudLoggingClient(project string) (*logging.Client, error) {
+	loggerContext := context.Background()
+	hostname = x.MustReturn(os.Hostname()).(string)
+
+	// initialize the client
+	client, err := logging.NewClient(loggerContext, project)
+	return client, err
+}
+
+type GoogleCloudLoggerHook struct {
+	client  *logging.Client
+	cfg     Config
+	project string
+}
+
+func NewGoogleCloudLoggerHook(cfg Config, project string) (*GoogleCloudLoggerHook, error) {
+	client, err := googleCloudLoggingClient(project)
+	return &GoogleCloudLoggerHook{cfg: cfg, client: client, project: project}, err
+}
+
+func (h *GoogleCloudLoggerHook) Fire(entry *logrus.Entry) error {
+	// upload to google cloud logging
+	// using google cloud API
+	// https://cloud.google.com/logging/docs/reference/libraries#client-libraries-install-gow
+	client := h.client
+	logger := client.Logger(meta.AppName)
+	severityLevel := logging.Default
+	switch entry.Level {
+	case logrus.DebugLevel:
+		severityLevel = logging.Debug
+	case logrus.InfoLevel:
+		severityLevel = logging.Info
+	case logrus.WarnLevel:
+		severityLevel = logging.Warning
+	case logrus.ErrorLevel:
+		severityLevel = logging.Error
+	case logrus.FatalLevel:
+		severityLevel = logging.Critical
+	case logrus.PanicLevel:
+		severityLevel = logging.Alert
+	}
+	logger.Log(logging.Entry{
+		Payload: map[string]interface{}{
+			"message": stripansi.Strip(entry.Message),
+			"labels":  entry.Data,
+			"app":     meta.AppName,
+			"version": meta.AppVersion,
+			"host":    hostname,
+		},
+		Resource: &monitoredres.MonitoredResource{Type: "global"},
+		Trace:    "togomak",
+		Severity: severityLevel,
+		Labels: map[string]string{
+			"app":          meta.AppName,
+			"version":      meta.AppVersion,
+			"instanceName": meta.AppName,
+			"instanceId":   h.cfg.CorrelationID,
+		},
+	})
+	return nil
+}
+
+func (h *GoogleCloudLoggerHook) Levels() []logrus.Level {
+	return []logrus.Level{logrus.InfoLevel, logrus.WarnLevel, logrus.ErrorLevel, logrus.FatalLevel, logrus.PanicLevel}
+}