[RD-552] Use git clone instead of github api requests, agent side che…

…cksums for checking module changes (#49)

* Use git cloning for sources

* Added checksum calculations to agent side

* Warn when updating a source with a forced version

README.md

@@ -104,7 +104,6 @@ OPTIONS:
 * location - location used when creating gcloud resources [$LOCATION]
 * zone - zone used in gcloud run jobs [$ZONE]
 * role-arn - **optional** role arn for assume role, used when creating aws resources in external account [$ROLE_ARN]
-* github-token - **optional** GitHub token for querying releases as unauthenticated rate limit is low [$GITHUB_TOKEN]
 * steps - **optional** comma separated list of steps to run [$STEPS]
 * allow-parallel - allow running steps in parallel on first execution cycle (default: **true**) [$ALLOW_PARALLEL]
 * pipeline-type - pipeline execution type (local | cloud), local is meant to be run inside the infralib image (default: **cloud**) [$PIPELINE_TYPE]
@@ -130,7 +129,6 @@ OPTIONS:
 * location - location used when creating gcloud resources [$LOCATION]
 * zone - zone used in gcloud run jobs [$ZONE]
 * role-arn - **optional** role arn for assume role, used when creating aws resources in external account [$ROLE_ARN]
-* github-token - **optional** GitHub token for querying releases as unauthenticated rate limit is low [$GITHUB_TOKEN]
 * steps - **optional** comma separated list of steps to run [$STEPS]
 * pipeline-type - pipeline execution type (local | cloud), local is meant to be run inside the infralib image (default: **cloud**) [$PIPELINE_TYPE]
 * print-logs - print terraform/helm logs to stdout when using local execution (default: **true**) [$PRINT_LOGS]
@@ -257,6 +255,10 @@ sources:
     include: []string
     exclude: []string
     force_version: bool
+    username: string
+    password: string
+    insecure: bool
+    repo_path: string
 destinations:
   - name:
     git:
@@ -316,6 +318,10 @@ Source version is overwritten by module version. Default version is **stable** w
   * include - list of module sources to exclusively include from the source repository
   * exclude - list of module sources to exclude from the source repository
   * force_version - sets the specified version to all modules that use this source, useful for specifying a branch or tag instead of semver, default **false**. **Warning!** Before changing from true to false, force a version that follows semver.
+  * username - username for git authentication
+  * password - password for git authentication, it's recommended to use custom replacement tags, e.g. `"{{ .output-custom.git-password}}"`
+  * insecure - allow insecure connection, default **false**
+  * repo_path - path to the git repository root directory, default uses Go's TempDir to create a directory named after the repository url. Use debug logging to see the path. **Warning!** Agent prunes the repo to match the remote.
 * destinations - list of destinations where the agent will push the generated step files, in addition to the default bucket
   * name - name of the destination
   * git - git repository must be accessible by the agent. For authentication, use either key or username/password. For the key and password, it's recommended to use custom replacement tags, e.g. `"{{ .output-custom.git-key }}"`

argocd/argocd.go

@@ -5,7 +5,6 @@ import (
 	_ "embed"
 	"errors"
 	"fmt"
-	"github.com/entigolabs/entigo-infralib-agent/git"
 	"github.com/entigolabs/entigo-infralib-agent/model"
 	"github.com/entigolabs/entigo-infralib-agent/util"
 	"log"
@@ -19,9 +18,9 @@ var appYaml []byte
 
 var planRegex = regexp.MustCompile(`ArgoCD Applications: (\d+) has changed objects, (\d+) has RequiredPruning objects`)
 
-func GetApplicationFile(storage git.Storage, module model.Module, source, version string, values []byte, provider model.ProviderType) ([]byte, error) {
+func GetApplicationFile(storage model.Storage, module model.Module, source, version string, values []byte, provider model.ProviderType) ([]byte, error) {
 	baseBytes := getBaseApplicationFile()
-	moduleFile, err := getModuleApplicationFile(storage, version, module.Source, source)
+	moduleFile, err := getModuleApplicationFile(storage, version, module.Source)
 	if err != nil {
 		return nil, err
 	}
@@ -77,8 +76,8 @@ func getValuesString(file string, bytes []byte, values []byte) string {
 	return strings.Join(replaceLines, "\n")
 }
 
-func getModuleApplicationFile(storage git.Storage, release, moduleSource, source string) (map[string]interface{}, error) {
-	bytes, err := storage.GetFile(source, fmt.Sprintf("modules/k8s/%s/argo-apps.yaml", moduleSource), release)
+func getModuleApplicationFile(storage model.Storage, release, moduleSource string) (map[string]interface{}, error) {
+	bytes, err := storage.GetFile(fmt.Sprintf("modules/k8s/%s/argo-apps.yaml", moduleSource), release)
 	if err != nil {
 		var fileError model.FileNotFoundError
 		if errors.As(err, &fileError) {

aws/s3.go

@@ -29,8 +29,11 @@ type S3 struct {
 
 func NewS3(ctx context.Context, awsConfig aws.Config, bucket string) *S3 {
 	return &S3{
-		ctx:    ctx,
-		awsS3:  awsS3.NewFromConfig(awsConfig),
+		ctx: ctx,
+		awsS3: awsS3.NewFromConfig(awsConfig, func(o *awsS3.Options) {
+			// Avoids checksum warn on error responses https://github.com/aws/aws-sdk-go-v2/issues/3020
+			o.DisableLogOutputChecksumValidationSkipped = true
+		}),
 		region: awsConfig.Region,
 		bucket: bucket,
 	}

cli/flags.go

@@ -21,10 +21,10 @@ func appendCmdSpecificFlags(baseFlags []cli.Flag, cmd common.Command) []cli.Flag
 	case common.DeleteCommand:
 		return append(append(baseFlags, getProviderFlags()...), &yesFlag, &deleteBucketFlag, &deleteSAFlag)
 	case common.UpdateCommand:
-		return append(append(baseFlags, getProviderFlags()...), &githubToken, &stepsFlag, &pipelineTypeFlag,
+		return append(append(baseFlags, getProviderFlags()...), &stepsFlag, &pipelineTypeFlag,
 			&logsPathFlag, &printLogsFlag, &skipBucketDelayFlag)
 	case common.RunCommand:
-		return append(append(baseFlags, getProviderFlags()...), &allowParallelFlag, &githubToken, &stepsFlag,
+		return append(append(baseFlags, getProviderFlags()...), &allowParallelFlag, &stepsFlag,
 			&pipelineTypeFlag, &logsPathFlag, &printLogsFlag, &skipBucketDelayFlag)
 	case common.PullCommand:
 		return append(append(baseFlags, getProviderFlags()...), &forceFlag)
@@ -137,16 +137,6 @@ var allowParallelFlag = cli.BoolFlag{
 	Destination: &flags.AllowParallel,
 }
 
-var githubToken = cli.StringFlag{
-	Name:        "github-token",
-	Aliases:     []string{"gt"},
-	EnvVars:     []string{"GITHUB_TOKEN"},
-	Usage:       "github token used for github requests",
-	DefaultText: "",
-	Value:       "",
-	Destination: &flags.GithubToken,
-}
-
 var yesFlag = cli.BoolFlag{
 	Name:        "yes",
 	Aliases:     []string{"y"},

common/flags.go

@@ -15,7 +15,6 @@ type Flags struct {
 	Config                  string
 	Prefix                  string
 	AllowParallel           bool
-	GithubToken             string
 	Force                   bool
 	SkipBucketCreationDelay bool
 	Steps                   cli.StringSlice

git/git.go → git/dest.go

@@ -33,7 +33,7 @@ Changes are pushed to the plan and apply branches.`
 	authorEmail = "no-reply@localhost"
 )
 
-type Client struct {
+type DestClient struct {
 	ctx      context.Context
 	auth     transport.AuthMethod
 	author   *object.Signature
@@ -44,7 +44,7 @@ type Client struct {
 	mu       sync.Mutex
 }
 
-func NewGitClient(ctx context.Context, name string, config model.Git) (*Client, error) {
+func NewDestClient(ctx context.Context, name string, config model.Git) (*DestClient, error) {
 	log.Printf("Preparing git repository %s", config.URL)
 	auth, err := getAuth(config)
 	if err != nil {
@@ -54,7 +54,7 @@ func NewGitClient(ctx context.Context, name string, config model.Git) (*Client,
 	if err != nil {
 		return nil, err
 	}
-	return &Client{
+	return &DestClient{
 		ctx:      ctx,
 		auth:     auth,
 		author:   getAuthor(config),
@@ -263,7 +263,7 @@ func createRemoteBranch(auth transport.AuthMethod, repo *git.Repository, worktre
 	})
 }
 
-func (g *Client) UpdateFiles(branch, folder string, files map[string][]byte) error {
+func (g *DestClient) UpdateFiles(branch, folder string, files map[string]model.File) error {
 	if len(files) == 0 {
 		return nil
 	}
@@ -305,7 +305,7 @@ func (g *Client) UpdateFiles(branch, folder string, files map[string][]byte) err
 	return err
 }
 
-func (g *Client) checkoutCleanBranch(branch string) error {
+func (g *DestClient) checkoutCleanBranch(branch string) error {
 	branchName := plumbing.NewBranchReferenceName(branch)
 	err := g.worktree.Checkout(&git.CheckoutOptions{
 		Branch: branchName,
@@ -316,7 +316,7 @@ func (g *Client) checkoutCleanBranch(branch string) error {
 	}
 
 	err = g.worktree.PullContext(g.ctx, &git.PullOptions{
-		ReferenceName:   plumbing.NewBranchReferenceName(branch),
+		ReferenceName:   branchName,
 		SingleBranch:    true,
 		Auth:            g.auth,
 		InsecureSkipTLS: g.insecure,
@@ -327,15 +327,15 @@ func (g *Client) checkoutCleanBranch(branch string) error {
 	return nil
 }
 
-func (g *Client) updateFiles(folder string, files map[string][]byte) error {
+func (g *DestClient) updateFiles(folder string, files map[string]model.File) error {
 	err := g.worktree.Filesystem.MkdirAll(folder, os.ModeDir)
 	if err != nil {
 		return err
 	}
 
 	currentFiles := model.NewSet[string]()
-	for path, content := range files {
-		err = updateFile(g.worktree, path, content)
+	for path, file := range files {
+		err = updateFile(g.worktree, path, file.Content)
 		if err != nil {
 			return err
 		}
@@ -361,7 +361,7 @@ func updateFile(worktree *git.Worktree, path string, content []byte) error {
 	return err
 }
 
-func (g *Client) removeUnusedFiles(path string, currentFiles model.Set[string]) error {
+func (g *DestClient) removeUnusedFiles(path string, currentFiles model.Set[string]) error {
 	infos, err := g.worktree.Filesystem.ReadDir(path)
 	if err != nil {
 		return err
@@ -397,7 +397,7 @@ func (g *Client) removeUnusedFiles(path string, currentFiles model.Set[string])
 	return g.worktree.Filesystem.Remove(path)
 }
 
-func (g *Client) hasChanges(folder string) (bool, error) {
+func (g *DestClient) hasChanges(folder string) (bool, error) {
 	status, err := g.worktree.Status()
 	if err != nil {
 		return false, err