From 770c33a396cc77152dec57c9a11aeede47cfc732 Mon Sep 17 00:00:00 2001 From: Will Alexander Date: Thu, 9 Dec 2021 09:49:59 -0800 Subject: [PATCH] Exclude most recent image when pruning a repo This commit adds any tags associated with the most-recently-pushed image in a repo to the whitelist when pruning, ensuring that Thermite will always leave at least one image. As part of these changes, this commit also refactors the collection and processing of ECR image details to reduce cyclomatic complexity. --- pkg/prune/prune.go | 97 ++++++++++++++++++++++++----------------- pkg/prune/prune_test.go | 47 ++++++++++++++++++++ 2 files changed, 105 insertions(+), 39 deletions(-) diff --git a/pkg/prune/prune.go b/pkg/prune/prune.go index 51bad6e..a57d67f 100644 --- a/pkg/prune/prune.go +++ b/pkg/prune/prune.go @@ -202,9 +202,8 @@ func (gc *Client) PruneRepo(ctx context.Context, name string, until time.Time, e period, name, ) - whitelist := newWhitelist(excluded...) - pruneableImageIDs := []*ecr.ImageIdentifier{} - var pageErr error + imageDetails := make([]*ecr.ImageDetail, 0) + var mostRecentImageDetail *ecr.ImageDetail if err := gc.client.DescribeImagesPagesWithContext( ctx, &ecr.DescribeImagesInput{ @@ -213,41 +212,13 @@ func (gc *Client) PruneRepo(ctx context.Context, name string, until time.Time, e }, func(page *ecr.DescribeImagesOutput, lastPage bool) bool { for _, imageDetail := range page.ImageDetails { - excluded := false - for _, imageTag := range imageDetail.ImageTags { - if imageTag == nil { - pageErr = fmt.Errorf( - "found unexpected nil image tag in Elastic Container Registry repository %s", - *repo.RepositoryUri, - ) - return false - } - imageRef := fmt.Sprintf("%s:%s", *repo.RepositoryUri, *imageTag) - if whitelist.IsExcluded(imageRef) { - excluded = true - break - } - } - if excluded { - continue - } - if imageDetail.ImagePushedAt == nil { - pageErr = fmt.Errorf( - "found unexpected nil image pushed at time in Elastic Container Registry repository %s", - *repo.RepositoryUri, - ) - return false - } - pushedAt := imageDetail.ImagePushedAt.UTC() - period := -time.Duration(period) * 24 * time.Hour - cutoff := until.UTC().Add(period) - if pushedAt.After(cutoff) { - continue - } - for _, imageTag := range imageDetail.ImageTags { - pruneableImageIDs = append(pruneableImageIDs, &ecr.ImageIdentifier{ImageTag: imageTag}) + isFirst := mostRecentImageDetail == nil + isMostRecent := isFirst || imageDetail.ImagePushedAt.After(*mostRecentImageDetail.ImagePushedAt) + if isMostRecent { + mostRecentImageDetail = imageDetail } } + imageDetails = append(imageDetails, page.ImageDetails...) return true }, ); err != nil { @@ -258,9 +229,57 @@ func (gc *Client) PruneRepo(ctx context.Context, name string, until time.Time, e err, ) } - if pageErr != nil { - span.Finish(tracer.WithError(pageErr)) - return pruned, pageErr + if mostRecentImageDetail != nil { + log.Println("*****") + mostRecentImageIDs := make([]*ecr.ImageIdentifier, 0, len(mostRecentImageDetail.ImageTags)) + for _, imageTag := range mostRecentImageDetail.ImageTags { + imageID := &ecr.ImageIdentifier{ImageTag: imageTag} + mostRecentImageIDs = append(mostRecentImageIDs, imageID) + } + mostRecentImageRefs, err := repoImageRefsFromURIAndImageIDs( + ctx, + *repo.RepositoryUri, + mostRecentImageIDs, + ) + if err != nil { + span.Finish(tracer.WithError(err)) + return nil, err + } + excluded = append(excluded, mostRecentImageRefs...) + } + whitelist := newWhitelist(excluded...) + log.Println(whitelist) + pruneableImageIDs := make([]*ecr.ImageIdentifier, 0, len(imageDetails)) +excluded: + for _, imageDetail := range imageDetails { + if imageDetail.ImagePushedAt == nil { + span.Finish(tracer.WithError(err)) + return nil, fmt.Errorf( + "found unexpected nil image pushed at time in Elastic Container Registry repository %s", + *repo.RepositoryUri, + ) + } + pushedAt := imageDetail.ImagePushedAt.UTC() + cutoff := until.UTC().Add(-time.Duration(period) * 24 * time.Hour) + if pushedAt.After(cutoff) { + continue excluded + } + for _, imageTag := range imageDetail.ImageTags { + if imageTag == nil { + span.Finish(tracer.WithError(err)) + return nil, fmt.Errorf( + "found unexpected nil image tag in Elastic Container Registry repository %s", + *repo.RepositoryUri, + ) + } + imageRef := fmt.Sprintf("%s:%s", *repo.RepositoryUri, *imageTag) + if whitelist.IsExcluded(imageRef) { + continue excluded + } + log.Println(imageRef, "is prunable") + imageID := &ecr.ImageIdentifier{ImageTag: imageTag} + pruneableImageIDs = append(pruneableImageIDs, imageID) + } } log.Printf( "found %d unique pruneable images for Elastic Container Registry repository %s", diff --git a/pkg/prune/prune_test.go b/pkg/prune/prune_test.go index 88920dc..c76b97d 100644 --- a/pkg/prune/prune_test.go +++ b/pkg/prune/prune_test.go @@ -289,6 +289,53 @@ func TestGarbageCollector_PruneAllRepos(t *testing.T) { }, DeletedCount: 1, }, + { + Name: "ExcludeMostRecent", + Repositories: []*ecr.Repository{ + { + RepositoryArn: aws.String( + "arn:aws:ecr:us-east-1:000123456789:repository/thermite", + ), + RepositoryName: aws.String("thermite"), + RepositoryUri: aws.String( + "000123456789.dkr.ecr.us-east-1.amazonaws.com/thermite", + ), + }, + }, + TagsByResourceARN: map[string][]*ecr.Tag{ + "arn:aws:ecr:us-east-1:000123456789:repository/thermite": { + { + Key: aws.String("thermite:prune-period"), + Value: aws.String("30"), + }, + }, + }, + ImageDetailsByRepositoryName: map[string][]*ecr.ImageDetail{ + "thermite": { + { + ImagePushedAt: aws.Time(until.Add(-(30*24 + 1) * time.Hour)), + ImageTags: []*string{ + aws.String("0437aec133abca7f3d054a5be48dde8ed9b2af22"), + }, + }, + { + ImagePushedAt: aws.Time(until.Add(-(30*24 + 2) * time.Hour)), + ImageTags: []*string{ + aws.String("878d0cb2b7e6f6017c096fa613b1b521b95325a6"), + }, + }, + }, + }, + Opts: []Option{WithRemoveImages()}, + Until: until, + Excluded: []string{ + "000123456789.dkr.ecr.us-east-1.amazonaws.com/thermite:5379a3dcddb42eb007a68ea7990c643066263fb8", + }, + Pruned: []string{ + "000123456789.dkr.ecr.us-east-1.amazonaws.com/thermite:878d0cb2b7e6f6017c096fa613b1b521b95325a6", + }, + DeletedCount: 1, + }, { Name: "WithDryRun", Repositories: []*ecr.Repository{