From 9396bcb04f502cb88bd9f29028f665cedb919372 Mon Sep 17 00:00:00 2001 From: bgdanix Date: Mon, 13 Dec 2021 13:30:09 +0000 Subject: [PATCH 1/9] feat: add support for google_folder --- pkg/remote/google/config/config.go | 7 +- pkg/remote/google/google_folder_enumerator.go | 56 +++++++ pkg/remote/google/init.go | 1 + pkg/remote/google/provider.go | 7 +- pkg/remote/google/repository/asset.go | 91 ++++++++--- .../google/repository/mock_AssetRepository.go | 25 ++- pkg/remote/google_folder_scanner_test.go | 154 ++++++++++++++++++ pkg/resource/google/google_folder.go | 3 + pkg/resource/resource_types.go | 1 + 9 files changed, 317 insertions(+), 28 deletions(-) create mode 100644 pkg/remote/google/google_folder_enumerator.go create mode 100644 pkg/remote/google_folder_scanner_test.go create mode 100644 pkg/resource/google/google_folder.go diff --git a/pkg/remote/google/config/config.go b/pkg/remote/google/config/config.go index e209a03ab..d42b4e7f3 100644 --- a/pkg/remote/google/config/config.go +++ b/pkg/remote/google/config/config.go @@ -1,7 +1,8 @@ package config type GCPTerraformConfig struct { - Project string `cty:"project"` - Region string `cty:"region"` - Zone string `cty:"zone"` + Organization string `cty:"organization"` + Project string `cty:"project"` + Region string `cty:"region"` + Zone string `cty:"zone"` } diff --git a/pkg/remote/google/google_folder_enumerator.go b/pkg/remote/google/google_folder_enumerator.go new file mode 100644 index 000000000..c267b891b --- /dev/null +++ b/pkg/remote/google/google_folder_enumerator.go @@ -0,0 +1,56 @@ +package google + +import ( + "strings" + + "github.com/sirupsen/logrus" + remoteerror "github.com/snyk/driftctl/pkg/remote/error" + "github.com/snyk/driftctl/pkg/remote/google/repository" + "github.com/snyk/driftctl/pkg/resource" + "github.com/snyk/driftctl/pkg/resource/google" +) + +type GoogleFolderEnumerator struct { + repository repository.AssetRepository + factory resource.ResourceFactory +} + +func NewGoogleFolderEnumerator(repo repository.AssetRepository, factory resource.ResourceFactory) *GoogleFolderEnumerator { + return &GoogleFolderEnumerator{ + repository: repo, + factory: factory, + } +} + +func (e *GoogleFolderEnumerator) SupportedType() resource.ResourceType { + return google.GoogleFolderResourceType +} + +func (e *GoogleFolderEnumerator) Enumerate() ([]*resource.Resource, error) { + resources, err := e.repository.SearchAllFolders() + + if err != nil { + return nil, remoteerror.NewResourceListingError(err, string(e.SupportedType())) + } + + results := make([]*resource.Resource, 0, len(resources)) + + for _, res := range resources { + id := trimResourceName(res.Name) + splittedId := strings.Split(id, "folders/") + if len(splittedId) != 2 { + logrus.WithField("id", res.Name).Warn("Cannot parse google_folder ID") + continue + } + results = append( + results, + e.factory.CreateAbstractResource( + string(e.SupportedType()), + id, + map[string]interface{}{}, + ), + ) + } + + return results, err +} diff --git a/pkg/remote/google/init.go b/pkg/remote/google/init.go index 6e7646a40..d668154bf 100644 --- a/pkg/remote/google/init.go +++ b/pkg/remote/google/init.go @@ -99,6 +99,7 @@ func Init(version string, alerter *alerter.Alerter, remoteLibrary.AddEnumerator(NewGoogleComputeHealthCheckEnumerator(assetRepository, factory)) remoteLibrary.AddEnumerator(NewGoogleCloudRunServiceEnumerator(assetRepository, factory)) remoteLibrary.AddEnumerator(NewGoogleComputeNodeGroupEnumerator(assetRepository, factory)) + remoteLibrary.AddEnumerator(NewGoogleFolderEnumerator(assetRepository, factory)) err = resourceSchemaRepository.Init(terraform.GOOGLE, provider.Version(), provider.Schema()) if err != nil { diff --git a/pkg/remote/google/provider.go b/pkg/remote/google/provider.go index db7174bf2..db96b163f 100644 --- a/pkg/remote/google/provider.go +++ b/pkg/remote/google/provider.go @@ -57,8 +57,9 @@ func (p *GCPTerraformProvider) Version() string { func (p *GCPTerraformProvider) GetConfig() config.GCPTerraformConfig { return config.GCPTerraformConfig{ - Project: os.Getenv("CLOUDSDK_CORE_PROJECT"), - Region: os.Getenv("CLOUDSDK_COMPUTE_REGION"), - Zone: os.Getenv("CLOUDSDK_COMPUTE_ZONE"), + Organization: os.Getenv("CLOUDSDK_ORGANIZATION"), + Project: os.Getenv("CLOUDSDK_CORE_PROJECT"), + Region: os.Getenv("CLOUDSDK_COMPUTE_REGION"), + Zone: os.Getenv("CLOUDSDK_COMPUTE_ZONE"), } } diff --git a/pkg/remote/google/repository/asset.go b/pkg/remote/google/repository/asset.go index ff3b96a20..ef57f9c52 100644 --- a/pkg/remote/google/repository/asset.go +++ b/pkg/remote/google/repository/asset.go @@ -13,27 +13,28 @@ import ( // https://cloud.google.com/asset-inventory/docs/supported-asset-types#supported_resource_types const ( - storageBucketAssetType = "storage.googleapis.com/Bucket" - computeFirewallAssetType = "compute.googleapis.com/Firewall" - computeRouterAssetType = "compute.googleapis.com/Router" - computeInstanceAssetType = "compute.googleapis.com/Instance" - computeNetworkAssetType = "compute.googleapis.com/Network" - computeSubnetworkAssetType = "compute.googleapis.com/Subnetwork" - computeDiskAssetType = "compute.googleapis.com/Disk" - computeImageAssetType = "compute.googleapis.com/Image" - dnsManagedZoneAssetType = "dns.googleapis.com/ManagedZone" - computeInstanceGroupAssetType = "compute.googleapis.com/InstanceGroup" - bigqueryDatasetAssetType = "bigquery.googleapis.com/Dataset" - bigqueryTableAssetType = "bigquery.googleapis.com/Table" - computeAddressAssetType = "compute.googleapis.com/Address" - computeGlobalAddressAssetType = "compute.googleapis.com/GlobalAddress" - cloudFunctionsFunction = "cloudfunctions.googleapis.com/CloudFunction" - bigtableInstanceAssetType = "bigtableadmin.googleapis.com/Instance" - bigtableTableAssetType = "bigtableadmin.googleapis.com/Table" - sqlDatabaseInstanceAssetType = "sqladmin.googleapis.com/Instance" - healthCheckAssetType = "compute.googleapis.com/HealthCheck" - cloudRunServiceAssetType = "run.googleapis.com/Service" - nodeGroupAssetType = "compute.googleapis.com/NodeGroup" + storageBucketAssetType = "storage.googleapis.com/Bucket" + computeFirewallAssetType = "compute.googleapis.com/Firewall" + computeRouterAssetType = "compute.googleapis.com/Router" + computeInstanceAssetType = "compute.googleapis.com/Instance" + computeNetworkAssetType = "compute.googleapis.com/Network" + computeSubnetworkAssetType = "compute.googleapis.com/Subnetwork" + computeDiskAssetType = "compute.googleapis.com/Disk" + computeImageAssetType = "compute.googleapis.com/Image" + dnsManagedZoneAssetType = "dns.googleapis.com/ManagedZone" + computeInstanceGroupAssetType = "compute.googleapis.com/InstanceGroup" + bigqueryDatasetAssetType = "bigquery.googleapis.com/Dataset" + bigqueryTableAssetType = "bigquery.googleapis.com/Table" + computeAddressAssetType = "compute.googleapis.com/Address" + computeGlobalAddressAssetType = "compute.googleapis.com/GlobalAddress" + cloudFunctionsFunction = "cloudfunctions.googleapis.com/CloudFunction" + bigtableInstanceAssetType = "bigtableadmin.googleapis.com/Instance" + bigtableTableAssetType = "bigtableadmin.googleapis.com/Table" + sqlDatabaseInstanceAssetType = "sqladmin.googleapis.com/Instance" + healthCheckAssetType = "compute.googleapis.com/HealthCheck" + cloudRunServiceAssetType = "run.googleapis.com/Service" + nodeGroupAssetType = "compute.googleapis.com/NodeGroup" + resourcemanagerFolderAssetType = "cloudresourcemanager.googleapis.com/Folder" ) type AssetRepository interface { @@ -58,6 +59,7 @@ type AssetRepository interface { SearchAllHealthChecks() ([]*assetpb.ResourceSearchResult, error) SearchAllCloudRunServices() ([]*assetpb.ResourceSearchResult, error) SearchAllNodeGroups() ([]*assetpb.Asset, error) + SearchAllFolders() ([]*assetpb.ResourceSearchResult, error) } type assetRepository struct { @@ -140,6 +142,7 @@ func (s assetRepository) searchAllResources(ty string) ([]*assetpb.ResourceSearc computeImageAssetType, healthCheckAssetType, cloudRunServiceAssetType, + resourcemanagerFolderAssetType, }, } var results []*assetpb.ResourceSearchResult @@ -176,6 +179,48 @@ func (s assetRepository) searchAllResources(ty string) ([]*assetpb.ResourceSearc return filteredResults, nil } +func (s assetRepository) searchAllOrgResources(ty string) ([]*assetpb.ResourceSearchResult, error) { + + req := &assetpb.SearchAllResourcesRequest{ + Scope: fmt.Sprintf("organizations/%s", s.config.Organization), + AssetTypes: []string{ + resourcemanagerFolderAssetType, + }, + } + var results []*assetpb.ResourceSearchResult + + cacheKey := "SearchAllOrgResources" + cachedResults := s.cache.GetAndLock(cacheKey) + defer s.cache.Unlock(cacheKey) + if cachedResults != nil { + results = cachedResults.([]*assetpb.ResourceSearchResult) + } + + if results == nil { + it := s.client.SearchAllResources(context.Background(), req) + for { + resource, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return nil, err + } + results = append(results, resource) + } + s.cache.Put(cacheKey, results) + } + + filteredResults := []*assetpb.ResourceSearchResult{} + for _, result := range results { + if result.AssetType == ty { + filteredResults = append(filteredResults, result) + } + } + + return filteredResults, nil +} + func (s assetRepository) SearchAllBuckets() ([]*assetpb.ResourceSearchResult, error) { return s.searchAllResources(storageBucketAssetType) } @@ -259,3 +304,7 @@ func (s assetRepository) SearchAllCloudRunServices() ([]*assetpb.ResourceSearchR func (s assetRepository) SearchAllNodeGroups() ([]*assetpb.Asset, error) { return s.listAllResources(nodeGroupAssetType) } + +func (s assetRepository) SearchAllFolders() ([]*assetpb.ResourceSearchResult, error) { + return s.searchAllOrgResources(resourcemanagerFolderAssetType) +} diff --git a/pkg/remote/google/repository/mock_AssetRepository.go b/pkg/remote/google/repository/mock_AssetRepository.go index db6f347df..3202ba8d1 100644 --- a/pkg/remote/google/repository/mock_AssetRepository.go +++ b/pkg/remote/google/repository/mock_AssetRepository.go @@ -1,4 +1,4 @@ -// Code generated by mockery v0.0.0-dev. DO NOT EDIT. +// Code generated by mockery v2.9.4. DO NOT EDIT. package repository @@ -219,6 +219,29 @@ func (_m *MockAssetRepository) SearchAllFirewalls() ([]*asset.ResourceSearchResu return r0, r1 } +// SearchAllFolders provides a mock function with given fields: +func (_m *MockAssetRepository) SearchAllFolders() ([]*asset.ResourceSearchResult, error) { + ret := _m.Called() + + var r0 []*asset.ResourceSearchResult + if rf, ok := ret.Get(0).(func() []*asset.ResourceSearchResult); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*asset.ResourceSearchResult) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // SearchAllFunctions provides a mock function with given fields: func (_m *MockAssetRepository) SearchAllFunctions() ([]*asset.Asset, error) { ret := _m.Called() diff --git a/pkg/remote/google_folder_scanner_test.go b/pkg/remote/google_folder_scanner_test.go new file mode 100644 index 000000000..48041b4b2 --- /dev/null +++ b/pkg/remote/google_folder_scanner_test.go @@ -0,0 +1,154 @@ +package remote + +import ( + "testing" + + "github.com/snyk/driftctl/mocks" + "github.com/snyk/driftctl/pkg/filter" + "github.com/snyk/driftctl/pkg/remote/alerts" + "github.com/snyk/driftctl/pkg/remote/cache" + "github.com/snyk/driftctl/pkg/remote/common" + remoteerr "github.com/snyk/driftctl/pkg/remote/error" + "github.com/snyk/driftctl/pkg/remote/google" + "github.com/snyk/driftctl/pkg/remote/google/repository" + "github.com/snyk/driftctl/pkg/resource" + googleresource "github.com/snyk/driftctl/pkg/resource/google" + "github.com/snyk/driftctl/pkg/terraform" + testgoogle "github.com/snyk/driftctl/test/google" + testresource "github.com/snyk/driftctl/test/resource" + terraform2 "github.com/snyk/driftctl/test/terraform" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + assetpb "google.golang.org/genproto/googleapis/cloud/asset/v1" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestGoogleFolder(t *testing.T) { + + cases := []struct { + test string + response []*assetpb.ResourceSearchResult + responseErr error + setupAlerterMock func(alerter *mocks.AlerterInterface) + wantErr error + assertExpected func(t *testing.T, got []*resource.Resource) + }{ + { + test: "no folder", + response: []*assetpb.ResourceSearchResult{}, + wantErr: nil, + assertExpected: func(t *testing.T, got []*resource.Resource) { + assert.Len(t, got, 0) + }, + }, + { + test: "multiple folders", + response: []*assetpb.ResourceSearchResult{ + { + AssetType: "cloudresourcemanager.googleapis.com/Folder", + Name: "invalid ID", // Should be ignored + }, + { + AssetType: "cloudresourcemanager.googleapis.com/Folder", + DisplayName: "test-folder-0", + Name: "//cloudresourcemanager.googleapis.com/folders/123456789", + }, + { + AssetType: "cloudresourcemanager.googleapis.com/Folder", + DisplayName: "test-folder-1", + Name: "//cloudresourcemanager.googleapis.com/folders/121556789", + }, + { + AssetType: "cloudresourcemanager.googleapis.com/Folder", + DisplayName: "test-folder-1", + Name: "//cloudresourcemanager.googleapis.com/folders/121556677", + }, + }, + wantErr: nil, + assertExpected: func(t *testing.T, got []*resource.Resource) { + assert.Len(t, got, 3) + + assert.Equal(t, got[0].ResourceId(), "folders/123456789") + assert.Equal(t, got[0].ResourceType(), googleresource.GoogleFolderResourceType) + + assert.Equal(t, got[1].ResourceId(), "folders/121556789") + assert.Equal(t, got[1].ResourceType(), googleresource.GoogleFolderResourceType) + + assert.Equal(t, got[2].ResourceId(), "folders/121556677") + assert.Equal(t, got[2].ResourceType(), googleresource.GoogleFolderResourceType) + }, + }, + { + test: "should return access denied error", + wantErr: nil, + responseErr: status.Error(codes.PermissionDenied, "The caller does not have permission"), + setupAlerterMock: func(alerter *mocks.AlerterInterface) { + alerter.On( + "SendAlert", + googleresource.GoogleFolderResourceType, + alerts.NewRemoteAccessDeniedAlert( + common.RemoteGoogleTerraform, + remoteerr.NewResourceListingError( + status.Error(codes.PermissionDenied, "The caller does not have permission"), + googleresource.GoogleFolderResourceType, + ), + alerts.EnumerationPhase, + ), + ).Once() + }, + assertExpected: func(t *testing.T, got []*resource.Resource) { + assert.Len(t, got, 0) + }, + }, + } + + providerVersion := "3.78.0" + schemaRepository := testresource.InitFakeSchemaRepository("google", providerVersion) + googleresource.InitResourcesMetadata(schemaRepository) + factory := terraform.NewTerraformResourceFactory(schemaRepository) + + for _, c := range cases { + t.Run(c.test, func(tt *testing.T) { + scanOptions := ScannerOptions{} + providerLibrary := terraform.NewProviderLibrary() + remoteLibrary := common.NewRemoteLibrary() + + // Initialize mocks + alerter := &mocks.AlerterInterface{} + if c.setupAlerterMock != nil { + c.setupAlerterMock(alerter) + } + + assetClient, err := testgoogle.NewFakeAssetServer(c.response, c.responseErr) + if err != nil { + tt.Fatal(err) + } + + realProvider, err := terraform2.InitTestGoogleProvider(providerLibrary, providerVersion) + if err != nil { + tt.Fatal(err) + } + + repo := repository.NewAssetRepository(assetClient, realProvider.GetConfig(), cache.New(0)) + + remoteLibrary.AddEnumerator(google.NewGoogleFolderEnumerator(repo, factory)) + + testFilter := &filter.MockFilter{} + testFilter.On("IsTypeIgnored", mock.Anything).Return(false) + + s := NewScanner(remoteLibrary, alerter, scanOptions, testFilter) + got, err := s.Resources() + assert.Equal(tt, c.wantErr, err) + if err != nil { + return + } + + alerter.AssertExpectations(tt) + testFilter.AssertExpectations(tt) + if c.assertExpected != nil { + c.assertExpected(t, got) + } + }) + } +} diff --git a/pkg/resource/google/google_folder.go b/pkg/resource/google/google_folder.go new file mode 100644 index 000000000..00af951b0 --- /dev/null +++ b/pkg/resource/google/google_folder.go @@ -0,0 +1,3 @@ +package google + +const GoogleFolderResourceType = "google_folder" diff --git a/pkg/resource/resource_types.go b/pkg/resource/resource_types.go index 62204fef3..2fa664b08 100644 --- a/pkg/resource/resource_types.go +++ b/pkg/resource/resource_types.go @@ -189,6 +189,7 @@ var supportedTypes = map[string]ResourceTypeMeta{ "google_compute_global_address": {}, "google_compute_node_group": {}, "google_cloud_run_service": {}, + "google_folder": {}, "azurerm_storage_account": {}, "azurerm_storage_container": {}, From 15cb0d23bea322d8e289608bbce3ff47124e5e82 Mon Sep 17 00:00:00 2001 From: bgdanix Date: Tue, 14 Dec 2021 10:04:26 +0000 Subject: [PATCH 2/9] fix: make CLOUDSDK_ORGANIZATION optional --- pkg/remote/google/provider.go | 9 ++++ pkg/remote/google/repository/asset.go | 66 ++++++++++++------------ pkg/remote/google_folder_scanner_test.go | 2 +- 3 files changed, 44 insertions(+), 33 deletions(-) diff --git a/pkg/remote/google/provider.go b/pkg/remote/google/provider.go index db96b163f..c24190a13 100644 --- a/pkg/remote/google/provider.go +++ b/pkg/remote/google/provider.go @@ -63,3 +63,12 @@ func (p *GCPTerraformProvider) GetConfig() config.GCPTerraformConfig { Zone: os.Getenv("CLOUDSDK_COMPUTE_ZONE"), } } + +func (p *GCPTerraformProvider) SetConfig(organization string, project string, region string, zone string) config.GCPTerraformConfig { + return config.GCPTerraformConfig{ + Organization: organization, + Project: project, + Region: region, + Zone: zone, + } +} diff --git a/pkg/remote/google/repository/asset.go b/pkg/remote/google/repository/asset.go index ef57f9c52..c7abd4a1a 100644 --- a/pkg/remote/google/repository/asset.go +++ b/pkg/remote/google/repository/asset.go @@ -180,45 +180,47 @@ func (s assetRepository) searchAllResources(ty string) ([]*assetpb.ResourceSearc } func (s assetRepository) searchAllOrgResources(ty string) ([]*assetpb.ResourceSearchResult, error) { + if len(s.config.Organization) > 0 { + req := &assetpb.SearchAllResourcesRequest{ + Scope: fmt.Sprintf("organizations/%s", s.config.Organization), + AssetTypes: []string{ + resourcemanagerFolderAssetType, + }, + } + var results []*assetpb.ResourceSearchResult - req := &assetpb.SearchAllResourcesRequest{ - Scope: fmt.Sprintf("organizations/%s", s.config.Organization), - AssetTypes: []string{ - resourcemanagerFolderAssetType, - }, - } - var results []*assetpb.ResourceSearchResult - - cacheKey := "SearchAllOrgResources" - cachedResults := s.cache.GetAndLock(cacheKey) - defer s.cache.Unlock(cacheKey) - if cachedResults != nil { - results = cachedResults.([]*assetpb.ResourceSearchResult) - } + cacheKey := "SearchAllOrgResources" + cachedResults := s.cache.GetAndLock(cacheKey) + defer s.cache.Unlock(cacheKey) + if cachedResults != nil { + results = cachedResults.([]*assetpb.ResourceSearchResult) + } - if results == nil { - it := s.client.SearchAllResources(context.Background(), req) - for { - resource, err := it.Next() - if err == iterator.Done { - break + if results == nil { + it := s.client.SearchAllResources(context.Background(), req) + for { + resource, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return nil, err + } + results = append(results, resource) } - if err != nil { - return nil, err - } - results = append(results, resource) + s.cache.Put(cacheKey, results) } - s.cache.Put(cacheKey, results) - } - filteredResults := []*assetpb.ResourceSearchResult{} - for _, result := range results { - if result.AssetType == ty { - filteredResults = append(filteredResults, result) + filteredResults := []*assetpb.ResourceSearchResult{} + for _, result := range results { + if result.AssetType == ty { + filteredResults = append(filteredResults, result) + } } - } - return filteredResults, nil + return filteredResults, nil + } + return nil, nil } func (s assetRepository) SearchAllBuckets() ([]*assetpb.ResourceSearchResult, error) { diff --git a/pkg/remote/google_folder_scanner_test.go b/pkg/remote/google_folder_scanner_test.go index 48041b4b2..c6ac4ad80 100644 --- a/pkg/remote/google_folder_scanner_test.go +++ b/pkg/remote/google_folder_scanner_test.go @@ -130,7 +130,7 @@ func TestGoogleFolder(t *testing.T) { tt.Fatal(err) } - repo := repository.NewAssetRepository(assetClient, realProvider.GetConfig(), cache.New(0)) + repo := repository.NewAssetRepository(assetClient, realProvider.SetConfig("123456", "", "", ""), cache.New(0)) remoteLibrary.AddEnumerator(google.NewGoogleFolderEnumerator(repo, factory)) From 8d57e8e9928b6ab6d0f7c8cd0834859f37b77ff5 Mon Sep 17 00:00:00 2001 From: bgdanix Date: Mon, 20 Dec 2021 09:43:13 +0000 Subject: [PATCH 3/9] feat: add support for GCP scopes --- pkg/cmd/scan.go | 44 +++- pkg/driftctl.go | 1 + pkg/remote/google/config/config.go | 5 +- .../google_project_iam_member_enumerator.go | 17 +- pkg/remote/google/init.go | 6 +- pkg/remote/google/provider.go | 20 +- pkg/remote/google/repository/asset.go | 208 ++++++++++-------- .../google/repository/cloudresourcemanager.go | 50 +++-- pkg/remote/remote.go | 4 +- 9 files changed, 204 insertions(+), 151 deletions(-) diff --git a/pkg/cmd/scan.go b/pkg/cmd/scan.go index 1b006992e..44d07f32f 100644 --- a/pkg/cmd/scan.go +++ b/pkg/cmd/scan.go @@ -62,6 +62,22 @@ func NewScanCmd(opts *pkg.ScanOptions) *cobra.Command { ) } + GCPScope, _ := cmd.Flags().GetStringSlice("gcp-scope") + limitScope := make([]string, 0) + + if to == common.RemoteGoogleTerraform && len(GCPScope) > 0 { + limitScope, err = parseScopeFlag(GCPScope) + if err != nil { + return err + } + } else if to != common.RemoteGoogleTerraform && len(GCPScope) > 0 { + return errors.New("gcp-scope can only be utilized when using " + common.RemoteGoogleTerraform + " flag") + } else if to == common.RemoteGoogleTerraform && len(GCPScope) == 0 { + return errors.New("gcp-scope must be specified when using " + common.RemoteGoogleTerraform + " flag") + } + + opts.GCPScope = limitScope + outputFlag, _ := cmd.Flags().GetStringSlice("output") out, err := parseOutputFlags(outputFlag) @@ -124,6 +140,12 @@ func NewScanCmd(opts *pkg.ScanOptions) *cobra.Command { false, "Do not display anything but scan results", ) + fl.StringSliceP( + "gcp-scope", + "s", + []string{}, + "Set the GCP scope for search", + ) fl.StringArray( "filter", []string{}, @@ -239,7 +261,7 @@ func scanRun(opts *pkg.ScanOptions) error { resFactory := terraform.NewTerraformResourceFactory(resourceSchemaRepository) - err := remote.Activate(opts.To, opts.ProviderVersion, alerter, providerLibrary, remoteLibrary, scanProgress, resourceSchemaRepository, resFactory, opts.ConfigDir) + err := remote.Activate(opts.To, opts.ProviderVersion, opts.GCPScope, alerter, providerLibrary, remoteLibrary, scanProgress, resourceSchemaRepository, resFactory, opts.ConfigDir) if err != nil { return err } @@ -323,6 +345,26 @@ func scanRun(opts *pkg.ScanOptions) error { return nil } +func parseScopeFlag(scope []string) ([]string, error) { + + scopeRegex := `projects/\S*$|folders/\d*$|organizations/\S*$` + r := regexp.MustCompile(scopeRegex) + + for _, v := range scope { + if !r.MatchString(v) { + return nil, errors.Wrapf( + cmderrors.NewUsageError( + "\nAccepted formats are: projects/, folders/, organizations/", + ), + "Unable to parse GCP scope '%s'", + v, + ) + } + } + + return scope, nil +} + func parseFromFlag(from []string) ([]config.SupplierConfig, error) { configs := make([]config.SupplierConfig, 0, len(from)) diff --git a/pkg/driftctl.go b/pkg/driftctl.go index 3d0706050..5e602940e 100644 --- a/pkg/driftctl.go +++ b/pkg/driftctl.go @@ -24,6 +24,7 @@ type ScanOptions struct { Detect bool From []config.SupplierConfig To string + GCPScope []string Output []output.OutputConfig Filter *jmespath.JMESPath Quiet bool diff --git a/pkg/remote/google/config/config.go b/pkg/remote/google/config/config.go index d42b4e7f3..0e6456f27 100644 --- a/pkg/remote/google/config/config.go +++ b/pkg/remote/google/config/config.go @@ -1,8 +1,5 @@ package config type GCPTerraformConfig struct { - Organization string `cty:"organization"` - Project string `cty:"project"` - Region string `cty:"region"` - Zone string `cty:"zone"` + Scope []string `cty:"scope"` } diff --git a/pkg/remote/google/google_project_iam_member_enumerator.go b/pkg/remote/google/google_project_iam_member_enumerator.go index 44f40d9c7..dd631ca83 100644 --- a/pkg/remote/google/google_project_iam_member_enumerator.go +++ b/pkg/remote/google/google_project_iam_member_enumerator.go @@ -3,7 +3,7 @@ package google import ( "fmt" - remoteerror "github.com/snyk/driftctl/pkg/remote/error" + "github.com/sirupsen/logrus" "github.com/snyk/driftctl/pkg/remote/google/repository" "github.com/snyk/driftctl/pkg/resource" "github.com/snyk/driftctl/pkg/resource/google" @@ -27,12 +27,17 @@ func (e *GoogleProjectIamMemberEnumerator) SupportedType() resource.ResourceType func (e *GoogleProjectIamMemberEnumerator) Enumerate() ([]*resource.Resource, error) { results := make([]*resource.Resource, 0) + errorsByProject := make(map[string]error) + + bindingsByProject, errorsByProject := e.repository.ListProjectsBindings() - bindingsByProject, err := e.repository.ListProjectsBindings() - if err != nil { - return nil, remoteerror.NewResourceListingError(err, string(e.SupportedType())) - } for project, bindings := range bindingsByProject { + if val, ok := errorsByProject[project]; ok { + logrus.WithFields(logrus.Fields{ + "project": project, + "error": val.Error(), + }).Debug("When trying to get project IAM members") + } for roleName, members := range bindings { for _, member := range members { id := fmt.Sprintf("%s/%s/%s", project, roleName, member) @@ -53,5 +58,5 @@ func (e *GoogleProjectIamMemberEnumerator) Enumerate() ([]*resource.Resource, er } } - return results, err + return results, nil } diff --git a/pkg/remote/google/init.go b/pkg/remote/google/init.go index d668154bf..98bca943d 100644 --- a/pkg/remote/google/init.go +++ b/pkg/remote/google/init.go @@ -16,7 +16,7 @@ import ( "google.golang.org/api/cloudresourcemanager/v1" ) -func Init(version string, alerter *alerter.Alerter, +func Init(version string, gcpScope []string, alerter *alerter.Alerter, providerLibrary *terraform.ProviderLibrary, remoteLibrary *common.RemoteLibrary, progress output.Progress, @@ -51,9 +51,9 @@ func Init(version string, alerter *alerter.Alerter, return err } - assetRepository := repository.NewAssetRepository(assetClient, provider.GetConfig(), repositoryCache) + assetRepository := repository.NewAssetRepository(assetClient, provider.SetConfig(gcpScope), repositoryCache) storageRepository := repository.NewStorageRepository(storageClient, repositoryCache) - iamRepository := repository.NewCloudResourceManagerRepository(crmService, provider.GetConfig(), repositoryCache) + iamRepository := repository.NewCloudResourceManagerRepository(crmService, provider.SetConfig(gcpScope), repositoryCache) providerLibrary.AddProvider(terraform.GOOGLE, provider) deserializer := resource.NewDeserializer(factory) diff --git a/pkg/remote/google/provider.go b/pkg/remote/google/provider.go index c24190a13..380e8019b 100644 --- a/pkg/remote/google/provider.go +++ b/pkg/remote/google/provider.go @@ -1,8 +1,6 @@ package google import ( - "os" - "github.com/snyk/driftctl/pkg/output" "github.com/snyk/driftctl/pkg/remote/google/config" "github.com/snyk/driftctl/pkg/remote/terraform" @@ -34,7 +32,7 @@ func NewGCPTerraformProvider(version string, progress output.Progress, configDir tfProvider, err := terraform.NewTerraformProvider(installer, terraform.TerraformProviderConfig{ Name: p.name, GetProviderConfig: func(alias string) interface{} { - return p.GetConfig() + return p.SetConfig(nil) }, }, progress) @@ -55,20 +53,8 @@ func (p *GCPTerraformProvider) Version() string { return p.version } -func (p *GCPTerraformProvider) GetConfig() config.GCPTerraformConfig { - return config.GCPTerraformConfig{ - Organization: os.Getenv("CLOUDSDK_ORGANIZATION"), - Project: os.Getenv("CLOUDSDK_CORE_PROJECT"), - Region: os.Getenv("CLOUDSDK_COMPUTE_REGION"), - Zone: os.Getenv("CLOUDSDK_COMPUTE_ZONE"), - } -} - -func (p *GCPTerraformProvider) SetConfig(organization string, project string, region string, zone string) config.GCPTerraformConfig { +func (p *GCPTerraformProvider) SetConfig(scope []string) config.GCPTerraformConfig { return config.GCPTerraformConfig{ - Organization: organization, - Project: project, - Region: region, - Zone: zone, + Scope: scope, } } diff --git a/pkg/remote/google/repository/asset.go b/pkg/remote/google/repository/asset.go index c7abd4a1a..45fa093fd 100644 --- a/pkg/remote/google/repository/asset.go +++ b/pkg/remote/google/repository/asset.go @@ -2,7 +2,7 @@ package repository import ( "context" - "fmt" + "strings" asset "cloud.google.com/go/asset/apiv1" "github.com/snyk/driftctl/pkg/remote/cache" @@ -77,121 +77,89 @@ func NewAssetRepository(client *asset.Client, config config.GCPTerraformConfig, } func (s assetRepository) listAllResources(ty string) ([]*assetpb.Asset, error) { - req := &assetpb.ListAssetsRequest{ - Parent: fmt.Sprintf("projects/%s", s.config.Project), - ContentType: assetpb.ContentType_RESOURCE, - AssetTypes: []string{ - cloudFunctionsFunction, - bigtableInstanceAssetType, - bigtableTableAssetType, - sqlDatabaseInstanceAssetType, - computeGlobalAddressAssetType, - nodeGroupAssetType, - }, - } - var results []*assetpb.Asset - cacheKey := "listAllResources" - cachedResults := s.cache.GetAndLock(cacheKey) - defer s.cache.Unlock(cacheKey) - if cachedResults != nil { - results = cachedResults.([]*assetpb.Asset) - } + filteredResults := []*assetpb.Asset{} - if results == nil { - it := s.client.ListAssets(context.Background(), req) - for { - resource, err := it.Next() - if err == iterator.Done { - break - } - if err != nil { - return nil, err - } - results = append(results, resource) - } - s.cache.Put(cacheKey, results) - } + for _, scope := range s.config.Scope { + cacheKey := "listAllResources" + scope + cachedResults := s.cache.GetAndLock(cacheKey) + defer s.cache.Unlock(cacheKey) - filteredResults := []*assetpb.Asset{} - for _, result := range results { - if result.AssetType == ty { - filteredResults = append(filteredResults, result) + req := &assetpb.ListAssetsRequest{ + Parent: scope, + ContentType: assetpb.ContentType_RESOURCE, + AssetTypes: []string{ + cloudFunctionsFunction, + bigtableInstanceAssetType, + bigtableTableAssetType, + sqlDatabaseInstanceAssetType, + computeGlobalAddressAssetType, + nodeGroupAssetType, + }, } - } - return filteredResults, nil -} - -func (s assetRepository) searchAllResources(ty string) ([]*assetpb.ResourceSearchResult, error) { - req := &assetpb.SearchAllResourcesRequest{ - Scope: fmt.Sprintf("projects/%s", s.config.Project), - AssetTypes: []string{ - storageBucketAssetType, - computeFirewallAssetType, - computeRouterAssetType, - computeInstanceAssetType, - computeNetworkAssetType, - computeSubnetworkAssetType, - dnsManagedZoneAssetType, - computeInstanceGroupAssetType, - bigqueryDatasetAssetType, - bigqueryTableAssetType, - computeAddressAssetType, - computeDiskAssetType, - computeImageAssetType, - healthCheckAssetType, - cloudRunServiceAssetType, - resourcemanagerFolderAssetType, - }, - } - var results []*assetpb.ResourceSearchResult + var results []*assetpb.Asset - cacheKey := "SearchAllResources" - cachedResults := s.cache.GetAndLock(cacheKey) - defer s.cache.Unlock(cacheKey) - if cachedResults != nil { - results = cachedResults.([]*assetpb.ResourceSearchResult) - } + if cachedResults != nil { + results = cachedResults.([]*assetpb.Asset) + } - if results == nil { - it := s.client.SearchAllResources(context.Background(), req) - for { - resource, err := it.Next() - if err == iterator.Done { - break - } - if err != nil { - return nil, err + if results == nil { + it := s.client.ListAssets(context.Background(), req) + for { + resource, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return nil, err + } + results = append(results, resource) } - results = append(results, resource) + s.cache.Put(cacheKey, results) } - s.cache.Put(cacheKey, results) - } - filteredResults := []*assetpb.ResourceSearchResult{} - for _, result := range results { - if result.AssetType == ty { - filteredResults = append(filteredResults, result) + for _, result := range results { + if result.AssetType == ty { + filteredResults = append(filteredResults, result) + } } } - return filteredResults, nil } -func (s assetRepository) searchAllOrgResources(ty string) ([]*assetpb.ResourceSearchResult, error) { - if len(s.config.Organization) > 0 { +func (s assetRepository) searchAllResources(ty string) ([]*assetpb.ResourceSearchResult, error) { + + filteredResults := []*assetpb.ResourceSearchResult{} + + for _, scope := range s.config.Scope { + cacheKey := "SearchAllResources" + scope + cachedResults := s.cache.GetAndLock(cacheKey) + defer s.cache.Unlock(cacheKey) + req := &assetpb.SearchAllResourcesRequest{ - Scope: fmt.Sprintf("organizations/%s", s.config.Organization), + Scope: scope, AssetTypes: []string{ + storageBucketAssetType, + computeFirewallAssetType, + computeRouterAssetType, + computeInstanceAssetType, + computeNetworkAssetType, + computeSubnetworkAssetType, + dnsManagedZoneAssetType, + computeInstanceGroupAssetType, + bigqueryDatasetAssetType, + bigqueryTableAssetType, + computeAddressAssetType, + computeDiskAssetType, + computeImageAssetType, + healthCheckAssetType, + cloudRunServiceAssetType, resourcemanagerFolderAssetType, }, } var results []*assetpb.ResourceSearchResult - cacheKey := "SearchAllOrgResources" - cachedResults := s.cache.GetAndLock(cacheKey) - defer s.cache.Unlock(cacheKey) if cachedResults != nil { results = cachedResults.([]*assetpb.ResourceSearchResult) } @@ -211,16 +179,62 @@ func (s assetRepository) searchAllOrgResources(ty string) ([]*assetpb.ResourceSe s.cache.Put(cacheKey, results) } - filteredResults := []*assetpb.ResourceSearchResult{} for _, result := range results { if result.AssetType == ty { filteredResults = append(filteredResults, result) } } + } - return filteredResults, nil + return filteredResults, nil +} + +func (s assetRepository) searchAllOrgResources(ty string) ([]*assetpb.ResourceSearchResult, error) { + + filteredResults := []*assetpb.ResourceSearchResult{} + + for _, scope := range s.config.Scope { + if strings.Contains(scope, "organizations/") { + cacheKey := "SearchAllOrgResources" + scope + cachedResults := s.cache.GetAndLock(cacheKey) + defer s.cache.Unlock(cacheKey) + + req := &assetpb.SearchAllResourcesRequest{ + Scope: scope, + AssetTypes: []string{ + resourcemanagerFolderAssetType, + }, + } + var results []*assetpb.ResourceSearchResult + + if cachedResults != nil { + results = cachedResults.([]*assetpb.ResourceSearchResult) + } + + if results == nil { + it := s.client.SearchAllResources(context.Background(), req) + for { + resource, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return nil, err + } + results = append(results, resource) + } + s.cache.Put(cacheKey, results) + } + + for _, result := range results { + if result.AssetType == ty { + filteredResults = append(filteredResults, result) + } + } + } } - return nil, nil + + return filteredResults, nil } func (s assetRepository) SearchAllBuckets() ([]*assetpb.ResourceSearchResult, error) { diff --git a/pkg/remote/google/repository/cloudresourcemanager.go b/pkg/remote/google/repository/cloudresourcemanager.go index 1c7f4e3bb..be58adb8b 100644 --- a/pkg/remote/google/repository/cloudresourcemanager.go +++ b/pkg/remote/google/repository/cloudresourcemanager.go @@ -1,13 +1,15 @@ package repository import ( + "strings" + "github.com/snyk/driftctl/pkg/remote/cache" "github.com/snyk/driftctl/pkg/remote/google/config" "google.golang.org/api/cloudresourcemanager/v1" ) type CloudResourceManagerRepository interface { - ListProjectsBindings() (map[string]map[string][]string, error) + ListProjectsBindings() (map[string]map[string][]string, map[string]error) } type cloudResourceManagerRepository struct { @@ -24,27 +26,33 @@ func NewCloudResourceManagerRepository(service *cloudresourcemanager.Service, co } } -func (s *cloudResourceManagerRepository) ListProjectsBindings() (map[string]map[string][]string, error) { - if cachedResults := s.cache.Get("ListProjectsBindings"); cachedResults != nil { - return cachedResults.(map[string]map[string][]string), nil - } - - request := new(cloudresourcemanager.GetIamPolicyRequest) - policy, err := s.service.Projects.GetIamPolicy(s.config.Project, request).Do() - if err != nil { - return nil, err - } - - bindings := make(map[string][]string) - - for _, binding := range policy.Bindings { - bindings[binding.Role] = binding.Members - } +func (s *cloudResourceManagerRepository) ListProjectsBindings() (map[string]map[string][]string, map[string]error) { bindingsByProject := make(map[string]map[string][]string) - bindingsByProject[s.config.Project] = bindings - - s.cache.Put("ListProjectsBindings", bindingsByProject) + errorsByProject := make(map[string]error) + + for _, scope := range s.config.Scope { + if strings.Contains(scope, "projects/") { + project := strings.Split(scope, "projects/")[1] + request := new(cloudresourcemanager.GetIamPolicyRequest) + policy, err := s.service.Projects.GetIamPolicy(project, request).Do() + if err != nil { + errorsByProject[project] = err + bindingsByProject[project] = nil + continue + } + + bindings := make(map[string][]string) + for _, binding := range policy.Bindings { + bindings[binding.Role] = binding.Members + } + + bindingsByProject[project] = bindings + + s.cache.Put("ListProjectsBindings", bindingsByProject) + + } + } - return bindingsByProject, nil + return bindingsByProject, errorsByProject } diff --git a/pkg/remote/remote.go b/pkg/remote/remote.go index 30f4b1122..ac967a248 100644 --- a/pkg/remote/remote.go +++ b/pkg/remote/remote.go @@ -29,7 +29,7 @@ func IsSupported(remote string) bool { return false } -func Activate(remote, version string, alerter *alerter.Alerter, +func Activate(remote, version string, gcpScope []string, alerter *alerter.Alerter, providerLibrary *terraform.ProviderLibrary, remoteLibrary *common.RemoteLibrary, progress output.Progress, @@ -42,7 +42,7 @@ func Activate(remote, version string, alerter *alerter.Alerter, case common.RemoteGithubTerraform: return github.Init(version, alerter, providerLibrary, remoteLibrary, progress, resourceSchemaRepository, factory, configDir) case common.RemoteGoogleTerraform: - return google.Init(version, alerter, providerLibrary, remoteLibrary, progress, resourceSchemaRepository, factory, configDir) + return google.Init(version, gcpScope, alerter, providerLibrary, remoteLibrary, progress, resourceSchemaRepository, factory, configDir) case common.RemoteAzureTerraform: return azurerm.Init(version, alerter, providerLibrary, remoteLibrary, progress, resourceSchemaRepository, factory, configDir) From 78cc7a937d60194c9698f9dd62f6f7bdea742024 Mon Sep 17 00:00:00 2001 From: bgdanix Date: Tue, 21 Dec 2021 12:04:12 +0000 Subject: [PATCH 4/9] test: fix tests to cope with new cmd param gcp-scope --- pkg/cmd/scan_test.go | 4 +- pkg/remote/google/google_folder_enumerator.go | 56 ------- .../google_project_iam_member_enumerator.go | 17 +- pkg/remote/google/init.go | 1 - pkg/remote/google/repository/asset.go | 98 +++-------- pkg/remote/google/repository/asset_test.go | 4 +- .../google/repository/cloudresourcemanager.go | 15 +- .../google/repository/mock_AssetRepository.go | 23 --- .../mock_CloudResourceManagerRepository.go | 2 +- pkg/remote/google_bigquery_scanner_test.go | 4 +- pkg/remote/google_bigtable_scanner_test.go | 4 +- .../google_cloudfunctions_scanner_test.go | 2 +- pkg/remote/google_cloudrun_scanner_test.go | 2 +- pkg/remote/google_compute_scanner_test.go | 24 +-- pkg/remote/google_folder_scanner_test.go | 154 ------------------ pkg/remote/google_network_scanner_test.go | 2 +- ....go => google_project_iam_scanner_test.go} | 0 pkg/remote/google_sql_scanner_test.go | 2 +- pkg/remote/google_storage_scanner_test.go | 2 +- pkg/resource/google/google_folder.go | 3 - pkg/resource/resource_types.go | 1 - 21 files changed, 65 insertions(+), 355 deletions(-) delete mode 100644 pkg/remote/google/google_folder_enumerator.go delete mode 100644 pkg/remote/google_folder_scanner_test.go rename pkg/remote/{google_project_scanner_test.go => google_project_iam_scanner_test.go} (100%) delete mode 100644 pkg/resource/google/google_folder.go diff --git a/pkg/cmd/scan_test.go b/pkg/cmd/scan_test.go index f562d73ca..c13062fc8 100644 --- a/pkg/cmd/scan_test.go +++ b/pkg/cmd/scan_test.go @@ -356,14 +356,14 @@ func Test_Options(t *testing.T) { }, { name: "should not find provider version in lockfile", - args: []string{"scan", "--to", "gcp+tf", "--tf-lockfile", "testdata/terraform_valid.lock.hcl"}, + args: []string{"scan", "--to", "gcp+tf", "--gcp-scope", "organizations/123", "--tf-lockfile", "testdata/terraform_valid.lock.hcl"}, assertOptions: func(t *testing.T, opts *pkg.ScanOptions) { assert.Equal(t, "", opts.ProviderVersion) }, }, { name: "should fail to read lockfile with silent error", - args: []string{"scan", "--to", "gcp+tf", "--tf-lockfile", "testdata/terraform_invalid.lock.hcl"}, + args: []string{"scan", "--to", "gcp+tf", "--gcp-scope", "organizations/123", "--tf-lockfile", "testdata/terraform_invalid.lock.hcl"}, assertOptions: func(t *testing.T, opts *pkg.ScanOptions) { assert.Equal(t, "", opts.ProviderVersion) }, diff --git a/pkg/remote/google/google_folder_enumerator.go b/pkg/remote/google/google_folder_enumerator.go deleted file mode 100644 index c267b891b..000000000 --- a/pkg/remote/google/google_folder_enumerator.go +++ /dev/null @@ -1,56 +0,0 @@ -package google - -import ( - "strings" - - "github.com/sirupsen/logrus" - remoteerror "github.com/snyk/driftctl/pkg/remote/error" - "github.com/snyk/driftctl/pkg/remote/google/repository" - "github.com/snyk/driftctl/pkg/resource" - "github.com/snyk/driftctl/pkg/resource/google" -) - -type GoogleFolderEnumerator struct { - repository repository.AssetRepository - factory resource.ResourceFactory -} - -func NewGoogleFolderEnumerator(repo repository.AssetRepository, factory resource.ResourceFactory) *GoogleFolderEnumerator { - return &GoogleFolderEnumerator{ - repository: repo, - factory: factory, - } -} - -func (e *GoogleFolderEnumerator) SupportedType() resource.ResourceType { - return google.GoogleFolderResourceType -} - -func (e *GoogleFolderEnumerator) Enumerate() ([]*resource.Resource, error) { - resources, err := e.repository.SearchAllFolders() - - if err != nil { - return nil, remoteerror.NewResourceListingError(err, string(e.SupportedType())) - } - - results := make([]*resource.Resource, 0, len(resources)) - - for _, res := range resources { - id := trimResourceName(res.Name) - splittedId := strings.Split(id, "folders/") - if len(splittedId) != 2 { - logrus.WithField("id", res.Name).Warn("Cannot parse google_folder ID") - continue - } - results = append( - results, - e.factory.CreateAbstractResource( - string(e.SupportedType()), - id, - map[string]interface{}{}, - ), - ) - } - - return results, err -} diff --git a/pkg/remote/google/google_project_iam_member_enumerator.go b/pkg/remote/google/google_project_iam_member_enumerator.go index dd631ca83..44f40d9c7 100644 --- a/pkg/remote/google/google_project_iam_member_enumerator.go +++ b/pkg/remote/google/google_project_iam_member_enumerator.go @@ -3,7 +3,7 @@ package google import ( "fmt" - "github.com/sirupsen/logrus" + remoteerror "github.com/snyk/driftctl/pkg/remote/error" "github.com/snyk/driftctl/pkg/remote/google/repository" "github.com/snyk/driftctl/pkg/resource" "github.com/snyk/driftctl/pkg/resource/google" @@ -27,17 +27,12 @@ func (e *GoogleProjectIamMemberEnumerator) SupportedType() resource.ResourceType func (e *GoogleProjectIamMemberEnumerator) Enumerate() ([]*resource.Resource, error) { results := make([]*resource.Resource, 0) - errorsByProject := make(map[string]error) - - bindingsByProject, errorsByProject := e.repository.ListProjectsBindings() + bindingsByProject, err := e.repository.ListProjectsBindings() + if err != nil { + return nil, remoteerror.NewResourceListingError(err, string(e.SupportedType())) + } for project, bindings := range bindingsByProject { - if val, ok := errorsByProject[project]; ok { - logrus.WithFields(logrus.Fields{ - "project": project, - "error": val.Error(), - }).Debug("When trying to get project IAM members") - } for roleName, members := range bindings { for _, member := range members { id := fmt.Sprintf("%s/%s/%s", project, roleName, member) @@ -58,5 +53,5 @@ func (e *GoogleProjectIamMemberEnumerator) Enumerate() ([]*resource.Resource, er } } - return results, nil + return results, err } diff --git a/pkg/remote/google/init.go b/pkg/remote/google/init.go index 98bca943d..64f727a26 100644 --- a/pkg/remote/google/init.go +++ b/pkg/remote/google/init.go @@ -99,7 +99,6 @@ func Init(version string, gcpScope []string, alerter *alerter.Alerter, remoteLibrary.AddEnumerator(NewGoogleComputeHealthCheckEnumerator(assetRepository, factory)) remoteLibrary.AddEnumerator(NewGoogleCloudRunServiceEnumerator(assetRepository, factory)) remoteLibrary.AddEnumerator(NewGoogleComputeNodeGroupEnumerator(assetRepository, factory)) - remoteLibrary.AddEnumerator(NewGoogleFolderEnumerator(assetRepository, factory)) err = resourceSchemaRepository.Init(terraform.GOOGLE, provider.Version(), provider.Schema()) if err != nil { diff --git a/pkg/remote/google/repository/asset.go b/pkg/remote/google/repository/asset.go index 45fa093fd..9290bda79 100644 --- a/pkg/remote/google/repository/asset.go +++ b/pkg/remote/google/repository/asset.go @@ -2,7 +2,6 @@ package repository import ( "context" - "strings" asset "cloud.google.com/go/asset/apiv1" "github.com/snyk/driftctl/pkg/remote/cache" @@ -13,28 +12,27 @@ import ( // https://cloud.google.com/asset-inventory/docs/supported-asset-types#supported_resource_types const ( - storageBucketAssetType = "storage.googleapis.com/Bucket" - computeFirewallAssetType = "compute.googleapis.com/Firewall" - computeRouterAssetType = "compute.googleapis.com/Router" - computeInstanceAssetType = "compute.googleapis.com/Instance" - computeNetworkAssetType = "compute.googleapis.com/Network" - computeSubnetworkAssetType = "compute.googleapis.com/Subnetwork" - computeDiskAssetType = "compute.googleapis.com/Disk" - computeImageAssetType = "compute.googleapis.com/Image" - dnsManagedZoneAssetType = "dns.googleapis.com/ManagedZone" - computeInstanceGroupAssetType = "compute.googleapis.com/InstanceGroup" - bigqueryDatasetAssetType = "bigquery.googleapis.com/Dataset" - bigqueryTableAssetType = "bigquery.googleapis.com/Table" - computeAddressAssetType = "compute.googleapis.com/Address" - computeGlobalAddressAssetType = "compute.googleapis.com/GlobalAddress" - cloudFunctionsFunction = "cloudfunctions.googleapis.com/CloudFunction" - bigtableInstanceAssetType = "bigtableadmin.googleapis.com/Instance" - bigtableTableAssetType = "bigtableadmin.googleapis.com/Table" - sqlDatabaseInstanceAssetType = "sqladmin.googleapis.com/Instance" - healthCheckAssetType = "compute.googleapis.com/HealthCheck" - cloudRunServiceAssetType = "run.googleapis.com/Service" - nodeGroupAssetType = "compute.googleapis.com/NodeGroup" - resourcemanagerFolderAssetType = "cloudresourcemanager.googleapis.com/Folder" + storageBucketAssetType = "storage.googleapis.com/Bucket" + computeFirewallAssetType = "compute.googleapis.com/Firewall" + computeRouterAssetType = "compute.googleapis.com/Router" + computeInstanceAssetType = "compute.googleapis.com/Instance" + computeNetworkAssetType = "compute.googleapis.com/Network" + computeSubnetworkAssetType = "compute.googleapis.com/Subnetwork" + computeDiskAssetType = "compute.googleapis.com/Disk" + computeImageAssetType = "compute.googleapis.com/Image" + dnsManagedZoneAssetType = "dns.googleapis.com/ManagedZone" + computeInstanceGroupAssetType = "compute.googleapis.com/InstanceGroup" + bigqueryDatasetAssetType = "bigquery.googleapis.com/Dataset" + bigqueryTableAssetType = "bigquery.googleapis.com/Table" + computeAddressAssetType = "compute.googleapis.com/Address" + computeGlobalAddressAssetType = "compute.googleapis.com/GlobalAddress" + cloudFunctionsFunction = "cloudfunctions.googleapis.com/CloudFunction" + bigtableInstanceAssetType = "bigtableadmin.googleapis.com/Instance" + bigtableTableAssetType = "bigtableadmin.googleapis.com/Table" + sqlDatabaseInstanceAssetType = "sqladmin.googleapis.com/Instance" + healthCheckAssetType = "compute.googleapis.com/HealthCheck" + cloudRunServiceAssetType = "run.googleapis.com/Service" + nodeGroupAssetType = "compute.googleapis.com/NodeGroup" ) type AssetRepository interface { @@ -59,7 +57,6 @@ type AssetRepository interface { SearchAllHealthChecks() ([]*assetpb.ResourceSearchResult, error) SearchAllCloudRunServices() ([]*assetpb.ResourceSearchResult, error) SearchAllNodeGroups() ([]*assetpb.Asset, error) - SearchAllFolders() ([]*assetpb.ResourceSearchResult, error) } type assetRepository struct { @@ -155,7 +152,6 @@ func (s assetRepository) searchAllResources(ty string) ([]*assetpb.ResourceSearc computeImageAssetType, healthCheckAssetType, cloudRunServiceAssetType, - resourcemanagerFolderAssetType, }, } var results []*assetpb.ResourceSearchResult @@ -189,54 +185,6 @@ func (s assetRepository) searchAllResources(ty string) ([]*assetpb.ResourceSearc return filteredResults, nil } -func (s assetRepository) searchAllOrgResources(ty string) ([]*assetpb.ResourceSearchResult, error) { - - filteredResults := []*assetpb.ResourceSearchResult{} - - for _, scope := range s.config.Scope { - if strings.Contains(scope, "organizations/") { - cacheKey := "SearchAllOrgResources" + scope - cachedResults := s.cache.GetAndLock(cacheKey) - defer s.cache.Unlock(cacheKey) - - req := &assetpb.SearchAllResourcesRequest{ - Scope: scope, - AssetTypes: []string{ - resourcemanagerFolderAssetType, - }, - } - var results []*assetpb.ResourceSearchResult - - if cachedResults != nil { - results = cachedResults.([]*assetpb.ResourceSearchResult) - } - - if results == nil { - it := s.client.SearchAllResources(context.Background(), req) - for { - resource, err := it.Next() - if err == iterator.Done { - break - } - if err != nil { - return nil, err - } - results = append(results, resource) - } - s.cache.Put(cacheKey, results) - } - - for _, result := range results { - if result.AssetType == ty { - filteredResults = append(filteredResults, result) - } - } - } - } - - return filteredResults, nil -} - func (s assetRepository) SearchAllBuckets() ([]*assetpb.ResourceSearchResult, error) { return s.searchAllResources(storageBucketAssetType) } @@ -320,7 +268,3 @@ func (s assetRepository) SearchAllCloudRunServices() ([]*assetpb.ResourceSearchR func (s assetRepository) SearchAllNodeGroups() ([]*assetpb.Asset, error) { return s.listAllResources(nodeGroupAssetType) } - -func (s assetRepository) SearchAllFolders() ([]*assetpb.ResourceSearchResult, error) { - return s.searchAllOrgResources(resourcemanagerFolderAssetType) -} diff --git a/pkg/remote/google/repository/asset_test.go b/pkg/remote/google/repository/asset_test.go index 4db980a60..b51943084 100644 --- a/pkg/remote/google/repository/asset_test.go +++ b/pkg/remote/google/repository/asset_test.go @@ -27,7 +27,7 @@ func Test_assetRepository_searchAllResources_CacheHit(t *testing.T) { c := &cache.MockCache{} c.On("GetAndLock", "SearchAllResources").Return(expectedResults).Times(1) c.On("Unlock", "SearchAllResources").Times(1) - repo := NewAssetRepository(nil, config.GCPTerraformConfig{Project: ""}, c) + repo := NewAssetRepository(nil, config.GCPTerraformConfig{Scope: []string{""}}, c) got, err := repo.searchAllResources("google_fake_type") c.AssertExpectations(t) @@ -55,7 +55,7 @@ func Test_assetRepository_searchAllResources_CacheMiss(t *testing.T) { c.On("GetAndLock", "SearchAllResources").Return(nil).Times(1) c.On("Unlock", "SearchAllResources").Times(1) c.On("Put", "SearchAllResources", mock.IsType([]*assetpb.ResourceSearchResult{})).Return(false).Times(1) - repo := NewAssetRepository(assetClient, config.GCPTerraformConfig{Project: ""}, c) + repo := NewAssetRepository(assetClient, config.GCPTerraformConfig{Scope: []string{""}}, c) got, err := repo.searchAllResources("google_fake_type") c.AssertExpectations(t) diff --git a/pkg/remote/google/repository/cloudresourcemanager.go b/pkg/remote/google/repository/cloudresourcemanager.go index be58adb8b..48ab7349f 100644 --- a/pkg/remote/google/repository/cloudresourcemanager.go +++ b/pkg/remote/google/repository/cloudresourcemanager.go @@ -1,6 +1,7 @@ package repository import ( + "errors" "strings" "github.com/snyk/driftctl/pkg/remote/cache" @@ -9,7 +10,7 @@ import ( ) type CloudResourceManagerRepository interface { - ListProjectsBindings() (map[string]map[string][]string, map[string]error) + ListProjectsBindings() (map[string]map[string][]string, error) } type cloudResourceManagerRepository struct { @@ -26,10 +27,11 @@ func NewCloudResourceManagerRepository(service *cloudresourcemanager.Service, co } } -func (s *cloudResourceManagerRepository) ListProjectsBindings() (map[string]map[string][]string, map[string]error) { +func (s *cloudResourceManagerRepository) ListProjectsBindings() (map[string]map[string][]string, error) { bindingsByProject := make(map[string]map[string][]string) errorsByProject := make(map[string]error) + var erorsString string for _, scope := range s.config.Scope { if strings.Contains(scope, "projects/") { @@ -54,5 +56,12 @@ func (s *cloudResourceManagerRepository) ListProjectsBindings() (map[string]map[ } } - return bindingsByProject, errorsByProject + if len(errorsByProject) > 0 { + for project, errval := range errorsByProject { + erorsString = erorsString + "Project: " + project + " had the following error: " + errval.Error() + "; " + } + return bindingsByProject, errors.New(erorsString) + } else { + return bindingsByProject, nil + } } diff --git a/pkg/remote/google/repository/mock_AssetRepository.go b/pkg/remote/google/repository/mock_AssetRepository.go index 3202ba8d1..3561a8e10 100644 --- a/pkg/remote/google/repository/mock_AssetRepository.go +++ b/pkg/remote/google/repository/mock_AssetRepository.go @@ -219,29 +219,6 @@ func (_m *MockAssetRepository) SearchAllFirewalls() ([]*asset.ResourceSearchResu return r0, r1 } -// SearchAllFolders provides a mock function with given fields: -func (_m *MockAssetRepository) SearchAllFolders() ([]*asset.ResourceSearchResult, error) { - ret := _m.Called() - - var r0 []*asset.ResourceSearchResult - if rf, ok := ret.Get(0).(func() []*asset.ResourceSearchResult); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*asset.ResourceSearchResult) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - // SearchAllFunctions provides a mock function with given fields: func (_m *MockAssetRepository) SearchAllFunctions() ([]*asset.Asset, error) { ret := _m.Called() diff --git a/pkg/remote/google/repository/mock_CloudResourceManagerRepository.go b/pkg/remote/google/repository/mock_CloudResourceManagerRepository.go index 4b2a7658f..63f40c09e 100644 --- a/pkg/remote/google/repository/mock_CloudResourceManagerRepository.go +++ b/pkg/remote/google/repository/mock_CloudResourceManagerRepository.go @@ -1,4 +1,4 @@ -// Code generated by mockery v0.0.0-dev. DO NOT EDIT. +// Code generated by mockery v2.9.4. DO NOT EDIT. package repository diff --git a/pkg/remote/google_bigquery_scanner_test.go b/pkg/remote/google_bigquery_scanner_test.go index c37e42874..8c58726dc 100644 --- a/pkg/remote/google_bigquery_scanner_test.go +++ b/pkg/remote/google_bigquery_scanner_test.go @@ -105,7 +105,7 @@ func TestGoogleBigqueryDataset(t *testing.T) { tt.Fatal(err) } - repo := repository.NewAssetRepository(assetClient, realProvider.GetConfig(), cache.New(0)) + repo := repository.NewAssetRepository(assetClient, realProvider.SetConfig([]string{"projects/123456"}), cache.New(0)) remoteLibrary.AddEnumerator(google.NewGoogleBigqueryDatasetEnumerator(repo, factory)) @@ -208,7 +208,7 @@ func TestGoogleBigqueryTable(t *testing.T) { tt.Fatal(err) } - repo := repository.NewAssetRepository(assetClient, realProvider.GetConfig(), cache.New(0)) + repo := repository.NewAssetRepository(assetClient, realProvider.SetConfig([]string{"projects/123456"}), cache.New(0)) remoteLibrary.AddEnumerator(google.NewGoogleBigqueryTableEnumerator(repo, factory)) diff --git a/pkg/remote/google_bigtable_scanner_test.go b/pkg/remote/google_bigtable_scanner_test.go index f75798cda..dc037401b 100644 --- a/pkg/remote/google_bigtable_scanner_test.go +++ b/pkg/remote/google_bigtable_scanner_test.go @@ -142,7 +142,7 @@ func TestGoogleBigtableInstance(t *testing.T) { tt.Fatal(err) } - repo := repository.NewAssetRepository(assetClient, realProvider.GetConfig(), cache.New(0)) + repo := repository.NewAssetRepository(assetClient, realProvider.SetConfig([]string{"projects/123456"}), cache.New(0)) remoteLibrary.AddEnumerator(google.NewGoogleBigTableInstanceEnumerator(repo, factory)) @@ -268,7 +268,7 @@ func TestGoogleBigtableTable(t *testing.T) { tt.Fatal(err) } - repo := repository.NewAssetRepository(assetClient, realProvider.GetConfig(), cache.New(0)) + repo := repository.NewAssetRepository(assetClient, realProvider.SetConfig([]string{"projects/123456"}), cache.New(0)) remoteLibrary.AddEnumerator(google.NewGoogleBigtableTableEnumerator(repo, factory)) diff --git a/pkg/remote/google_cloudfunctions_scanner_test.go b/pkg/remote/google_cloudfunctions_scanner_test.go index bec5fb240..6227c5d4a 100644 --- a/pkg/remote/google_cloudfunctions_scanner_test.go +++ b/pkg/remote/google_cloudfunctions_scanner_test.go @@ -129,7 +129,7 @@ func TestGoogleCloudFunctionsFunction(t *testing.T) { tt.Fatal(err) } - repo := repository.NewAssetRepository(assetClient, realProvider.GetConfig(), cache.New(0)) + repo := repository.NewAssetRepository(assetClient, realProvider.SetConfig([]string{"projects/123456"}), cache.New(0)) remoteLibrary.AddEnumerator(google.NewGoogleCloudFunctionsFunctionEnumerator(repo, factory)) diff --git a/pkg/remote/google_cloudrun_scanner_test.go b/pkg/remote/google_cloudrun_scanner_test.go index 032752ec2..ddc57c189 100644 --- a/pkg/remote/google_cloudrun_scanner_test.go +++ b/pkg/remote/google_cloudrun_scanner_test.go @@ -124,7 +124,7 @@ func TestGoogleCloudRunService(t *testing.T) { tt.Fatal(err) } - repo := repository.NewAssetRepository(assetClient, realProvider.GetConfig(), cache.New(0)) + repo := repository.NewAssetRepository(assetClient, realProvider.SetConfig([]string{"projects/123456"}), cache.New(0)) remoteLibrary.AddEnumerator(google.NewGoogleCloudRunServiceEnumerator(repo, factory)) diff --git a/pkg/remote/google_compute_scanner_test.go b/pkg/remote/google_compute_scanner_test.go index c0b35bbe1..c53440a24 100644 --- a/pkg/remote/google_compute_scanner_test.go +++ b/pkg/remote/google_compute_scanner_test.go @@ -129,7 +129,7 @@ func TestGoogleComputeFirewall(t *testing.T) { provider.ShouldUpdate() } - repo := repository.NewAssetRepository(assetClient, realProvider.GetConfig(), cache.New(0)) + repo := repository.NewAssetRepository(assetClient, realProvider.SetConfig([]string{"projects/123456"}), cache.New(0)) remoteLibrary.AddEnumerator(google.NewGoogleComputeFirewallEnumerator(repo, factory)) remoteLibrary.AddDetailsFetcher(resType, common.NewGenericDetailsFetcher(resType, provider, deserializer)) @@ -252,7 +252,7 @@ func TestGoogleComputeRouter(t *testing.T) { tt.Fatal(err) } - repo := repository.NewAssetRepository(assetClient, realProvider.GetConfig(), cache.New(0)) + repo := repository.NewAssetRepository(assetClient, realProvider.SetConfig([]string{"projects/123456"}), cache.New(0)) remoteLibrary.AddEnumerator(google.NewGoogleComputeRouterEnumerator(repo, factory)) @@ -356,7 +356,7 @@ func TestGoogleComputeInstance(t *testing.T) { tt.Fatal(err) } - repo := repository.NewAssetRepository(assetClient, realProvider.GetConfig(), cache.New(0)) + repo := repository.NewAssetRepository(assetClient, realProvider.SetConfig([]string{"projects/123456"}), cache.New(0)) remoteLibrary.AddEnumerator(google.NewGoogleComputeInstanceEnumerator(repo, factory)) @@ -480,7 +480,7 @@ func TestGoogleComputeNetwork(t *testing.T) { provider.ShouldUpdate() } - repo := repository.NewAssetRepository(assetClient, realProvider.GetConfig(), cache.New(0)) + repo := repository.NewAssetRepository(assetClient, realProvider.SetConfig([]string{"projects/123456"}), cache.New(0)) remoteLibrary.AddEnumerator(google.NewGoogleComputeNetworkEnumerator(repo, factory)) remoteLibrary.AddDetailsFetcher(resType, common.NewGenericDetailsFetcher(resType, provider, deserializer)) @@ -602,7 +602,7 @@ func TestGoogleComputeInstanceGroup(t *testing.T) { provider.ShouldUpdate() } - repo := repository.NewAssetRepository(assetClient, realProvider.GetConfig(), cache.New(0)) + repo := repository.NewAssetRepository(assetClient, realProvider.SetConfig([]string{"projects/123456"}), cache.New(0)) remoteLibrary.AddEnumerator(google.NewGoogleComputeInstanceGroupEnumerator(repo, factory)) remoteLibrary.AddDetailsFetcher(googleresource.GoogleComputeInstanceGroupResourceType, common.NewGenericDetailsFetcher(googleresource.GoogleComputeInstanceGroupResourceType, provider, deserializer)) @@ -723,7 +723,7 @@ func TestGoogleComputeAddress(t *testing.T) { tt.Fatal(err) } - repo := repository.NewAssetRepository(assetClient, realProvider.GetConfig(), cache.New(0)) + repo := repository.NewAssetRepository(assetClient, realProvider.SetConfig([]string{"projects/123456"}), cache.New(0)) remoteLibrary.AddEnumerator(google.NewGoogleComputeAddressEnumerator(repo, factory)) @@ -849,7 +849,7 @@ func TestGoogleComputeGlobalAddress(t *testing.T) { tt.Fatal(err) } - repo := repository.NewAssetRepository(assetClient, realProvider.GetConfig(), cache.New(0)) + repo := repository.NewAssetRepository(assetClient, realProvider.SetConfig([]string{"projects/123456"}), cache.New(0)) remoteLibrary.AddEnumerator(google.NewGoogleComputeGlobalAddressEnumerator(repo, factory)) @@ -973,7 +973,7 @@ func TestGoogleComputeSubnetwork(t *testing.T) { provider.ShouldUpdate() } - repo := repository.NewAssetRepository(assetClient, realProvider.GetConfig(), cache.New(0)) + repo := repository.NewAssetRepository(assetClient, realProvider.SetConfig([]string{"projects/123456"}), cache.New(0)) remoteLibrary.AddEnumerator(google.NewGoogleComputeSubnetworkEnumerator(repo, factory)) remoteLibrary.AddDetailsFetcher(resType, common.NewGenericDetailsFetcher(resType, provider, deserializer)) @@ -1082,7 +1082,7 @@ func TestGoogleComputeDisk(t *testing.T) { tt.Fatal(err) } - repo := repository.NewAssetRepository(assetClient, realProvider.GetConfig(), cache.New(0)) + repo := repository.NewAssetRepository(assetClient, realProvider.SetConfig([]string{"projects/123456"}), cache.New(0)) remoteLibrary.AddEnumerator(google.NewGoogleComputeDiskEnumerator(repo, factory)) @@ -1192,7 +1192,7 @@ func TestGoogleComputeImage(t *testing.T) { tt.Fatal(err) } - repo := repository.NewAssetRepository(assetClient, realProvider.GetConfig(), cache.New(0)) + repo := repository.NewAssetRepository(assetClient, realProvider.SetConfig([]string{"projects/123456"}), cache.New(0)) remoteLibrary.AddEnumerator(google.NewGoogleComputeImageEnumerator(repo, factory)) @@ -1302,7 +1302,7 @@ func TestGoogleComputeHealthCheck(t *testing.T) { tt.Fatal(err) } - repo := repository.NewAssetRepository(assetClient, realProvider.GetConfig(), cache.New(0)) + repo := repository.NewAssetRepository(assetClient, realProvider.SetConfig([]string{"projects/123456"}), cache.New(0)) remoteLibrary.AddEnumerator(google.NewGoogleComputeHealthCheckEnumerator(repo, factory)) @@ -1412,7 +1412,7 @@ func TestGoogleComputeNodeGroup(t *testing.T) { tt.Fatal(err) } - repo := repository.NewAssetRepository(assetClient, realProvider.GetConfig(), cache.New(0)) + repo := repository.NewAssetRepository(assetClient, realProvider.SetConfig([]string{"projects/123456"}), cache.New(0)) remoteLibrary.AddEnumerator(google.NewGoogleComputeNodeGroupEnumerator(repo, factory)) diff --git a/pkg/remote/google_folder_scanner_test.go b/pkg/remote/google_folder_scanner_test.go deleted file mode 100644 index c6ac4ad80..000000000 --- a/pkg/remote/google_folder_scanner_test.go +++ /dev/null @@ -1,154 +0,0 @@ -package remote - -import ( - "testing" - - "github.com/snyk/driftctl/mocks" - "github.com/snyk/driftctl/pkg/filter" - "github.com/snyk/driftctl/pkg/remote/alerts" - "github.com/snyk/driftctl/pkg/remote/cache" - "github.com/snyk/driftctl/pkg/remote/common" - remoteerr "github.com/snyk/driftctl/pkg/remote/error" - "github.com/snyk/driftctl/pkg/remote/google" - "github.com/snyk/driftctl/pkg/remote/google/repository" - "github.com/snyk/driftctl/pkg/resource" - googleresource "github.com/snyk/driftctl/pkg/resource/google" - "github.com/snyk/driftctl/pkg/terraform" - testgoogle "github.com/snyk/driftctl/test/google" - testresource "github.com/snyk/driftctl/test/resource" - terraform2 "github.com/snyk/driftctl/test/terraform" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - assetpb "google.golang.org/genproto/googleapis/cloud/asset/v1" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -func TestGoogleFolder(t *testing.T) { - - cases := []struct { - test string - response []*assetpb.ResourceSearchResult - responseErr error - setupAlerterMock func(alerter *mocks.AlerterInterface) - wantErr error - assertExpected func(t *testing.T, got []*resource.Resource) - }{ - { - test: "no folder", - response: []*assetpb.ResourceSearchResult{}, - wantErr: nil, - assertExpected: func(t *testing.T, got []*resource.Resource) { - assert.Len(t, got, 0) - }, - }, - { - test: "multiple folders", - response: []*assetpb.ResourceSearchResult{ - { - AssetType: "cloudresourcemanager.googleapis.com/Folder", - Name: "invalid ID", // Should be ignored - }, - { - AssetType: "cloudresourcemanager.googleapis.com/Folder", - DisplayName: "test-folder-0", - Name: "//cloudresourcemanager.googleapis.com/folders/123456789", - }, - { - AssetType: "cloudresourcemanager.googleapis.com/Folder", - DisplayName: "test-folder-1", - Name: "//cloudresourcemanager.googleapis.com/folders/121556789", - }, - { - AssetType: "cloudresourcemanager.googleapis.com/Folder", - DisplayName: "test-folder-1", - Name: "//cloudresourcemanager.googleapis.com/folders/121556677", - }, - }, - wantErr: nil, - assertExpected: func(t *testing.T, got []*resource.Resource) { - assert.Len(t, got, 3) - - assert.Equal(t, got[0].ResourceId(), "folders/123456789") - assert.Equal(t, got[0].ResourceType(), googleresource.GoogleFolderResourceType) - - assert.Equal(t, got[1].ResourceId(), "folders/121556789") - assert.Equal(t, got[1].ResourceType(), googleresource.GoogleFolderResourceType) - - assert.Equal(t, got[2].ResourceId(), "folders/121556677") - assert.Equal(t, got[2].ResourceType(), googleresource.GoogleFolderResourceType) - }, - }, - { - test: "should return access denied error", - wantErr: nil, - responseErr: status.Error(codes.PermissionDenied, "The caller does not have permission"), - setupAlerterMock: func(alerter *mocks.AlerterInterface) { - alerter.On( - "SendAlert", - googleresource.GoogleFolderResourceType, - alerts.NewRemoteAccessDeniedAlert( - common.RemoteGoogleTerraform, - remoteerr.NewResourceListingError( - status.Error(codes.PermissionDenied, "The caller does not have permission"), - googleresource.GoogleFolderResourceType, - ), - alerts.EnumerationPhase, - ), - ).Once() - }, - assertExpected: func(t *testing.T, got []*resource.Resource) { - assert.Len(t, got, 0) - }, - }, - } - - providerVersion := "3.78.0" - schemaRepository := testresource.InitFakeSchemaRepository("google", providerVersion) - googleresource.InitResourcesMetadata(schemaRepository) - factory := terraform.NewTerraformResourceFactory(schemaRepository) - - for _, c := range cases { - t.Run(c.test, func(tt *testing.T) { - scanOptions := ScannerOptions{} - providerLibrary := terraform.NewProviderLibrary() - remoteLibrary := common.NewRemoteLibrary() - - // Initialize mocks - alerter := &mocks.AlerterInterface{} - if c.setupAlerterMock != nil { - c.setupAlerterMock(alerter) - } - - assetClient, err := testgoogle.NewFakeAssetServer(c.response, c.responseErr) - if err != nil { - tt.Fatal(err) - } - - realProvider, err := terraform2.InitTestGoogleProvider(providerLibrary, providerVersion) - if err != nil { - tt.Fatal(err) - } - - repo := repository.NewAssetRepository(assetClient, realProvider.SetConfig("123456", "", "", ""), cache.New(0)) - - remoteLibrary.AddEnumerator(google.NewGoogleFolderEnumerator(repo, factory)) - - testFilter := &filter.MockFilter{} - testFilter.On("IsTypeIgnored", mock.Anything).Return(false) - - s := NewScanner(remoteLibrary, alerter, scanOptions, testFilter) - got, err := s.Resources() - assert.Equal(tt, c.wantErr, err) - if err != nil { - return - } - - alerter.AssertExpectations(tt) - testFilter.AssertExpectations(tt) - if c.assertExpected != nil { - c.assertExpected(t, got) - } - }) - } -} diff --git a/pkg/remote/google_network_scanner_test.go b/pkg/remote/google_network_scanner_test.go index 08cf9332d..9ae0882a4 100644 --- a/pkg/remote/google_network_scanner_test.go +++ b/pkg/remote/google_network_scanner_test.go @@ -130,7 +130,7 @@ func TestGoogleDNSNanagedZone(t *testing.T) { tt.Fatal(err) } - repo := repository.NewAssetRepository(assetClient, realProvider.GetConfig(), cache.New(0)) + repo := repository.NewAssetRepository(assetClient, realProvider.SetConfig([]string{"projects/123456"}), cache.New(0)) remoteLibrary.AddEnumerator(google.NewGoogleDNSManagedZoneEnumerator(repo, factory)) diff --git a/pkg/remote/google_project_scanner_test.go b/pkg/remote/google_project_iam_scanner_test.go similarity index 100% rename from pkg/remote/google_project_scanner_test.go rename to pkg/remote/google_project_iam_scanner_test.go diff --git a/pkg/remote/google_sql_scanner_test.go b/pkg/remote/google_sql_scanner_test.go index 650374428..3f2fb233a 100644 --- a/pkg/remote/google_sql_scanner_test.go +++ b/pkg/remote/google_sql_scanner_test.go @@ -127,7 +127,7 @@ func TestGoogleSQLDatabaseInstance(t *testing.T) { tt.Fatal(err) } - repo := repository.NewAssetRepository(assetClient, realProvider.GetConfig(), cache.New(0)) + repo := repository.NewAssetRepository(assetClient, realProvider.SetConfig([]string{"projects/123456"}), cache.New(0)) remoteLibrary.AddEnumerator(google.NewGoogleSQLDatabaseInstanceEnumerator(repo, factory)) diff --git a/pkg/remote/google_storage_scanner_test.go b/pkg/remote/google_storage_scanner_test.go index cb7180164..425dd0746 100644 --- a/pkg/remote/google_storage_scanner_test.go +++ b/pkg/remote/google_storage_scanner_test.go @@ -138,7 +138,7 @@ func TestGoogleStorageBucket(t *testing.T) { provider.ShouldUpdate() } - repo := repository.NewAssetRepository(assetClient, realProvider.GetConfig(), cache.New(0)) + repo := repository.NewAssetRepository(assetClient, realProvider.SetConfig([]string{"projects/123456"}), cache.New(0)) remoteLibrary.AddEnumerator(google.NewGoogleStorageBucketEnumerator(repo, factory)) remoteLibrary.AddDetailsFetcher(resType, common.NewGenericDetailsFetcher(resType, provider, deserializer)) diff --git a/pkg/resource/google/google_folder.go b/pkg/resource/google/google_folder.go deleted file mode 100644 index 00af951b0..000000000 --- a/pkg/resource/google/google_folder.go +++ /dev/null @@ -1,3 +0,0 @@ -package google - -const GoogleFolderResourceType = "google_folder" diff --git a/pkg/resource/resource_types.go b/pkg/resource/resource_types.go index 2fa664b08..62204fef3 100644 --- a/pkg/resource/resource_types.go +++ b/pkg/resource/resource_types.go @@ -189,7 +189,6 @@ var supportedTypes = map[string]ResourceTypeMeta{ "google_compute_global_address": {}, "google_compute_node_group": {}, "google_cloud_run_service": {}, - "google_folder": {}, "azurerm_storage_account": {}, "azurerm_storage_container": {}, From 0a8f5f8d1dd486c9a99e09c481bb7412b725e9b1 Mon Sep 17 00:00:00 2001 From: bgdanix Date: Tue, 21 Dec 2021 12:08:46 +0000 Subject: [PATCH 5/9] fix: go formatted code --- pkg/remote/google/repository/asset.go | 42 +++++++++---------- .../google/repository/cloudresourcemanager.go | 4 +- pkg/remote/google_bigquery_scanner_test.go | 2 +- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/pkg/remote/google/repository/asset.go b/pkg/remote/google/repository/asset.go index 9290bda79..f7763b14e 100644 --- a/pkg/remote/google/repository/asset.go +++ b/pkg/remote/google/repository/asset.go @@ -12,27 +12,27 @@ import ( // https://cloud.google.com/asset-inventory/docs/supported-asset-types#supported_resource_types const ( - storageBucketAssetType = "storage.googleapis.com/Bucket" - computeFirewallAssetType = "compute.googleapis.com/Firewall" - computeRouterAssetType = "compute.googleapis.com/Router" - computeInstanceAssetType = "compute.googleapis.com/Instance" - computeNetworkAssetType = "compute.googleapis.com/Network" - computeSubnetworkAssetType = "compute.googleapis.com/Subnetwork" - computeDiskAssetType = "compute.googleapis.com/Disk" - computeImageAssetType = "compute.googleapis.com/Image" - dnsManagedZoneAssetType = "dns.googleapis.com/ManagedZone" - computeInstanceGroupAssetType = "compute.googleapis.com/InstanceGroup" - bigqueryDatasetAssetType = "bigquery.googleapis.com/Dataset" - bigqueryTableAssetType = "bigquery.googleapis.com/Table" - computeAddressAssetType = "compute.googleapis.com/Address" - computeGlobalAddressAssetType = "compute.googleapis.com/GlobalAddress" - cloudFunctionsFunction = "cloudfunctions.googleapis.com/CloudFunction" - bigtableInstanceAssetType = "bigtableadmin.googleapis.com/Instance" - bigtableTableAssetType = "bigtableadmin.googleapis.com/Table" - sqlDatabaseInstanceAssetType = "sqladmin.googleapis.com/Instance" - healthCheckAssetType = "compute.googleapis.com/HealthCheck" - cloudRunServiceAssetType = "run.googleapis.com/Service" - nodeGroupAssetType = "compute.googleapis.com/NodeGroup" + storageBucketAssetType = "storage.googleapis.com/Bucket" + computeFirewallAssetType = "compute.googleapis.com/Firewall" + computeRouterAssetType = "compute.googleapis.com/Router" + computeInstanceAssetType = "compute.googleapis.com/Instance" + computeNetworkAssetType = "compute.googleapis.com/Network" + computeSubnetworkAssetType = "compute.googleapis.com/Subnetwork" + computeDiskAssetType = "compute.googleapis.com/Disk" + computeImageAssetType = "compute.googleapis.com/Image" + dnsManagedZoneAssetType = "dns.googleapis.com/ManagedZone" + computeInstanceGroupAssetType = "compute.googleapis.com/InstanceGroup" + bigqueryDatasetAssetType = "bigquery.googleapis.com/Dataset" + bigqueryTableAssetType = "bigquery.googleapis.com/Table" + computeAddressAssetType = "compute.googleapis.com/Address" + computeGlobalAddressAssetType = "compute.googleapis.com/GlobalAddress" + cloudFunctionsFunction = "cloudfunctions.googleapis.com/CloudFunction" + bigtableInstanceAssetType = "bigtableadmin.googleapis.com/Instance" + bigtableTableAssetType = "bigtableadmin.googleapis.com/Table" + sqlDatabaseInstanceAssetType = "sqladmin.googleapis.com/Instance" + healthCheckAssetType = "compute.googleapis.com/HealthCheck" + cloudRunServiceAssetType = "run.googleapis.com/Service" + nodeGroupAssetType = "compute.googleapis.com/NodeGroup" ) type AssetRepository interface { diff --git a/pkg/remote/google/repository/cloudresourcemanager.go b/pkg/remote/google/repository/cloudresourcemanager.go index 48ab7349f..cbd4b00f0 100644 --- a/pkg/remote/google/repository/cloudresourcemanager.go +++ b/pkg/remote/google/repository/cloudresourcemanager.go @@ -48,9 +48,9 @@ func (s *cloudResourceManagerRepository) ListProjectsBindings() (map[string]map[ for _, binding := range policy.Bindings { bindings[binding.Role] = binding.Members } - + bindingsByProject[project] = bindings - + s.cache.Put("ListProjectsBindings", bindingsByProject) } diff --git a/pkg/remote/google_bigquery_scanner_test.go b/pkg/remote/google_bigquery_scanner_test.go index 8c58726dc..641c133c3 100644 --- a/pkg/remote/google_bigquery_scanner_test.go +++ b/pkg/remote/google_bigquery_scanner_test.go @@ -105,7 +105,7 @@ func TestGoogleBigqueryDataset(t *testing.T) { tt.Fatal(err) } - repo := repository.NewAssetRepository(assetClient, realProvider.SetConfig([]string{"projects/123456"}), cache.New(0)) + repo := repository.NewAssetRepository(assetClient, realProvider.SetConfig([]string{"projects/123456"}), cache.New(0)) remoteLibrary.AddEnumerator(google.NewGoogleBigqueryDatasetEnumerator(repo, factory)) From aca1cf5e2bdc519c9413a4c771f4a839da954a07 Mon Sep 17 00:00:00 2001 From: bgdanix Date: Wed, 22 Dec 2021 11:54:31 +0000 Subject: [PATCH 6/9] fix: PR comments and errors for asset scanning --- pkg/cmd/scan.go | 4 +-- pkg/remote/google/config/config.go | 2 +- pkg/remote/google/provider.go | 4 +-- pkg/remote/google/repository/asset.go | 33 +++++++++++++++---- pkg/remote/google/repository/asset_test.go | 14 ++++---- .../google/repository/cloudresourcemanager.go | 13 ++++---- 6 files changed, 46 insertions(+), 24 deletions(-) diff --git a/pkg/cmd/scan.go b/pkg/cmd/scan.go index 44d07f32f..81360799d 100644 --- a/pkg/cmd/scan.go +++ b/pkg/cmd/scan.go @@ -142,7 +142,7 @@ func NewScanCmd(opts *pkg.ScanOptions) *cobra.Command { ) fl.StringSliceP( "gcp-scope", - "s", + "", []string{}, "Set the GCP scope for search", ) @@ -347,7 +347,7 @@ func scanRun(opts *pkg.ScanOptions) error { func parseScopeFlag(scope []string) ([]string, error) { - scopeRegex := `projects/\S*$|folders/\d*$|organizations/\S*$` + scopeRegex := `projects/\S*$|folders/\d*$|organizations/\d*$` r := regexp.MustCompile(scopeRegex) for _, v := range scope { diff --git a/pkg/remote/google/config/config.go b/pkg/remote/google/config/config.go index 0e6456f27..0ecbd4170 100644 --- a/pkg/remote/google/config/config.go +++ b/pkg/remote/google/config/config.go @@ -1,5 +1,5 @@ package config type GCPTerraformConfig struct { - Scope []string `cty:"scope"` + Scopes []string `cty:"scopes"` } diff --git a/pkg/remote/google/provider.go b/pkg/remote/google/provider.go index 380e8019b..06bd591c4 100644 --- a/pkg/remote/google/provider.go +++ b/pkg/remote/google/provider.go @@ -53,8 +53,8 @@ func (p *GCPTerraformProvider) Version() string { return p.version } -func (p *GCPTerraformProvider) SetConfig(scope []string) config.GCPTerraformConfig { +func (p *GCPTerraformProvider) SetConfig(scopes []string) config.GCPTerraformConfig { return config.GCPTerraformConfig{ - Scope: scope, + Scopes: scopes, } } diff --git a/pkg/remote/google/repository/asset.go b/pkg/remote/google/repository/asset.go index f7763b14e..7b15569df 100644 --- a/pkg/remote/google/repository/asset.go +++ b/pkg/remote/google/repository/asset.go @@ -2,6 +2,8 @@ package repository import ( "context" + "errors" + "fmt" asset "cloud.google.com/go/asset/apiv1" "github.com/snyk/driftctl/pkg/remote/cache" @@ -76,9 +78,10 @@ func NewAssetRepository(client *asset.Client, config config.GCPTerraformConfig, func (s assetRepository) listAllResources(ty string) ([]*assetpb.Asset, error) { filteredResults := []*assetpb.Asset{} + var erorString string - for _, scope := range s.config.Scope { - cacheKey := "listAllResources" + scope + for _, scope := range s.config.Scopes { + cacheKey := fmt.Sprintf("listAllResources_%s", scope) cachedResults := s.cache.GetAndLock(cacheKey) defer s.cache.Unlock(cacheKey) @@ -108,7 +111,11 @@ func (s assetRepository) listAllResources(ty string) ([]*assetpb.Asset, error) { if err == iterator.Done { break } - if err != nil { + if err != nil && resource != nil{ + erorString = erorString + fmt.Sprintf("For scope %s on resource %s got error: %s; ", scope, resource.AssetType, err.Error()) + continue + } + if err != nil && resource == nil{ return nil, err } results = append(results, resource) @@ -122,15 +129,21 @@ func (s assetRepository) listAllResources(ty string) ([]*assetpb.Asset, error) { } } } + + if len(erorString) > 0 { + return filteredResults, errors.New(erorString) + } + return filteredResults, nil } func (s assetRepository) searchAllResources(ty string) ([]*assetpb.ResourceSearchResult, error) { filteredResults := []*assetpb.ResourceSearchResult{} + var erorString string - for _, scope := range s.config.Scope { - cacheKey := "SearchAllResources" + scope + for _, scope := range s.config.Scopes { + cacheKey := fmt.Sprintf("SearchAllResources_%s", scope) cachedResults := s.cache.GetAndLock(cacheKey) defer s.cache.Unlock(cacheKey) @@ -167,7 +180,11 @@ func (s assetRepository) searchAllResources(ty string) ([]*assetpb.ResourceSearc if err == iterator.Done { break } - if err != nil { + if err != nil && resource != nil{ + erorString = erorString + fmt.Sprintf("For scope %s on resource %s got error: %s; ", scope, resource.AssetType, err.Error()) + continue + } + if err != nil && resource == nil{ return nil, err } results = append(results, resource) @@ -182,6 +199,10 @@ func (s assetRepository) searchAllResources(ty string) ([]*assetpb.ResourceSearc } } + if len(erorString) > 0 { + return filteredResults, errors.New(erorString) + } + return filteredResults, nil } diff --git a/pkg/remote/google/repository/asset_test.go b/pkg/remote/google/repository/asset_test.go index b51943084..a1f0643c0 100644 --- a/pkg/remote/google/repository/asset_test.go +++ b/pkg/remote/google/repository/asset_test.go @@ -25,9 +25,9 @@ func Test_assetRepository_searchAllResources_CacheHit(t *testing.T) { } c := &cache.MockCache{} - c.On("GetAndLock", "SearchAllResources").Return(expectedResults).Times(1) - c.On("Unlock", "SearchAllResources").Times(1) - repo := NewAssetRepository(nil, config.GCPTerraformConfig{Scope: []string{""}}, c) + c.On("GetAndLock", "SearchAllResources_").Return(expectedResults).Times(1) + c.On("Unlock", "SearchAllResources_").Times(1) + repo := NewAssetRepository(nil, config.GCPTerraformConfig{Scopes: []string{""}}, c) got, err := repo.searchAllResources("google_fake_type") c.AssertExpectations(t) @@ -52,10 +52,10 @@ func Test_assetRepository_searchAllResources_CacheMiss(t *testing.T) { t.Fatal(err) } c := &cache.MockCache{} - c.On("GetAndLock", "SearchAllResources").Return(nil).Times(1) - c.On("Unlock", "SearchAllResources").Times(1) - c.On("Put", "SearchAllResources", mock.IsType([]*assetpb.ResourceSearchResult{})).Return(false).Times(1) - repo := NewAssetRepository(assetClient, config.GCPTerraformConfig{Scope: []string{""}}, c) + c.On("GetAndLock", "SearchAllResources_").Return(nil).Times(1) + c.On("Unlock", "SearchAllResources_").Times(1) + c.On("Put", "SearchAllResources_", mock.IsType([]*assetpb.ResourceSearchResult{})).Return(false).Times(1) + repo := NewAssetRepository(assetClient, config.GCPTerraformConfig{Scopes: []string{""}}, c) got, err := repo.searchAllResources("google_fake_type") c.AssertExpectations(t) diff --git a/pkg/remote/google/repository/cloudresourcemanager.go b/pkg/remote/google/repository/cloudresourcemanager.go index cbd4b00f0..4220e8cfb 100644 --- a/pkg/remote/google/repository/cloudresourcemanager.go +++ b/pkg/remote/google/repository/cloudresourcemanager.go @@ -2,6 +2,7 @@ package repository import ( "errors" + "fmt" "strings" "github.com/snyk/driftctl/pkg/remote/cache" @@ -31,9 +32,9 @@ func (s *cloudResourceManagerRepository) ListProjectsBindings() (map[string]map[ bindingsByProject := make(map[string]map[string][]string) errorsByProject := make(map[string]error) - var erorsString string + var erorString string - for _, scope := range s.config.Scope { + for _, scope := range s.config.Scopes { if strings.Contains(scope, "projects/") { project := strings.Split(scope, "projects/")[1] request := new(cloudresourcemanager.GetIamPolicyRequest) @@ -58,10 +59,10 @@ func (s *cloudResourceManagerRepository) ListProjectsBindings() (map[string]map[ if len(errorsByProject) > 0 { for project, errval := range errorsByProject { - erorsString = erorsString + "Project: " + project + " had the following error: " + errval.Error() + "; " + erorString = erorString + fmt.Sprintf("Project: %s had the following error: %s; ", project, errval.Error()) } - return bindingsByProject, errors.New(erorsString) - } else { - return bindingsByProject, nil + return bindingsByProject, errors.New(erorString) } + + return bindingsByProject, nil } From dced4e9c4fe06499916d6713fe3cc27b0aca034d Mon Sep 17 00:00:00 2001 From: bgdanix Date: Wed, 22 Dec 2021 12:00:43 +0000 Subject: [PATCH 7/9] fix: Go formatter --- pkg/remote/google/repository/asset.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/remote/google/repository/asset.go b/pkg/remote/google/repository/asset.go index 7b15569df..7ad86e2bb 100644 --- a/pkg/remote/google/repository/asset.go +++ b/pkg/remote/google/repository/asset.go @@ -111,11 +111,11 @@ func (s assetRepository) listAllResources(ty string) ([]*assetpb.Asset, error) { if err == iterator.Done { break } - if err != nil && resource != nil{ + if err != nil && resource != nil { erorString = erorString + fmt.Sprintf("For scope %s on resource %s got error: %s; ", scope, resource.AssetType, err.Error()) continue } - if err != nil && resource == nil{ + if err != nil && resource == nil { return nil, err } results = append(results, resource) @@ -180,11 +180,11 @@ func (s assetRepository) searchAllResources(ty string) ([]*assetpb.ResourceSearc if err == iterator.Done { break } - if err != nil && resource != nil{ + if err != nil && resource != nil { erorString = erorString + fmt.Sprintf("For scope %s on resource %s got error: %s; ", scope, resource.AssetType, err.Error()) continue } - if err != nil && resource == nil{ + if err != nil && resource == nil { return nil, err } results = append(results, resource) From 1214a02a1f6f1e6d3df4765359f4756455183f59 Mon Sep 17 00:00:00 2001 From: bgdanix Date: Wed, 22 Dec 2021 12:53:16 +0000 Subject: [PATCH 8/9] fix: improved tests and scan cmd option --- pkg/cmd/scan.go | 3 +-- pkg/remote/google/repository/asset.go | 16 ++++++++-------- pkg/remote/google/repository/asset_test.go | 14 +++++++------- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/pkg/cmd/scan.go b/pkg/cmd/scan.go index 81360799d..f518e1b71 100644 --- a/pkg/cmd/scan.go +++ b/pkg/cmd/scan.go @@ -140,9 +140,8 @@ func NewScanCmd(opts *pkg.ScanOptions) *cobra.Command { false, "Do not display anything but scan results", ) - fl.StringSliceP( + fl.StringSlice( "gcp-scope", - "", []string{}, "Set the GCP scope for search", ) diff --git a/pkg/remote/google/repository/asset.go b/pkg/remote/google/repository/asset.go index 7ad86e2bb..a1d5d179a 100644 --- a/pkg/remote/google/repository/asset.go +++ b/pkg/remote/google/repository/asset.go @@ -78,7 +78,7 @@ func NewAssetRepository(client *asset.Client, config config.GCPTerraformConfig, func (s assetRepository) listAllResources(ty string) ([]*assetpb.Asset, error) { filteredResults := []*assetpb.Asset{} - var erorString string + var errorString string for _, scope := range s.config.Scopes { cacheKey := fmt.Sprintf("listAllResources_%s", scope) @@ -112,7 +112,7 @@ func (s assetRepository) listAllResources(ty string) ([]*assetpb.Asset, error) { break } if err != nil && resource != nil { - erorString = erorString + fmt.Sprintf("For scope %s on resource %s got error: %s; ", scope, resource.AssetType, err.Error()) + errorString = errorString + fmt.Sprintf("For scope %s on resource %s got error: %s; ", scope, resource.AssetType, err.Error()) continue } if err != nil && resource == nil { @@ -130,8 +130,8 @@ func (s assetRepository) listAllResources(ty string) ([]*assetpb.Asset, error) { } } - if len(erorString) > 0 { - return filteredResults, errors.New(erorString) + if len(errorString) > 0 { + return filteredResults, errors.New(errorString) } return filteredResults, nil @@ -140,7 +140,7 @@ func (s assetRepository) listAllResources(ty string) ([]*assetpb.Asset, error) { func (s assetRepository) searchAllResources(ty string) ([]*assetpb.ResourceSearchResult, error) { filteredResults := []*assetpb.ResourceSearchResult{} - var erorString string + var errorString string for _, scope := range s.config.Scopes { cacheKey := fmt.Sprintf("SearchAllResources_%s", scope) @@ -181,7 +181,7 @@ func (s assetRepository) searchAllResources(ty string) ([]*assetpb.ResourceSearc break } if err != nil && resource != nil { - erorString = erorString + fmt.Sprintf("For scope %s on resource %s got error: %s; ", scope, resource.AssetType, err.Error()) + errorString = errorString + fmt.Sprintf("For scope %s on resource %s got error: %s; ", scope, resource.AssetType, err.Error()) continue } if err != nil && resource == nil { @@ -199,8 +199,8 @@ func (s assetRepository) searchAllResources(ty string) ([]*assetpb.ResourceSearc } } - if len(erorString) > 0 { - return filteredResults, errors.New(erorString) + if len(errorString) > 0 { + return filteredResults, errors.New(errorString) } return filteredResults, nil diff --git a/pkg/remote/google/repository/asset_test.go b/pkg/remote/google/repository/asset_test.go index a1f0643c0..cbaf793f7 100644 --- a/pkg/remote/google/repository/asset_test.go +++ b/pkg/remote/google/repository/asset_test.go @@ -25,9 +25,9 @@ func Test_assetRepository_searchAllResources_CacheHit(t *testing.T) { } c := &cache.MockCache{} - c.On("GetAndLock", "SearchAllResources_").Return(expectedResults).Times(1) - c.On("Unlock", "SearchAllResources_").Times(1) - repo := NewAssetRepository(nil, config.GCPTerraformConfig{Scopes: []string{""}}, c) + c.On("GetAndLock", "SearchAllResources_folders/345612").Return(expectedResults).Times(1) + c.On("Unlock", "SearchAllResources_folders/345612").Times(1) + repo := NewAssetRepository(nil, config.GCPTerraformConfig{Scopes: []string{"folders/345612"}}, c) got, err := repo.searchAllResources("google_fake_type") c.AssertExpectations(t) @@ -52,10 +52,10 @@ func Test_assetRepository_searchAllResources_CacheMiss(t *testing.T) { t.Fatal(err) } c := &cache.MockCache{} - c.On("GetAndLock", "SearchAllResources_").Return(nil).Times(1) - c.On("Unlock", "SearchAllResources_").Times(1) - c.On("Put", "SearchAllResources_", mock.IsType([]*assetpb.ResourceSearchResult{})).Return(false).Times(1) - repo := NewAssetRepository(assetClient, config.GCPTerraformConfig{Scopes: []string{""}}, c) + c.On("GetAndLock", "SearchAllResources_folders/345612").Return(nil).Times(1) + c.On("Unlock", "SearchAllResources_folders/345612").Times(1) + c.On("Put", "SearchAllResources_folders/345612", mock.IsType([]*assetpb.ResourceSearchResult{})).Return(false).Times(1) + repo := NewAssetRepository(assetClient, config.GCPTerraformConfig{Scopes: []string{"folders/345612"}}, c) got, err := repo.searchAllResources("google_fake_type") c.AssertExpectations(t) From 2ccb9a9e8aa0e70829bb04c113228854469a9b55 Mon Sep 17 00:00:00 2001 From: bgdanix Date: Thu, 23 Dec 2021 10:26:21 +0000 Subject: [PATCH 9/9] fix: support multiple errors by scope --- pkg/remote/google/repository/asset.go | 24 +++++++++++++++---- pkg/remote/google_bigquery_scanner_test.go | 4 ++-- pkg/remote/google_bigtable_scanner_test.go | 4 ++-- .../google_cloudfunctions_scanner_test.go | 2 +- pkg/remote/google_cloudrun_scanner_test.go | 2 +- pkg/remote/google_compute_scanner_test.go | 24 +++++++++---------- pkg/remote/google_network_scanner_test.go | 2 +- pkg/remote/google_sql_scanner_test.go | 2 +- pkg/remote/google_storage_scanner_test.go | 2 +- 9 files changed, 41 insertions(+), 25 deletions(-) diff --git a/pkg/remote/google/repository/asset.go b/pkg/remote/google/repository/asset.go index a1d5d179a..dddb39b3a 100644 --- a/pkg/remote/google/repository/asset.go +++ b/pkg/remote/google/repository/asset.go @@ -10,6 +10,8 @@ import ( "github.com/snyk/driftctl/pkg/remote/google/config" "google.golang.org/api/iterator" assetpb "google.golang.org/genproto/googleapis/cloud/asset/v1" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // https://cloud.google.com/asset-inventory/docs/supported-asset-types#supported_resource_types @@ -79,6 +81,7 @@ func (s assetRepository) listAllResources(ty string) ([]*assetpb.Asset, error) { filteredResults := []*assetpb.Asset{} var errorString string + var errCode codes.Code for _, scope := range s.config.Scopes { cacheKey := fmt.Sprintf("listAllResources_%s", scope) @@ -116,7 +119,9 @@ func (s assetRepository) listAllResources(ty string) ([]*assetpb.Asset, error) { continue } if err != nil && resource == nil { - return nil, err + errorString = errorString + fmt.Sprintf("For scope %s got error: %s; ", scope, err.Error()) + errCode = status.Code(err) + break } results = append(results, resource) } @@ -130,10 +135,14 @@ func (s assetRepository) listAllResources(ty string) ([]*assetpb.Asset, error) { } } - if len(errorString) > 0 { + if len(errorString) > 0 && len(filteredResults) > 0 { return filteredResults, errors.New(errorString) } + if len(errorString) > 0 && len(filteredResults) == 0 { + return nil, status.Error(errCode, errorString) + } + return filteredResults, nil } @@ -141,6 +150,7 @@ func (s assetRepository) searchAllResources(ty string) ([]*assetpb.ResourceSearc filteredResults := []*assetpb.ResourceSearchResult{} var errorString string + var errCode codes.Code for _, scope := range s.config.Scopes { cacheKey := fmt.Sprintf("SearchAllResources_%s", scope) @@ -185,7 +195,9 @@ func (s assetRepository) searchAllResources(ty string) ([]*assetpb.ResourceSearc continue } if err != nil && resource == nil { - return nil, err + errorString = errorString + fmt.Sprintf("For scope %s got error: %s; ", scope, err.Error()) + errCode = status.Code(err) + break } results = append(results, resource) } @@ -199,10 +211,14 @@ func (s assetRepository) searchAllResources(ty string) ([]*assetpb.ResourceSearc } } - if len(errorString) > 0 { + if len(errorString) > 0 && len(filteredResults) > 0 { return filteredResults, errors.New(errorString) } + if len(errorString) > 0 && len(filteredResults) == 0 { + return nil, status.Error(errCode, errorString) + } + return filteredResults, nil } diff --git a/pkg/remote/google_bigquery_scanner_test.go b/pkg/remote/google_bigquery_scanner_test.go index 641c133c3..1b3daeb23 100644 --- a/pkg/remote/google_bigquery_scanner_test.go +++ b/pkg/remote/google_bigquery_scanner_test.go @@ -68,7 +68,7 @@ func TestGoogleBigqueryDataset(t *testing.T) { alerts.NewRemoteAccessDeniedAlert( common.RemoteGoogleTerraform, remoteerr.NewResourceListingError( - status.Error(codes.PermissionDenied, "The caller does not have permission"), + status.Error(codes.PermissionDenied, "For scope projects/123456 got error: "+status.Error(codes.PermissionDenied, "The caller does not have permission").Error()+"; "), "google_bigquery_dataset", ), alerts.EnumerationPhase, @@ -171,7 +171,7 @@ func TestGoogleBigqueryTable(t *testing.T) { alerts.NewRemoteAccessDeniedAlert( common.RemoteGoogleTerraform, remoteerr.NewResourceListingError( - status.Error(codes.PermissionDenied, "The caller does not have permission"), + status.Error(codes.PermissionDenied, "For scope projects/123456 got error: "+status.Error(codes.PermissionDenied, "The caller does not have permission").Error()+"; "), "google_bigquery_table", ), alerts.EnumerationPhase, diff --git a/pkg/remote/google_bigtable_scanner_test.go b/pkg/remote/google_bigtable_scanner_test.go index dc037401b..765ebf130 100644 --- a/pkg/remote/google_bigtable_scanner_test.go +++ b/pkg/remote/google_bigtable_scanner_test.go @@ -105,7 +105,7 @@ func TestGoogleBigtableInstance(t *testing.T) { alerts.NewRemoteAccessDeniedAlert( common.RemoteGoogleTerraform, remoteerr.NewResourceListingError( - status.Error(codes.PermissionDenied, "The caller does not have permission"), + status.Error(codes.PermissionDenied, "For scope projects/123456 got error: "+status.Error(codes.PermissionDenied, "The caller does not have permission").Error()+"; "), "google_bigtable_instance", ), alerts.EnumerationPhase, @@ -231,7 +231,7 @@ func TestGoogleBigtableTable(t *testing.T) { alerts.NewRemoteAccessDeniedAlert( common.RemoteGoogleTerraform, remoteerr.NewResourceListingError( - status.Error(codes.PermissionDenied, "The caller does not have permission"), + status.Error(codes.PermissionDenied, "For scope projects/123456 got error: "+status.Error(codes.PermissionDenied, "The caller does not have permission").Error()+"; "), "google_bigtable_table", ), alerts.EnumerationPhase, diff --git a/pkg/remote/google_cloudfunctions_scanner_test.go b/pkg/remote/google_cloudfunctions_scanner_test.go index 6227c5d4a..5c928b122 100644 --- a/pkg/remote/google_cloudfunctions_scanner_test.go +++ b/pkg/remote/google_cloudfunctions_scanner_test.go @@ -92,7 +92,7 @@ func TestGoogleCloudFunctionsFunction(t *testing.T) { alerts.NewRemoteAccessDeniedAlert( common.RemoteGoogleTerraform, remoteerr.NewResourceListingError( - status.Error(codes.PermissionDenied, "The caller does not have permission"), + status.Error(codes.PermissionDenied, "For scope projects/123456 got error: "+status.Error(codes.PermissionDenied, "The caller does not have permission").Error()+"; "), "google_cloudfunctions_function", ), alerts.EnumerationPhase, diff --git a/pkg/remote/google_cloudrun_scanner_test.go b/pkg/remote/google_cloudrun_scanner_test.go index ddc57c189..980d9aee1 100644 --- a/pkg/remote/google_cloudrun_scanner_test.go +++ b/pkg/remote/google_cloudrun_scanner_test.go @@ -84,7 +84,7 @@ func TestGoogleCloudRunService(t *testing.T) { alerts.NewRemoteAccessDeniedAlert( common.RemoteGoogleTerraform, remoteerr.NewResourceListingError( - status.Error(codes.PermissionDenied, "The caller does not have permission"), + status.Error(codes.PermissionDenied, "For scope projects/123456 got error: "+status.Error(codes.PermissionDenied, "The caller does not have permission").Error()+"; "), googleresource.GoogleCloudRunServiceResourceType, ), alerts.EnumerationPhase, diff --git a/pkg/remote/google_compute_scanner_test.go b/pkg/remote/google_compute_scanner_test.go index c53440a24..d4d90f8a1 100644 --- a/pkg/remote/google_compute_scanner_test.go +++ b/pkg/remote/google_compute_scanner_test.go @@ -76,7 +76,7 @@ func TestGoogleComputeFirewall(t *testing.T) { alerts.NewRemoteAccessDeniedAlert( common.RemoteGoogleTerraform, remoteerr.NewResourceListingError( - status.Error(codes.PermissionDenied, "The caller does not have permission"), + status.Error(codes.PermissionDenied, "For scope projects/123456 got error: "+status.Error(codes.PermissionDenied, "The caller does not have permission").Error()+"; "), "google_compute_firewall", ), alerts.EnumerationPhase, @@ -212,7 +212,7 @@ func TestGoogleComputeRouter(t *testing.T) { alerts.NewRemoteAccessDeniedAlert( common.RemoteGoogleTerraform, remoteerr.NewResourceListingError( - status.Error(codes.PermissionDenied, "The caller does not have permission"), + status.Error(codes.PermissionDenied, "For scope projects/123456 got error: "+status.Error(codes.PermissionDenied, "The caller does not have permission").Error()+"; "), googleresource.GoogleComputeRouterResourceType, ), alerts.EnumerationPhase, @@ -319,7 +319,7 @@ func TestGoogleComputeInstance(t *testing.T) { alerts.NewRemoteAccessDeniedAlert( common.RemoteGoogleTerraform, remoteerr.NewResourceListingError( - status.Error(codes.PermissionDenied, "The caller does not have permission"), + status.Error(codes.PermissionDenied, "For scope projects/123456 got error: "+status.Error(codes.PermissionDenied, "The caller does not have permission").Error()+"; "), "google_compute_instance", ), alerts.EnumerationPhase, @@ -427,7 +427,7 @@ func TestGoogleComputeNetwork(t *testing.T) { alerts.NewRemoteAccessDeniedAlert( common.RemoteGoogleTerraform, remoteerr.NewResourceListingError( - status.Error(codes.PermissionDenied, "The caller does not have permission"), + status.Error(codes.PermissionDenied, "For scope projects/123456 got error: "+status.Error(codes.PermissionDenied, "The caller does not have permission").Error()+"; "), "google_compute_network", ), alerts.EnumerationPhase, @@ -549,7 +549,7 @@ func TestGoogleComputeInstanceGroup(t *testing.T) { alerts.NewRemoteAccessDeniedAlert( common.RemoteGoogleTerraform, remoteerr.NewResourceListingError( - status.Error(codes.PermissionDenied, "The caller does not have permission"), + status.Error(codes.PermissionDenied, "For scope projects/123456 got error: "+status.Error(codes.PermissionDenied, "The caller does not have permission").Error()+"; "), "google_compute_instance_group", ), alerts.EnumerationPhase, @@ -686,7 +686,7 @@ func TestGoogleComputeAddress(t *testing.T) { alerts.NewRemoteAccessDeniedAlert( common.RemoteGoogleTerraform, remoteerr.NewResourceListingError( - status.Error(codes.PermissionDenied, "The caller does not have permission"), + status.Error(codes.PermissionDenied, "For scope projects/123456 got error: "+status.Error(codes.PermissionDenied, "The caller does not have permission").Error()+"; "), "google_compute_address", ), alerts.EnumerationPhase, @@ -812,7 +812,7 @@ func TestGoogleComputeGlobalAddress(t *testing.T) { alerts.NewRemoteAccessDeniedAlert( common.RemoteGoogleTerraform, remoteerr.NewResourceListingError( - status.Error(codes.PermissionDenied, "The caller does not have permission"), + status.Error(codes.PermissionDenied, "For scope projects/123456 got error: "+status.Error(codes.PermissionDenied, "The caller does not have permission").Error()+"; "), "google_compute_global_address", ), alerts.EnumerationPhase, @@ -920,7 +920,7 @@ func TestGoogleComputeSubnetwork(t *testing.T) { alerts.NewRemoteAccessDeniedAlert( common.RemoteGoogleTerraform, remoteerr.NewResourceListingError( - status.Error(codes.PermissionDenied, "The caller does not have permission"), + status.Error(codes.PermissionDenied, "For scope projects/123456 got error: "+status.Error(codes.PermissionDenied, "The caller does not have permission").Error()+"; "), "google_compute_subnetwork", ), alerts.EnumerationPhase, @@ -1045,7 +1045,7 @@ func TestGoogleComputeDisk(t *testing.T) { alerts.NewRemoteAccessDeniedAlert( common.RemoteGoogleTerraform, remoteerr.NewResourceListingError( - status.Error(codes.PermissionDenied, "The caller does not have permission"), + status.Error(codes.PermissionDenied, "For scope projects/123456 got error: "+status.Error(codes.PermissionDenied, "The caller does not have permission").Error()+"; "), "google_compute_disk", ), alerts.EnumerationPhase, @@ -1155,7 +1155,7 @@ func TestGoogleComputeImage(t *testing.T) { alerts.NewRemoteAccessDeniedAlert( common.RemoteGoogleTerraform, remoteerr.NewResourceListingError( - status.Error(codes.PermissionDenied, "The caller does not have permission"), + status.Error(codes.PermissionDenied, "For scope projects/123456 got error: "+status.Error(codes.PermissionDenied, "The caller does not have permission").Error()+"; "), "google_compute_image", ), alerts.EnumerationPhase, @@ -1265,7 +1265,7 @@ func TestGoogleComputeHealthCheck(t *testing.T) { alerts.NewRemoteAccessDeniedAlert( common.RemoteGoogleTerraform, remoteerr.NewResourceListingError( - status.Error(codes.PermissionDenied, "The caller does not have permission"), + status.Error(codes.PermissionDenied, "For scope projects/123456 got error: "+status.Error(codes.PermissionDenied, "The caller does not have permission").Error()+"; "), "google_compute_health_check", ), alerts.EnumerationPhase, @@ -1375,7 +1375,7 @@ func TestGoogleComputeNodeGroup(t *testing.T) { alerts.NewRemoteAccessDeniedAlert( common.RemoteGoogleTerraform, remoteerr.NewResourceListingError( - status.Error(codes.PermissionDenied, "The caller does not have permission"), + status.Error(codes.PermissionDenied, "For scope projects/123456 got error: "+status.Error(codes.PermissionDenied, "The caller does not have permission").Error()+"; "), "google_compute_node_group", ), alerts.EnumerationPhase, diff --git a/pkg/remote/google_network_scanner_test.go b/pkg/remote/google_network_scanner_test.go index 9ae0882a4..a1373e86d 100644 --- a/pkg/remote/google_network_scanner_test.go +++ b/pkg/remote/google_network_scanner_test.go @@ -90,7 +90,7 @@ func TestGoogleDNSNanagedZone(t *testing.T) { alerts.NewRemoteAccessDeniedAlert( common.RemoteGoogleTerraform, remoteerr.NewResourceListingError( - status.Error(codes.PermissionDenied, "The caller does not have permission"), + status.Error(codes.PermissionDenied, "For scope projects/123456 got error: "+status.Error(codes.PermissionDenied, "The caller does not have permission").Error()+"; "), googleresource.GoogleDNSManagedZoneResourceType, ), alerts.EnumerationPhase, diff --git a/pkg/remote/google_sql_scanner_test.go b/pkg/remote/google_sql_scanner_test.go index 3f2fb233a..1872d3d02 100644 --- a/pkg/remote/google_sql_scanner_test.go +++ b/pkg/remote/google_sql_scanner_test.go @@ -90,7 +90,7 @@ func TestGoogleSQLDatabaseInstance(t *testing.T) { alerts.NewRemoteAccessDeniedAlert( common.RemoteGoogleTerraform, remoteerr.NewResourceListingError( - status.Error(codes.PermissionDenied, "The caller does not have permission"), + status.Error(codes.PermissionDenied, "For scope projects/123456 got error: "+status.Error(codes.PermissionDenied, "The caller does not have permission").Error()+"; "), "google_sql_database_instance", ), alerts.EnumerationPhase, diff --git a/pkg/remote/google_storage_scanner_test.go b/pkg/remote/google_storage_scanner_test.go index 425dd0746..b155464df 100644 --- a/pkg/remote/google_storage_scanner_test.go +++ b/pkg/remote/google_storage_scanner_test.go @@ -76,7 +76,7 @@ func TestGoogleStorageBucket(t *testing.T) { alerts.NewRemoteAccessDeniedAlert( common.RemoteGoogleTerraform, remoteerr.NewResourceListingError( - status.Error(codes.PermissionDenied, "The caller does not have permission"), + status.Error(codes.PermissionDenied, "For scope projects/123456 got error: "+status.Error(codes.PermissionDenied, "The caller does not have permission").Error()+"; "), "google_storage_bucket", ), alerts.EnumerationPhase,