Merge pull request #200 from dbt-labs/release-0.2.10

Release 0.2.10

CHANGELOG.md

@@ -2,7 +2,20 @@
 
 All notable changes to this project will be documented in this file.
 
-## [Unreleased](https://github.com/dbt-labs/terraform-provider-dbtcloud/compare/v0.2.9...HEAD)
+## [Unreleased](https://github.com/dbt-labs/terraform-provider-dbtcloud/compare/v0.2.10...HEAD)
+
+
+## [0.2.10](https://github.com/dbt-labs/terraform-provider-dbtcloud/compare/v0.2.9...v0.2.10)
+
+## Fix
+
+- [#197](https://github.com/dbt-labs/terraform-provider-dbtcloud/issues/197) - Community contribution to handle cases where more than 100 groups are created in dbt Cloud
+- [#199](https://github.com/dbt-labs/terraform-provider-dbtcloud/issues/199) - Update logic to allow finding users by their email addresses in a cases insensitive way
+- [#198](https://github.com/dbt-labs/terraform-provider-dbtcloud/issues/198) - Update some internal logic to call endpoints by their unique IDs instead of looping through answers to avoid issues like #199 and paginate through results for endpoints where we can't query the ID directly
+
+## Changes
+
+- [#189](https://github.com/dbt-labs/terraform-provider-dbtcloud/issues/189) - Allow users to retrieve project data sources by providing project names instead of project IDs. This will return an error if more than 1 project has the given name and takes care of the pagination required for handling more than 100 projects
 
 ## [0.2.9](https://github.com/dbt-labs/terraform-provider-dbtcloud/compare/v0.2.8...v0.2.9)
 

examples/data-sources/dbtcloud_project/data-source.tf

@@ -1,6 +1,13 @@
 // use dbt_cloud_project instead of dbtcloud_project for the legacy resource names
 // legacy names will be removed from 0.3 onwards
 
+// projects data sources can use the project_id parameter (preferred uniqueness is ensured)
 data "dbtcloud_project" "test_project" {
   project_id = var.dbt_cloud_project_id
 }
+
+// or they can use project names
+// the provider will raise an error if more than one project is found with the same name
+data "dbtcloud_project" "test_project" {
+  name = "My project name"
+}

pkg/data_sources/project.go

@@ -2,6 +2,7 @@ package data_sources
 
 import (
 	"context"
+	"fmt"
 	"strconv"
 
 	"github.com/dbt-labs/terraform-provider-dbtcloud/pkg/dbt_cloud"
@@ -12,12 +13,13 @@ import (
 var projectSchema = map[string]*schema.Schema{
 	"project_id": &schema.Schema{
 		Type:        schema.TypeInt,
-		Required:    true,
+		Optional:    true,
 		Description: "ID of the project to represent",
 	},
 	"name": &schema.Schema{
 		Type:        schema.TypeString,
 		Computed:    true,
+		Optional:    true,
 		Description: "Given name for project",
 	},
 	"connection_id": &schema.Schema{
@@ -58,12 +60,32 @@ func datasourceProjectRead(ctx context.Context, d *schema.ResourceData, m interf
 	c := m.(*dbt_cloud.Client)
 
 	var diags diag.Diagnostics
+	var project *dbt_cloud.Project
 
-	projectId := strconv.Itoa(d.Get("project_id").(int))
+	if _, ok := d.GetOk("project_id"); ok {
+		projectId := strconv.Itoa(d.Get("project_id").(int))
 
-	project, err := c.GetProject(projectId)
-	if err != nil {
-		return diag.FromErr(err)
+		if _, ok := d.GetOk("name"); ok {
+			return diag.FromErr(fmt.Errorf("Both project_id and name were provided, only one is allowed"))
+		}
+
+		var err error
+		project, err = c.GetProject(projectId)
+		if err != nil {
+			return diag.FromErr(err)
+		}
+
+	} else if _, ok := d.GetOk("name"); ok {
+		projectName := d.Get("name").(string)
+
+		var err error
+		project, err = c.GetProjectByName(projectName)
+		if err != nil {
+			return diag.FromErr(err)
+		}
+
+	} else {
+		return diag.FromErr(fmt.Errorf("Either project_id or name must be provided"))
 	}
 
 	if err := d.Set("project_id", project.ID); err != nil {
@@ -88,7 +110,7 @@ func datasourceProjectRead(ctx context.Context, d *schema.ResourceData, m interf
 		return diag.FromErr(err)
 	}
 
-	d.SetId(projectId)
+	d.SetId(strconv.Itoa(*project.ID))
 
 	return diags
 }

pkg/data_sources/project_acceptance_test.go

@@ -20,6 +20,12 @@ func TestAccDbtCloudProjectDataSource(t *testing.T) {
 		resource.TestCheckResourceAttrSet("data.dbtcloud_project.test", "connection_id"),
 		resource.TestCheckResourceAttrSet("data.dbtcloud_project.test", "repository_id"),
 		resource.TestCheckResourceAttrSet("data.dbtcloud_project.test", "state"),
+
+		resource.TestCheckResourceAttrSet("data.dbtcloud_project.test_with_name", "project_id"),
+		resource.TestCheckResourceAttr("data.dbtcloud_project.test_with_name", "name", randomProjectName),
+		resource.TestCheckResourceAttrSet("data.dbtcloud_project.test_with_name", "connection_id"),
+		resource.TestCheckResourceAttrSet("data.dbtcloud_project.test_with_name", "repository_id"),
+		resource.TestCheckResourceAttrSet("data.dbtcloud_project.test_with_name", "state"),
 	)
 
 	resource.ParallelTest(t, resource.TestCase{
@@ -43,5 +49,9 @@ func project(projectName string) string {
     data "dbtcloud_project" "test" {
 		project_id = dbtcloud_project.test.id
 	}
+
+	data "dbtcloud_project" "test_with_name" {
+		name = dbtcloud_project.test.name
+	}
     `, projectName)
 }

pkg/dbt_cloud/bigquery_credential.go

@@ -7,11 +7,6 @@ import (
 	"strings"
 )
 
-type BigQueryCredentialListResponse struct {
-	Data   []BigQueryCredential `json:"data"`
-	Status ResponseStatus       `json:"status"`
-}
-
 type BigQueryCredentialResponse struct {
 	Data   BigQueryCredential `json:"data"`
 	Status ResponseStatus     `json:"status"`
@@ -28,7 +23,7 @@ type BigQueryCredential struct {
 }
 
 func (c *Client) GetBigQueryCredential(projectId int, credentialId int) (*BigQueryCredential, error) {
-	req, err := http.NewRequest("GET", fmt.Sprintf("%s/v3/accounts/%d/projects/%d/credentials/", c.HostURL, c.AccountID, projectId), nil)
+	req, err := http.NewRequest("GET", fmt.Sprintf("%s/v3/accounts/%d/projects/%d/credentials/%d/", c.HostURL, c.AccountID, projectId, credentialId), nil)
 	if err != nil {
 		return nil, err
 	}
@@ -38,19 +33,13 @@ func (c *Client) GetBigQueryCredential(projectId int, credentialId int) (*BigQue
 		return nil, err
 	}
 
-	BigQueryCredentialListResponse := BigQueryCredentialListResponse{}
-	err = json.Unmarshal(body, &BigQueryCredentialListResponse)
+	BigQueryCredentialResponse := BigQueryCredentialResponse{}
+	err = json.Unmarshal(body, &BigQueryCredentialResponse)
 	if err != nil {
 		return nil, err
 	}
 
-	for i, credential := range BigQueryCredentialListResponse.Data {
-		if *credential.ID == credentialId {
-			return &BigQueryCredentialListResponse.Data[i], nil
-		}
-	}
-
-	return nil, fmt.Errorf("resource-not-found: did not find credential ID %d in project ID %d", credentialId, projectId)
+	return &BigQueryCredentialResponse.Data, nil
 }
 
 func (c *Client) CreateBigQueryCredential(projectId int, type_ string, isActive bool, dataset string, numThreads int) (*BigQueryCredential, error) {

pkg/dbt_cloud/client.go

@@ -24,6 +24,21 @@ type ResponseStatus struct {
 	Developer_Message string `json:"developer_message"`
 }
 
+type ResponseExtraFilters struct {
+	Limit  int `json:"limit"`
+	Offset int `json:"offset"`
+}
+
+type ResponseExtraPagination struct {
+	Count      int `json:"count"`
+	TotalCount int `json:"total_count"`
+}
+
+type ResponseExtra struct {
+	Pagination ResponseExtraPagination `json:"pagination"`
+	Filters    ResponseExtraFilters    `json:"filters"`
+}
+
 type AuthResponseData struct {
 	DocsJobId                      int    `json:"docs_job_id"`
 	FreshnessJobId                 int    `json:"freshness_job_id"`

pkg/dbt_cloud/group.go

@@ -34,11 +34,6 @@ type GroupResponse struct {
 	Status ResponseStatus `json:"status"`
 }
 
-type GroupListResponse struct {
-	Data   []Group        `json:"data"`
-	Status ResponseStatus `json:"status"`
-}
-
 type GroupPermissionListResponse struct {
 	Data   []GroupPermission `json:"data"`
 	Status ResponseStatus    `json:"status"`

pkg/dbt_cloud/postgres_credential.go

@@ -20,19 +20,14 @@ type PostgresCredential struct {
 	Password       string `json:"password,omitempty"`
 }
 
-type PostgresCredentialListResponse struct {
-	Data   []PostgresCredential `json:"data"`
-	Status ResponseStatus       `json:"status"`
-}
-
 type PostgresCredentialResponse struct {
 	Data   PostgresCredential `json:"data"`
 	Status ResponseStatus     `json:"status"`
 }
 
 // GetPostgresCredential retrieves a specific Postgres credential by its ID
 func (c *Client) GetPostgresCredential(projectId int, credentialId int) (*PostgresCredential, error) {
-	req, err := http.NewRequest("GET", fmt.Sprintf("%s/v3/accounts/%d/projects/%d/credentials/", c.HostURL, c.AccountID, projectId), nil)
+	req, err := http.NewRequest("GET", fmt.Sprintf("%s/v3/accounts/%d/projects/%d/credentials/%d/", c.HostURL, c.AccountID, projectId, credentialId), nil)
 	if err != nil {
 		return nil, err
 	}
@@ -42,19 +37,13 @@ func (c *Client) GetPostgresCredential(projectId int, credentialId int) (*Postgr
 		return nil, err
 	}
 
-	PostgresCredentialListResponse := PostgresCredentialListResponse{}
-	err = json.Unmarshal(body, &PostgresCredentialListResponse)
+	PostgresCredentialResponse := PostgresCredentialResponse{}
+	err = json.Unmarshal(body, &PostgresCredentialResponse)
 	if err != nil {
 		return nil, err
 	}
 
-	for i, credential := range PostgresCredentialListResponse.Data {
-		if *credential.ID == credentialId {
-			return &PostgresCredentialListResponse.Data[i], nil
-		}
-	}
-
-	return nil, fmt.Errorf("resource-not-found: did not find credential ID %d in project ID %d", credentialId, projectId)
+	return &PostgresCredentialResponse.Data, nil
 }
 
 // CreatePostgresCredential creates a new Postgres credential

pkg/dbt_cloud/project.go

@@ -23,13 +23,84 @@ type Project struct {
 type ProjectListResponse struct {
 	Data   []Project      `json:"data"`
 	Status ResponseStatus `json:"status"`
+	Extra  ResponseExtra  `json:"extra"`
 }
 
 type ProjectResponse struct {
 	Data   Project        `json:"data"`
 	Status ResponseStatus `json:"status"`
 }
 
+func (c *Client) GetProjectByName(projectName string) (*Project, error) {
+	req, err := http.NewRequest("GET", fmt.Sprintf("%s/v3/accounts/%s/projects/?include_related=[freshness_job_id,docs_job_id]", c.HostURL, strconv.Itoa(c.AccountID)), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	body, err := c.doRequest(req)
+	if err != nil {
+		return nil, err
+	}
+
+	projectListResponse := ProjectListResponse{}
+	err = json.Unmarshal(body, &projectListResponse)
+	if err != nil {
+		return nil, err
+	}
+
+	listAllProjects := projectListResponse.Data
+
+	// if there are more than the limit, we need to paginate
+	if projectListResponse.Extra.Pagination.TotalCount > projectListResponse.Extra.Filters.Limit {
+		numProjects := projectListResponse.Extra.Pagination.Count
+		for numProjects < projectListResponse.Extra.Pagination.TotalCount {
+
+			req, err := http.NewRequest("GET", fmt.Sprintf("%s/v3/accounts/%s/projects/?include_related=[freshness_job_id,docs_job_id]&offset=%d", c.HostURL, strconv.Itoa(c.AccountID), numProjects), nil)
+			if err != nil {
+				return nil, err
+			}
+
+			body, err := c.doRequest(req)
+			if err != nil {
+				return nil, err
+			}
+
+			projectListResponse := ProjectListResponse{}
+			err = json.Unmarshal(body, &projectListResponse)
+			if err != nil {
+				return nil, err
+			}
+
+			numProjectsLastCall := projectListResponse.Extra.Pagination.Count
+			if numProjectsLastCall > 0 {
+				listAllProjects = append(listAllProjects, projectListResponse.Data...)
+				numProjects += projectListResponse.Extra.Pagination.Count
+			} else {
+				// this means that most likely one item was deleted since the first call
+				// so the number of items is less than the initial total, we can break the loop
+				break
+			}
+
+		}
+	}
+
+	// we now loop though the projects to find the ones with the name we are looking for
+	matchingProjects := []Project{}
+	for _, project := range listAllProjects {
+		if strings.EqualFold(project.Name, projectName) {
+			matchingProjects = append(matchingProjects, project)
+		}
+	}
+
+	if len(matchingProjects) == 0 {
+		return nil, fmt.Errorf("Did not find any project with the name: %s", projectName)
+	} else if len(matchingProjects) > 1 {
+		return nil, fmt.Errorf("Found more than one project with the name: %s", projectName)
+	}
+
+	return &matchingProjects[0], nil
+}
+
 func (c *Client) GetProject(projectID string) (*Project, error) {
 	req, err := http.NewRequest("GET", fmt.Sprintf("%s/v3/accounts/%s/projects/%s/?include_related=[freshness_job_id,docs_job_id]", c.HostURL, strconv.Itoa(c.AccountID), projectID), nil)
 	if err != nil {

pkg/dbt_cloud/snowflake_credential.go

@@ -7,11 +7,6 @@ import (
 	"strings"
 )
 
-type SnowflakeCredentialListResponse struct {
-	Data   []SnowflakeCredential `json:"data"`
-	Status ResponseStatus        `json:"status"`
-}
-
 type SnowflakeCredentialResponse struct {
 	Data   SnowflakeCredential `json:"data"`
 	Status ResponseStatus      `json:"status"`
@@ -36,7 +31,7 @@ type SnowflakeCredential struct {
 }
 
 func (c *Client) GetSnowflakeCredential(projectId int, credentialId int) (*SnowflakeCredential, error) {
-	req, err := http.NewRequest("GET", fmt.Sprintf("%s/v3/accounts/%d/projects/%d/credentials/", c.HostURL, c.AccountID, projectId), nil)
+	req, err := http.NewRequest("GET", fmt.Sprintf("%s/v3/accounts/%d/projects/%d/credentials/%d/", c.HostURL, c.AccountID, projectId, credentialId), nil)
 	if err != nil {
 		return nil, err
 	}
@@ -46,19 +41,13 @@ func (c *Client) GetSnowflakeCredential(projectId int, credentialId int) (*Snowf
 		return nil, err
 	}
 
-	snowflakeCredentialListResponse := SnowflakeCredentialListResponse{}
-	err = json.Unmarshal(body, &snowflakeCredentialListResponse)
+	snowflakeCredentialResponse := SnowflakeCredentialResponse{}
+	err = json.Unmarshal(body, &snowflakeCredentialResponse)
 	if err != nil {
 		return nil, err
 	}
 
-	for i, credential := range snowflakeCredentialListResponse.Data {
-		if *credential.ID == credentialId {
-			return &snowflakeCredentialListResponse.Data[i], nil
-		}
-	}
-
-	return nil, fmt.Errorf("resource-not-found: did not find credential ID %d in project ID %d", credentialId, projectId)
+	return &snowflakeCredentialResponse.Data, nil
 }
 
 func (c *Client) CreateSnowflakeCredential(projectId int, type_ string, isActive bool, database string, role string, warehouse string, schema string, user string, password string, privateKey string, privateKeyPassphrase string, authType string, numThreads int) (*SnowflakeCredential, error) {