diff --git a/README.md b/README.md index ecc562c..01f233f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # Terraform GKE -This repository contains the standards for GKE cluster implementations, and is a work in progress. The GKE cluster module should be adaptable. Ideally, it should allow for public "standard" GKE clusters, public/VPC-native clusters, and private/VPC-native clusters. This module will only contain GKE-related terraform resources. Underlying network resources will be created in separate modules in separate repositories, which will have to be used in conjunction with this module/repository to create a complete infrastructure. [add links to these network module repos as they are created] +This repository contains the standards for GKE cluster implementations, and is a work in progress. The GKE cluster module should be adaptable. Ideally, it should allow for public/VPC-native and private/VPC-native clusters. This module will only contain GKE-related terraform resources. Underlying network resources will be created in separate modules in separate repositories, which will have to be used in conjunction with this module/repository to create a complete infrastructure. See the [terraform-gcp-vpc-native][] repository. ## Usage @@ -32,3 +32,6 @@ We do not anticipate retrofitting patches to older MINOR versions. If we are cur * Typos could be fixed with patch if it affects behavior of the terraform module * Fixes to older supported features of the module that broke with MINOR functionality changes * README and USAGE documentation changes may trigger a PATCH change and should be documented in CHANGELOG + + +[terraform-gcp-vpc-native]: https://github.com/reactiveops/terraform-gcp-vpc-native diff --git a/gke-public/README.md b/gke-public/README.md deleted file mode 100644 index e69de29..0000000 diff --git a/gke-vpc-native/README.md b/gke-vpc-native/README.md deleted file mode 100644 index e69de29..0000000 diff --git a/gke-private/CHANGELOG.md b/node_pool/CHANGELOG.md similarity index 100% rename from gke-private/CHANGELOG.md rename to node_pool/CHANGELOG.md diff --git a/node_pool/Makefile b/node_pool/Makefile new file mode 100644 index 0000000..0205a68 --- /dev/null +++ b/node_pool/Makefile @@ -0,0 +1,15 @@ +# This constructs README.md, including Terraform documentation in its middle. +# THIs requires `terraform-docs` to be installed from: https://github.com/segmentio/terraform-docs + +# The sed command below increases the markdown heading level of Terraform docs. +README.md:README.pre_terraform_inputs.md README.post_terraform_inputs.md + @echo Creating ReadMe with Terraform docs. . . + cat README.pre_tf_inputs.md >README.md + echo >>README.md + terraform-docs --with-aggregate-type-defaults md document . |sed 's/^#/##/g' >>README.md + echo >>README.md + cat README.post_tf_inputs.md >>README.md + +README.pre_terraform_inputs.md: +README.post_terraform_inputs.md: + diff --git a/node_pool/README.md b/node_pool/README.md new file mode 100644 index 0000000..fab4d13 --- /dev/null +++ b/node_pool/README.md @@ -0,0 +1,78 @@ +# Terraform GKE Node Pool Module + +This module manages a node pool attached to a Google Kubernetes Engine (GKE) cluster. + +## Using The Terraform Module + +This module requires version 2.0.0 or above of the Google Terraform provider. + +See the file [example-usage](./example-usage) for an example of how to use this module. Below are the available module inputs: + +### Required Inputs + +The following input variables are required: + +#### gke\_cluster\_name + +Description: The name of the GKE cluster to bind this node pool. + +Type: `string` + +#### kubernetes\_version + +Description: The kubernetes version for the nodes in the pool. This should match the Kubernetes version of the GKE cluster. + +Type: `string` + +#### max\_node\_count + +Description: Maximum number of nodes for autoscaling, per availability zone. + +Type: `string` + +#### min\_node\_count + +Description: Minimum number of nodes for autoscaling, per availability zone. + +Type: `string` + +#### name + +Description: The name of the node pool. A random string will be appended to this name, to allow replacement node pools to be created before destroying the current pool. + +Type: `string` + +#### region + +Description: The region for the node pool. + +Type: `string` + +### Optional Inputs + +The following input variables are optional (have default values): + +#### disk\_size\_in\_gb + +Description: Disk size, in GB, for the nodes in the pool. + +Type: `string` + +Default: `"100"` + +#### machine\_type + +Description: The machine type of nodes in the pool. + +Type: `string` + +Default: `"n1-standard-4"` + + +## Future To-do Items + +* Perform additional testing of conditions that will cause Terraform to recreate the node pool, based on the `keeper`s defined in the `random_id` resource. +* Determine whether, depending on results from the above testing, the node pool should have a `create_before_destroy` lifecycle. +* Do we want to be able to enable auto-upgrade on node pools? +* Do we want to be able to set taints on node pools? +* Do we want to enable auto-repair always on node pools? \ No newline at end of file diff --git a/node_pool/README.post_tf_inputs.md b/node_pool/README.post_tf_inputs.md new file mode 100644 index 0000000..339ad74 --- /dev/null +++ b/node_pool/README.post_tf_inputs.md @@ -0,0 +1,7 @@ +## Future To-do Items + +* Perform additional testing of conditions that will cause Terraform to recreate the node pool, based on the `keeper`s defined in the `random_id` resource. +* Determine whether, depending on results from the above testing, the node pool should have a `create_before_destroy` lifecycle. +* Do we want to be able to enable auto-upgrade on node pools? +* Do we want to be able to set taints on node pools? +* Do we want to enable auto-repair always on node pools? \ No newline at end of file diff --git a/node_pool/README.pre_tf_inputs.md b/node_pool/README.pre_tf_inputs.md new file mode 100644 index 0000000..e22d739 --- /dev/null +++ b/node_pool/README.pre_tf_inputs.md @@ -0,0 +1,9 @@ +# Terraform GKE Node Pool Module + +This module manages a node pool attached to a Google Kubernetes Engine (GKE) cluster. + +## Using The Terraform Module + +This module requires version 2.0.0 or above of the Google Terraform provider. + +See the file [example-usage](./example-usage) for an example of how to use this module. Below are the available module inputs: diff --git a/node_pool/example-usage b/node_pool/example-usage new file mode 100644 index 0000000..3ef1ee0 --- /dev/null +++ b/node_pool/example-usage @@ -0,0 +1,16 @@ +# In the below example, `module.customername_cluster` is an instance of +# one of the GKE modules from this repository. + +module "customername_cluster_node_pool" { + source = "git@github.com:/reactiveops/terraform-gke//node_pool" + + name = "node-pool-1" + region = "${module.customername_cluster.region}" + gke_cluster_name = "${module.customername_cluster.name}" + machine_type = "n1-standard-2" + min_node_count = "1" + max_node_count = "1" + + # Match the Kubernetes version from the GKE cluster! + kubernetes_version = "${module.customername_cluster.kubernetes_version}" +} diff --git a/node_pool/inputs.tf b/node_pool/inputs.tf new file mode 100644 index 0000000..e093123 --- /dev/null +++ b/node_pool/inputs.tf @@ -0,0 +1,33 @@ +variable "name" { + description = "The name of the node pool. A random string will be appended to this name, to allow replacement node pools to be created before destroying the current pool." +} + +variable "gke_cluster_name" { + description = "The name of the GKE cluster to bind this node pool." +} + +variable "region" { + description = "The region for the node pool." +} + +variable "min_node_count" { + description = "Minimum number of nodes for autoscaling, per availability zone." +} + +variable "max_node_count" { + description = "Maximum number of nodes for autoscaling, per availability zone." +} + +variable "kubernetes_version" { + description = "The kubernetes version for the nodes in the pool. This should match the Kubernetes version of the GKE cluster." +} + +variable "machine_type" { + description = "The machine type of nodes in the pool." + default = "n1-standard-4" +} + +variable "disk_size_in_gb" { + description = "Disk size, in GB, for the nodes in the pool." + default = "100" +} diff --git a/node_pool/main.tf b/node_pool/main.tf new file mode 100644 index 0000000..3c31156 --- /dev/null +++ b/node_pool/main.tf @@ -0,0 +1,54 @@ +# IMPORTANT +## Please BE CAREFUL when changing `keepers`. If you add a keeper +## then this will cause a CREATION CASCADE! Since a new keeper +## triggers a new `random_id` resource to generate then it will change +## the name of the node pool, which will recreate that resource! +## This is the preferred method for random naming now that +## name-prefix is being deprecated! +## IF YOU ADD A KEEPER YOU WILL NEED TO WARN DOWNSTREAM USERS THAT +## IT WILL TRIGGER A CASCADING NODEPOOL REPLACEMENT. +## The replacement should be safe because of the create_before_destroy +## but will cause some questions and confusion. +## All the current keepers are bound to params that will already cause +## a -/+ operation for the nodepool. +# https://github.com/terraform-providers/terraform-provider-google/issues/1054 +resource "random_id" "entropy" { + keepers { + machine_type = "${var.machine_type}" + name = "${var.name}" + region = "${var.region}" + disk_size = "${var.disk_size_in_gb}" + } + + byte_length = 2 +} + +resource "google_container_node_pool" "node_pool" { + name = "${var.name}-${random_id.entropy.hex}" + cluster = "${var.gke_cluster_name}" + region = "${var.region}" + version = "${var.kubernetes_version}" + initial_node_count = 1 + + autoscaling { + min_node_count = "${var.min_node_count}" + max_node_count = "${var.max_node_count}" + } + + node_config { + image_type = "COS" + disk_size_gb = "${var.disk_size_in_gb}" + machine_type = "${var.machine_type}" + + oauth_scopes = [ + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + "https://www.googleapis.com/auth/compute", + ] + } + + lifecycle { + create_before_destroy = true + } +} diff --git a/node_pool/outputs.tf b/node_pool/outputs.tf new file mode 100644 index 0000000..eadce30 --- /dev/null +++ b/node_pool/outputs.tf @@ -0,0 +1,2 @@ +# No known outputs needed + diff --git a/gke-public/CHANGELOG.md b/private-vpc-native/CHANGELOG.md similarity index 100% rename from gke-public/CHANGELOG.md rename to private-vpc-native/CHANGELOG.md diff --git a/gke-private/README.md b/private-vpc-native/README.md similarity index 100% rename from gke-private/README.md rename to private-vpc-native/README.md diff --git a/gke-vpc-native/CHANGELOG.md b/public-vpc-native/CHANGELOG.md similarity index 100% rename from gke-vpc-native/CHANGELOG.md rename to public-vpc-native/CHANGELOG.md diff --git a/public-vpc-native/Makefile b/public-vpc-native/Makefile new file mode 100644 index 0000000..0205a68 --- /dev/null +++ b/public-vpc-native/Makefile @@ -0,0 +1,15 @@ +# This constructs README.md, including Terraform documentation in its middle. +# THIs requires `terraform-docs` to be installed from: https://github.com/segmentio/terraform-docs + +# The sed command below increases the markdown heading level of Terraform docs. +README.md:README.pre_terraform_inputs.md README.post_terraform_inputs.md + @echo Creating ReadMe with Terraform docs. . . + cat README.pre_tf_inputs.md >README.md + echo >>README.md + terraform-docs --with-aggregate-type-defaults md document . |sed 's/^#/##/g' >>README.md + echo >>README.md + cat README.post_tf_inputs.md >>README.md + +README.pre_terraform_inputs.md: +README.post_terraform_inputs.md: + diff --git a/public-vpc-native/README.md b/public-vpc-native/README.md new file mode 100644 index 0000000..3f192b4 --- /dev/null +++ b/public-vpc-native/README.md @@ -0,0 +1,121 @@ +# Terraform Public VPC Native GKE Cluster Module + +This module manages a public Google Kubernetes Engine (GKE) VPC Native cluster. The subnet CIDRs used for cluster nodes, pods, and services, are specified in the form of existing Google Compute secondary IP ranges. Use a separate Terraform module, such as [this `terraform-gcp-vpc-native` one](https://github.com/reactiveops/terraform-gcp-vpc-native), to create these network resources in advance. + +## Using The Terraform Module + +This module requires version 2.0.0 or above of the Google Terraform provider. + +See the file [example-usage](./example-usage) for an example of how to use this module. Below are the available module inputs: + +### Required Inputs + +The following input variables are required: + +#### kubernetes\_version + +Description: The minimum version of master nodes. This can be changed to upgrade the cluster - remember to upgrade the Kubernetes version for node pools (managed separately). + +Type: `string` + +#### name + +Description: The name of the GKE cluster + +Type: `string` + +#### network\_name + +Description: The name of an existing google_compute_network resource to which the cluster will be connected. + +Type: `string` + +#### nodes\_subnetwork\_name + +Description: The name of an existing google_compute_subnetwork resource where cluster compute instances are launched. + +Type: `string` + +#### pods\_secondary\_ip\_range\_name + +Description: The name of an existing network secondary IP range to be used for pods. + +Type: `string` + +#### region + +Description: The region where the GKE cluster will be created. + +Type: `string` + +#### services\_secondary\_ip\_range\_name + +Description: The name of an existing network secondary IP range to be used for services. + +Type: `string` + +### Optional Inputs + +The following input variables are optional (have default values): + +#### maintenance\_policy\_start\_time + +Description: The time (in GMT) when the cluster maintenance window will start. + +Type: `string` + +Default: `"06:00"` + +#### master\_authorized\_network\_cidrs + +Description: A list of up to 20 maps containing `master_authorized_network_cidrs` and `display_name` keys, representing source network CIDRs that are allowed to connect master nodes over HTTPS. + +Type: `list` + +Default: + +```json +[ + { + "cidr_block": "0.0.0.0/0", + "display_name": "everywhere" + } +] +``` + +#### project + +Description: The project where the GKE cluster will be created. Leave unspecified to use the project from the provider. + +Type: `string` + +Default: `""` + +### Outputs + +The following outputs are exported: + +#### endpoint + +Description: The GKE Cluster Endpoints IP + +#### kubernetes\_version + +Description: The Kubernetes version used when creating or upgrading this cluster. This does not reflect the current version of master or worker nodes. + +#### master\_version + +Description: The current version of the Kubernetes master nodes, which will differ from the kubernetes_version output if GKE upgrades masters automatically. + +#### name + +Description: The static name of the GKE cluster + +#### region + +Description: The region in which this cluster exists + + +## Future To-do Items + +* Add inputs related to private VPC Native clusters, that default to false. These can be supplied by the private cluster Terraform module, which will call **this** module. diff --git a/public-vpc-native/README.post_tf_inputs.md b/public-vpc-native/README.post_tf_inputs.md new file mode 100644 index 0000000..7031a2b --- /dev/null +++ b/public-vpc-native/README.post_tf_inputs.md @@ -0,0 +1,3 @@ +## Future To-do Items + +* Add inputs related to private VPC Native clusters, that default to false. These can be supplied by the private cluster Terraform module, which will call **this** module. diff --git a/public-vpc-native/README.pre_tf_inputs.md b/public-vpc-native/README.pre_tf_inputs.md new file mode 100644 index 0000000..d825d4e --- /dev/null +++ b/public-vpc-native/README.pre_tf_inputs.md @@ -0,0 +1,9 @@ +# Terraform Public VPC Native GKE Cluster Module + +This module manages a public Google Kubernetes Engine (GKE) VPC Native cluster. The subnet CIDRs used for cluster nodes, pods, and services, are specified in the form of existing Google Compute secondary IP ranges. Use a separate Terraform module, such as [this `terraform-gcp-vpc-native` one](https://github.com/reactiveops/terraform-gcp-vpc-native), to create these network resources in advance. + +## Using The Terraform Module + +This module requires version 2.0.0 or above of the Google Terraform provider. + +See the file [example-usage](./example-usage) for an example of how to use this module. Below are the available module inputs: diff --git a/public-vpc-native/example-usage b/public-vpc-native/example-usage new file mode 100644 index 0000000..a4f7f6c --- /dev/null +++ b/public-vpc-native/example-usage @@ -0,0 +1,31 @@ +# These local variables can be used as inputs to both a network and this GKE VPC Native cluster module. +locals { + region = "us-central1" + network_name = "customername-test" + kubernetes_version = "1.11.6-gke.6" +} + + +# The `module.customername_vpc` below refers to an instance of a VPC module. +# Ref: https://github.com/reactiveops/terraform-gcp-vpc-native +module "customername_cluster" { + # Change the ref below to use a vX.Y.Z release instead of master. + source = "git@github.com:/reactiveops/terraform-gke//public-vpc-native?ref=master" + + name = "customername-cluster1" + region = "${local.region}" + project = "customername-dev" + kubernetes_version = "${local.kubernetes_version}" + network_name = "${local.network_name}" + nodes_subnetwork_name = "${module.customername_vpc.subnetwork}" + pods_secondary_ip_range_name = "${module.customername_vpc.gke_pods_1}" + services_secondary_ip_range_name = "${module.customername_vpc.gke_services_1}" + + master_authorized_network_cidrs = [ + { + # This is the module default, but demonstrates specifying this input. + cidr_block = "0.0.0.0/0" + display_name = "from the Internet" + }, + ] +} diff --git a/public-vpc-native/inputs.tf b/public-vpc-native/inputs.tf new file mode 100644 index 0000000..879b945 --- /dev/null +++ b/public-vpc-native/inputs.tf @@ -0,0 +1,49 @@ +variable "name" { + description = "The name of the GKE cluster" +} + +variable "region" { + description = "The region where the GKE cluster will be created." +} + +variable "project" { + description = "The project where the GKE cluster will be created. Leave unspecified to use the project from the provider." + default = "" +} + +variable "network_name" { + description = "The name of an existing google_compute_network resource to which the cluster will be connected." +} + +variable "nodes_subnetwork_name" { + description = "The name of an existing google_compute_subnetwork resource where cluster compute instances are launched." +} + +variable "kubernetes_version" { + description = "The minimum version of master nodes. This can be changed to upgrade the cluster - remember to upgrade the Kubernetes version for node pools (managed separately)." +} + +variable "pods_secondary_ip_range_name" { + description = "The name of an existing network secondary IP range to be used for pods." +} + +variable "services_secondary_ip_range_name" { + description = "The name of an existing network secondary IP range to be used for services." +} + +variable "master_authorized_network_cidrs" { + type = "list" + description = "A list of up to 20 maps containing `master_authorized_network_cidrs` and `display_name` keys, representing source network CIDRs that are allowed to connect master nodes over HTTPS." + + default = [ + { + cidr_block = "0.0.0.0/0" + display_name = "everywhere" + }, + ] +} + +variable "maintenance_policy_start_time" { + description = "The time (in GMT) when the cluster maintenance window will start." + default = "06:00" +} diff --git a/public-vpc-native/main.tf b/public-vpc-native/main.tf new file mode 100644 index 0000000..a940497 --- /dev/null +++ b/public-vpc-native/main.tf @@ -0,0 +1,60 @@ +resource "google_container_cluster" "cluster" { + name = "${var.name}" + region = "${var.region}" + min_master_version = "${var.kubernetes_version}" + network = "${var.network_name}" + subnetwork = "${var.nodes_subnetwork_name}" + + ip_allocation_policy { + cluster_secondary_range_name = "${var.pods_secondary_ip_range_name}" + services_secondary_range_name = "${var.services_secondary_ip_range_name}" + } + + # This is believed to apply to the default node pool, which gets created then deleted. + initial_node_count = 1 + remove_default_node_pool = true + + # The absence of a user and password here disables basic auth + master_auth { + username = "" + password = "" + + client_certificate_config { + issue_client_certificate = false + } + } + + # Disable automatic kubernetes dashboard + addons_config { + kubernetes_dashboard { + disabled = true + } + } + + network_policy { + enabled = true + } + + master_authorized_networks_config = { + cidr_blocks = "${var.master_authorized_network_cidrs}" + } + + maintenance_policy { + daily_maintenance_window { + start_time = "${var.maintenance_policy_start_time}" + } + } + + resource_labels { + kubernetescluster = "${var.name}" + } + + lifecycle { + # ignore changes to node_pool specifically so it doesn't + # try to recreate default node pool with every change + # ignore changes to network and subnetwork so it doesn't + # clutter up diff with dumb changes like: + # projects/[name]/regions/us-central1/subnetworks/[name]" => "name" + ignore_changes = ["node_pool", "network", "subnetwork"] + } +} diff --git a/public-vpc-native/outputs.tf b/public-vpc-native/outputs.tf new file mode 100644 index 0000000..f277655 --- /dev/null +++ b/public-vpc-native/outputs.tf @@ -0,0 +1,25 @@ +output "name" { + description = "The static name of the GKE cluster" + value = "${google_container_cluster.cluster.name}" +} + +output "endpoint" { + description = "The GKE Cluster Endpoints IP" + value = "${google_container_cluster.cluster.endpoint}" +} + +## This is passed back out in case it's needed to inherit for node pools +output "kubernetes_version" { + description = "The Kubernetes version used when creating or upgrading this cluster. This does not reflect the current version of master or worker nodes." + value = "${var.kubernetes_version}" +} + +output "master_version" { + description = "The current version of the Kubernetes master nodes, which will differ from the kubernetes_version output if GKE upgrades masters automatically." + value = "${google_container_cluster.cluster.master_version}" +} + +output "region" { + description = "The region in which this cluster exists" + value = "${var.region}" +}