Merge pull request #623 from GoogleCloudPlatform/lcaggio/sql-kms

Add KMS on CloudSQL module
2022-04-13 10:38:39 +02:00 · 2022-04-13 10:38:39 +02:00 · 565e3d99fc
parent eec0fd2fdf 4c4c879cce
commit 565e3d99fc
7 changed files with 113 additions and 35 deletions
--- a/modules/cloudsql-instance/README.md
+++ b/modules/cloudsql-instance/README.md
@ -56,8 +56,8 @@ module "db" {
  tier             = "db-g1-small"

  replicas = {
-    replica1 = "europe-west3"
-    replica2 = "us-central1"
+    replica1 = { region = "europe-west3", encryption_key_name = null }
+    replica2 = { region = "us-central1", encryption_key_name = null }
  }
 }
 # tftest modules=1 resources=3
@ -93,6 +93,53 @@ module "db" {
 }
 # tftest modules=1 resources=6
 ```
+
+### CMEK encryption
+```hcl
+
+module "project" {
+  source          = "./modules/project"
+  billing_account = var.billing_account_id
+  parent          = var.organization_id
+  name            = "my-db-project"
+  services = [
+    "servicenetworking.googleapis.com",
+    "sqladmin.googleapis.com",
+  ]
+}
+
+module "kms" {
+  source     = "./modules/kms"
+  project_id = module.project.project_id
+  keyring = {
+    name     = "keyring"
+    location = var.region
+  }
+  keys = {
+    key-sql = null
+  }
+  key_iam = {
+    key-sql = {
+      "roles/cloudkms.cryptoKeyEncrypterDecrypter" = [
+        "serviceAccount:${module.project.service_accounts.robots.sqladmin}"
+      ]
+    }
+  }
+}
+
+module "db" {
+  source              = "./modules/cloudsql-instance"
+  project_id          = module.project.project_id
+  encryption_key_name = module.kms.keys["key-sql"].id
+  network             = var.vpc.self_link
+  name                = "db"
+  region              = var.region
+  database_version    = "POSTGRES_13"
+  tier                = "db-g1-small"
+}
+
+# tftest modules=3 resources=10
+```
 <!-- BEGIN TFDOC -->

 ## Variables
@ -100,11 +147,11 @@ module "db" {
 | name | description | type | required | default |
 |---|---|:---:|:---:|:---:|
 | [database_version](variables.tf#L50) | Database type and version to create. | <code>string</code> | ✓ |  |
-| [name](variables.tf#L91) | Name of primary replica. | <code>string</code> | ✓ |  |
-| [network](variables.tf#L96) | VPC self link where the instances will be deployed. Private Service Networking must be enabled and configured in this VPC. | <code>string</code> | ✓ |  |
-| [project_id](variables.tf#L107) | The ID of the project where this instances will be created. | <code>string</code> | ✓ |  |
-| [region](variables.tf#L112) | Region of the primary replica. | <code>string</code> | ✓ |  |
-| [tier](variables.tf#L123) | The machine type to use for the instances. | <code>string</code> | ✓ |  |
+| [name](variables.tf#L97) | Name of primary instance. | <code>string</code> | ✓ |  |
+| [network](variables.tf#L102) | VPC self link where the instances will be deployed. Private Service Networking must be enabled and configured in this VPC. | <code>string</code> | ✓ |  |
+| [project_id](variables.tf#L113) | The ID of the project where this instances will be created. | <code>string</code> | ✓ |  |
+| [region](variables.tf#L118) | Region of the primary instance. | <code>string</code> | ✓ |  |
+| [tier](variables.tf#L132) | The machine type to use for the instances. | <code>string</code> | ✓ |  |
 | [authorized_networks](variables.tf#L17) | Map of NAME=>CIDR_RANGE to allow to connect to the database(s). | <code>map&#40;string&#41;</code> |  | <code>null</code> |
 | [availability_type](variables.tf#L23) | Availability type for the primary replica. Either `ZONAL` or `REGIONAL`. | <code>string</code> |  | <code>&#34;ZONAL&#34;</code> |
 | [backup_configuration](variables.tf#L29) | Backup settings for primary instance. Will be automatically enabled if using MySQL with one or more replicas. | <code title="object&#40;&#123;&#10;  enabled            &#61; bool&#10;  binary_log_enabled &#61; bool&#10;  start_time         &#61; string&#10;  location           &#61; string&#10;  log_retention_days &#61; number&#10;  retention_count    &#61; number&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code title="&#123;&#10;  enabled            &#61; false&#10;  binary_log_enabled &#61; false&#10;  start_time         &#61; &#34;23:00&#34;&#10;  location           &#61; null&#10;  log_retention_days &#61; 7&#10;  retention_count    &#61; 7&#10;&#125;">&#123;&#8230;&#125;</code> |
@ -112,11 +159,12 @@ module "db" {
 | [deletion_protection](variables.tf#L61) | Allow terraform to delete instances. | <code>bool</code> |  | <code>false</code> |
 | [disk_size](variables.tf#L67) | Disk size in GB. Set to null to enable autoresize. | <code>number</code> |  | <code>null</code> |
 | [disk_type](variables.tf#L73) | The type of data disk: `PD_SSD` or `PD_HDD`. | <code>string</code> |  | <code>&#34;PD_SSD&#34;</code> |
-| [flags](variables.tf#L79) | Map FLAG_NAME=>VALUE for database-specific tuning. | <code>map&#40;string&#41;</code> |  | <code>null</code> |
-| [labels](variables.tf#L85) | Labels to be attached to all instances. | <code>map&#40;string&#41;</code> |  | <code>null</code> |
-| [prefix](variables.tf#L101) | Prefix used to generate instance names. | <code>string</code> |  | <code>null</code> |
-| [replicas](variables.tf#L117) | Map of NAME=>REGION for additional read replicas. Set to null to disable replica creation. | <code>map&#40;any&#41;</code> |  | <code>null</code> |
-| [users](variables.tf#L128) | Map of users to create in the primary instance (and replicated to other replicas) in the format USER=>PASSWORD. For MySQL, anything afterr the first `@` (if persent) will be used as the user's host. Set PASSWORD to null if you want to get an autogenerated password. | <code>map&#40;string&#41;</code> |  | <code>null</code> |
+| [encryption_key_name](variables.tf#L79) | The full path to the encryption key used for the CMEK disk encryption of the primary instance. | <code>string</code> |  | <code>null</code> |
+| [flags](variables.tf#L85) | Map FLAG_NAME=>VALUE for database-specific tuning. | <code>map&#40;string&#41;</code> |  | <code>null</code> |
+| [labels](variables.tf#L91) | Labels to be attached to all instances. | <code>map&#40;string&#41;</code> |  | <code>null</code> |
+| [prefix](variables.tf#L107) | Prefix used to generate instance names. | <code>string</code> |  | <code>null</code> |
+| [replicas](variables.tf#L123) | Map of NAME=> {REGION, KMS_KEY} for additional read replicas. Set to null to disable replica creation. | <code title="map&#40;object&#40;&#123;&#10;  region              &#61; string&#10;  encryption_key_name &#61; string&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
+| [users](variables.tf#L137) | Map of users to create in the primary instance (and replicated to other replicas) in the format USER=>PASSWORD. For MySQL, anything afterr the first `@` (if persent) will be used as the user's host. Set PASSWORD to null if you want to get an autogenerated password. | <code>map&#40;string&#41;</code> |  | <code>null</code> |

 ## Outputs

@ -129,8 +177,10 @@ module "db" {
 | [instances](outputs.tf#L50) | Cloud SQL instance resources. | ✓ |
 | [ip](outputs.tf#L56) | IP address of the primary instance. |  |
 | [ips](outputs.tf#L61) | IP addresses of all instances. |  |
-| [self_link](outputs.tf#L69) | Self link of the primary instance. |  |
-| [self_links](outputs.tf#L74) | Self links of all instances. |  |
-| [user_passwords](outputs.tf#L82) | Map of containing the password of all users created through terraform. | ✓ |
+| [name](outputs.tf#L69) | Name of the primary instance. |  |
+| [names](outputs.tf#L74) | Names of all instances. |  |
+| [self_link](outputs.tf#L82) | Self link of the primary instance. |  |
+| [self_links](outputs.tf#L87) | Self links of all instances. |  |
+| [user_passwords](outputs.tf#L95) | Map of containing the password of all users created through terraform. | ✓ |

 <!-- END TFDOC -->
--- a/modules/cloudsql-instance/main.tf
+++ b/modules/cloudsql-instance/main.tf
@ -39,10 +39,12 @@ locals {
 }

 resource "google_sql_database_instance" "primary" {
-  project          = var.project_id
-  name             = "${local.prefix}${var.name}"
-  region           = var.region
-  database_version = var.database_version
+  provider            = google-beta
+  project             = var.project_id
+  name                = "${local.prefix}${var.name}"
+  region              = var.region
+  database_version    = var.database_version
+  encryption_key_name = var.encryption_key_name

  settings {
    tier              = var.tier
@ -99,11 +101,13 @@ resource "google_sql_database_instance" "primary" {
 }

 resource "google_sql_database_instance" "replicas" {
+  provider             = google-beta
  for_each             = local.has_replicas ? var.replicas : {}
  project              = var.project_id
  name                 = "${local.prefix}${each.key}"
-  region               = each.value
+  region               = each.value.region
  database_version     = var.database_version
+  encryption_key_name  = each.value.encryption_key_name
  master_instance_name = google_sql_database_instance.primary.name

  settings {
--- a/modules/cloudsql-instance/outputs.tf
+++ b/modules/cloudsql-instance/outputs.tf
@ -66,6 +66,19 @@ output "ips" {
  }
 }

+output "name" {
+  description = "Name of the primary instance."
+  value       = google_sql_database_instance.primary.name
+}
+
+output "names" {
+  description = "Names of all instances."
+  value = {
+    for id, instance in local._all_intances :
+    id => instance.name
+  }
+}
+
 output "self_link" {
  description = "Self link of the primary instance."
  value       = google_sql_database_instance.primary.self_link
--- a/modules/cloudsql-instance/variables.tf
+++ b/modules/cloudsql-instance/variables.tf
@ -76,6 +76,12 @@ variable "disk_type" {
  default     = "PD_SSD"
 }

+variable "encryption_key_name" {
+  description = "The full path to the encryption key used for the CMEK disk encryption of the primary instance."
+  type        = string
+  default     = null
+}
+
 variable "flags" {
  description = "Map FLAG_NAME=>VALUE for database-specific tuning."
  type        = map(string)
@ -89,7 +95,7 @@ variable "labels" {
 }

 variable "name" {
-  description = "Name of primary replica."
+  description = "Name of primary instance."
  type        = string
 }

@ -110,14 +116,17 @@ variable "project_id" {
 }

 variable "region" {
-  description = "Region of the primary replica."
+  description = "Region of the primary instance."
  type        = string
 }

 variable "replicas" {
-  description = "Map of NAME=>REGION for additional read replicas. Set to null to disable replica creation."
-  type        = map(any)
-  default     = null
+  description = "Map of NAME=> {REGION, KMS_KEY} for additional read replicas. Set to null to disable replica creation."
+  type = map(object({
+    region              = string
+    encryption_key_name = string
+  }))
+  default = {}
 }

 variable "tier" {
--- a/modules/project/service-accounts.tf
+++ b/modules/project/service-accounts.tf
@ -42,7 +42,9 @@ locals {
    gcf           = "service-%s@gcf-admin-robot"
    pubsub        = "service-%s@gcp-sa-pubsub"
    secretmanager = "service-%s@gcp-sa-secretmanager"
+    sql           = "service-%s@gcp-sa-cloud-sql"
    storage       = "service-%s@gs-project-accounts"
+    sqladmin      = "service-%s@gcp-sa-cloud-sql"
  }
  service_accounts_default = {
    compute = "${local.project.number}-compute@developer.gserviceaccount.com"
@ -56,9 +58,10 @@ locals {
    k => "${format(v, local.project.number)}.iam.gserviceaccount.com"
  }
  service_accounts_jit_services = [
-    "secretmanager.googleapis.com",
+    "cloudasset.googleapis.com",
    "pubsub.googleapis.com",
-    "cloudasset.googleapis.com"
+    "secretmanager.googleapis.com",
+    "sqladmin.googleapis.com"
  ]
  service_accounts_cmek_service_keys = distinct(flatten([
    for s in keys(var.service_encryption_key_ids) : [
--- a/tests/modules/cloudsql_instance/fixture/variables.tf
+++ b/tests/modules/cloudsql_instance/fixture/variables.tf
@ -94,7 +94,7 @@ variable "region" {
 }

 variable "replicas" {
-  type    = map(any)
+  type    = any
  default = null
 }

--- a/tests/modules/cloudsql_instance/test_plan.py
+++ b/tests/modules/cloudsql_instance/test_plan.py
@ -35,8 +35,8 @@ def test_prefix(plan_runner):
  assert r['values']['name'] == 'prefix-db'

  replicas = """{
-    replica1 = "europe-west3"
-    replica2 = "us-central1"
+    replica1 = { region = "europe-west3", encryption_key_name = null }
+    replica2 = { region = "us-central1", encryption_key_name = null }
  }"""

  _, resources = plan_runner(prefix="prefix")
@ -49,8 +49,8 @@ def test_replicas(plan_runner):
  "Test replicated instance."

  replicas = """{
-    replica1 = "europe-west3"
-    replica2 = "us-central1"
+    replica1 = { region = "europe-west3", encryption_key_name = null }
+    replica2 = { region = "us-central1", encryption_key_name = null }
  }"""

  _, resources = plan_runner(replicas=replicas, prefix="prefix")
@ -80,10 +80,9 @@ def test_mysql_replicas_enables_backup(plan_runner):
  "Test MySQL backup setup with replicas."

  replicas = """{
-    replica1 = "europe-west3"
+    replica1 = { region = "europe-west3", encryption_key_name = null }
  }"""
-  _, resources = plan_runner(replicas=replicas,
-                             database_version="MYSQL_8_0")
+  _, resources = plan_runner(replicas=replicas, database_version="MYSQL_8_0")
  assert len(resources) == 2
  primary = [r for r in resources if r['name'] == 'primary'][0]
  backup_config = primary['values']['settings'][0]['backup_configuration'][0]