From c7c77541fbcb3a405ec785f545d935b87f69f089 Mon Sep 17 00:00:00 2001 From: Lorenzo Caggioni Date: Tue, 12 Apr 2022 19:01:34 +0200 Subject: [PATCH 1/3] Add KMS on CloudSQL module --- modules/cloudsql-instance/README.md | 53 ++++++++++++++++++++++++++ modules/cloudsql-instance/main.tf | 16 +++++--- modules/cloudsql-instance/outputs.tf | 13 +++++++ modules/cloudsql-instance/variables.tf | 15 ++++++-- modules/project/service-accounts.tf | 6 ++- 5 files changed, 92 insertions(+), 11 deletions(-) diff --git a/modules/cloudsql-instance/README.md b/modules/cloudsql-instance/README.md index 213398d7..d5c050c8 100644 --- a/modules/cloudsql-instance/README.md +++ b/modules/cloudsql-instance/README.md @@ -93,6 +93,59 @@ module "db" { } # tftest modules=1 resources=6 ``` + +### CMEK encryption pippo +```hcl + +module "project" { + source = "./modules/project" + billing_account = var.billing_account_id + parent = var.organization_id + name = "my-db-project" + services = [ + "servicenetworking.googleapis.com" + ] +} + +resource "google_project_service_identity" "jit_si" { + provider = google-beta + project = module.project.project_id + service = "sqladmin.googleapis.com" +} + +module "kms" { + source = "./modules/kms" + project_id = module.project.project_id + keyring = { + name = "keyring" + location = var.region + } + keys = { + key-sql = null + } + key_iam = { + key-sql = { + "roles/cloudkms.cryptoKeyEncrypterDecrypter" = [ + "serviceAccount:${google_project_service_identity.jit_si.email}" + ] + } + } +} + +module "db" { + source = "./modules/cloudsql-instance" + project_id = module.project.project_id + encryption_key_name = module.kms.keys["key-sql"].id + network = var.vpc.self_link + name = "db" + region = var.region + database_version = "POSTGRES_13" + tier = "db-g1-small" +} + +# tftest modules=3 resources=8 +``` + ## Variables diff --git a/modules/cloudsql-instance/main.tf b/modules/cloudsql-instance/main.tf index f50121ab..d9c86378 100644 --- a/modules/cloudsql-instance/main.tf +++ b/modules/cloudsql-instance/main.tf @@ -39,10 +39,12 @@ locals { } resource "google_sql_database_instance" "primary" { - project = var.project_id - name = "${local.prefix}${var.name}" - region = var.region - database_version = var.database_version + provider = google-beta + project = var.project_id + name = "${local.prefix}${var.name}" + region = var.region + database_version = var.database_version + encryption_key_name = var.encryption_key_name settings { tier = var.tier @@ -99,11 +101,13 @@ resource "google_sql_database_instance" "primary" { } resource "google_sql_database_instance" "replicas" { - for_each = local.has_replicas ? var.replicas : {} + provider = google-beta + for_each = length(var.replicas) > 0 ? var.replicas : {} project = var.project_id name = "${local.prefix}${each.key}" - region = each.value + region = each.value.region database_version = var.database_version + encryption_key_name = each.value.encryption_key_name master_instance_name = google_sql_database_instance.primary.name settings { diff --git a/modules/cloudsql-instance/outputs.tf b/modules/cloudsql-instance/outputs.tf index e2ca316d..38bfc951 100644 --- a/modules/cloudsql-instance/outputs.tf +++ b/modules/cloudsql-instance/outputs.tf @@ -66,6 +66,19 @@ output "ips" { } } +output "name" { + description = "Name of the primary instance." + value = google_sql_database_instance.primary.name +} + +output "names" { + description = "Names of all instances." + value = { + for id, instance in local._all_intances : + id => instance.name + } +} + output "self_link" { description = "Self link of the primary instance." value = google_sql_database_instance.primary.self_link diff --git a/modules/cloudsql-instance/variables.tf b/modules/cloudsql-instance/variables.tf index e59736ba..00130045 100644 --- a/modules/cloudsql-instance/variables.tf +++ b/modules/cloudsql-instance/variables.tf @@ -76,6 +76,12 @@ variable "disk_type" { default = "PD_SSD" } +variable "encryption_key_name" { + description = "The full path to the encryption key used for the CMEK disk encryption." + type = string + default = null +} + variable "flags" { description = "Map FLAG_NAME=>VALUE for database-specific tuning." type = map(string) @@ -115,9 +121,12 @@ variable "region" { } variable "replicas" { - description = "Map of NAME=>REGION for additional read replicas. Set to null to disable replica creation." - type = map(any) - default = null + description = "Map of NAME=> {REGION, KMS_KEY} for additional read replicas. Set to null to disable replica creation." + type = map(object({ + region = string + encryption_key_name = string + })) + default = {} } variable "tier" { diff --git a/modules/project/service-accounts.tf b/modules/project/service-accounts.tf index eae98e23..f5cffed4 100644 --- a/modules/project/service-accounts.tf +++ b/modules/project/service-accounts.tf @@ -42,6 +42,7 @@ locals { gcf = "service-%s@gcf-admin-robot" pubsub = "service-%s@gcp-sa-pubsub" secretmanager = "service-%s@gcp-sa-secretmanager" + sql = "service-%s@gcp-sa-cloud-sql" storage = "service-%s@gs-project-accounts" } service_accounts_default = { @@ -56,9 +57,10 @@ locals { k => "${format(v, local.project.number)}.iam.gserviceaccount.com" } service_accounts_jit_services = [ - "secretmanager.googleapis.com", + "cloudasset.googleapis.com", "pubsub.googleapis.com", - "cloudasset.googleapis.com" + "secretmanager.googleapis.com", + "sqladmin.googleapis.com" ] service_accounts_cmek_service_keys = distinct(flatten([ for s in keys(var.service_encryption_key_ids) : [ From 1f254880b961d8cdd49e4249b15dce87ca5f7937 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Wed, 13 Apr 2022 00:22:54 +0200 Subject: [PATCH 2/3] Add sqladmin to project jit_si and fix some documentation --- modules/cloudsql-instance/README.md | 47 +++++++++---------- modules/cloudsql-instance/main.tf | 2 +- modules/cloudsql-instance/variables.tf | 6 +-- modules/project/service-accounts.tf | 1 + .../cloudsql_instance/fixture/variables.tf | 2 +- tests/modules/cloudsql_instance/test_plan.py | 13 +++-- 6 files changed, 34 insertions(+), 37 deletions(-) diff --git a/modules/cloudsql-instance/README.md b/modules/cloudsql-instance/README.md index d5c050c8..75a7af0b 100644 --- a/modules/cloudsql-instance/README.md +++ b/modules/cloudsql-instance/README.md @@ -56,8 +56,8 @@ module "db" { tier = "db-g1-small" replicas = { - replica1 = "europe-west3" - replica2 = "us-central1" + replica1 = { region = "europe-west3", encryption_key_name = null } + replica2 = { region = "us-central1", encryption_key_name = null } } } # tftest modules=1 resources=3 @@ -103,16 +103,11 @@ module "project" { parent = var.organization_id name = "my-db-project" services = [ - "servicenetworking.googleapis.com" + "servicenetworking.googleapis.com", + "sqladmin.googleapis.com", ] } -resource "google_project_service_identity" "jit_si" { - provider = google-beta - project = module.project.project_id - service = "sqladmin.googleapis.com" -} - module "kms" { source = "./modules/kms" project_id = module.project.project_id @@ -126,7 +121,7 @@ module "kms" { key_iam = { key-sql = { "roles/cloudkms.cryptoKeyEncrypterDecrypter" = [ - "serviceAccount:${google_project_service_identity.jit_si.email}" + "serviceAccount:${module.project.service_accounts.robots.sqladmin}" ] } } @@ -143,9 +138,8 @@ module "db" { tier = "db-g1-small" } -# tftest modules=3 resources=8 +# tftest modules=3 resources=10 ``` - ## Variables @@ -153,11 +147,11 @@ module "db" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| | [database_version](variables.tf#L50) | Database type and version to create. | string | ✓ | | -| [name](variables.tf#L91) | Name of primary replica. | string | ✓ | | -| [network](variables.tf#L96) | VPC self link where the instances will be deployed. Private Service Networking must be enabled and configured in this VPC. | string | ✓ | | -| [project_id](variables.tf#L107) | The ID of the project where this instances will be created. | string | ✓ | | -| [region](variables.tf#L112) | Region of the primary replica. | string | ✓ | | -| [tier](variables.tf#L123) | The machine type to use for the instances. | string | ✓ | | +| [name](variables.tf#L97) | Name of primary instance. | string | ✓ | | +| [network](variables.tf#L102) | VPC self link where the instances will be deployed. Private Service Networking must be enabled and configured in this VPC. | string | ✓ | | +| [project_id](variables.tf#L113) | The ID of the project where this instances will be created. | string | ✓ | | +| [region](variables.tf#L118) | Region of the primary instance. | string | ✓ | | +| [tier](variables.tf#L132) | The machine type to use for the instances. | string | ✓ | | | [authorized_networks](variables.tf#L17) | Map of NAME=>CIDR_RANGE to allow to connect to the database(s). | map(string) | | null | | [availability_type](variables.tf#L23) | Availability type for the primary replica. Either `ZONAL` or `REGIONAL`. | string | | "ZONAL" | | [backup_configuration](variables.tf#L29) | Backup settings for primary instance. Will be automatically enabled if using MySQL with one or more replicas. | object({…}) | | {…} | @@ -165,11 +159,12 @@ module "db" { | [deletion_protection](variables.tf#L61) | Allow terraform to delete instances. | bool | | false | | [disk_size](variables.tf#L67) | Disk size in GB. Set to null to enable autoresize. | number | | null | | [disk_type](variables.tf#L73) | The type of data disk: `PD_SSD` or `PD_HDD`. | string | | "PD_SSD" | -| [flags](variables.tf#L79) | Map FLAG_NAME=>VALUE for database-specific tuning. | map(string) | | null | -| [labels](variables.tf#L85) | Labels to be attached to all instances. | map(string) | | null | -| [prefix](variables.tf#L101) | Prefix used to generate instance names. | string | | null | -| [replicas](variables.tf#L117) | Map of NAME=>REGION for additional read replicas. Set to null to disable replica creation. | map(any) | | null | -| [users](variables.tf#L128) | Map of users to create in the primary instance (and replicated to other replicas) in the format USER=>PASSWORD. For MySQL, anything afterr the first `@` (if persent) will be used as the user's host. Set PASSWORD to null if you want to get an autogenerated password. | map(string) | | null | +| [encryption_key_name](variables.tf#L79) | The full path to the encryption key used for the CMEK disk encryption of the primary instance. | string | | null | +| [flags](variables.tf#L85) | Map FLAG_NAME=>VALUE for database-specific tuning. | map(string) | | null | +| [labels](variables.tf#L91) | Labels to be attached to all instances. | map(string) | | null | +| [prefix](variables.tf#L107) | Prefix used to generate instance names. | string | | null | +| [replicas](variables.tf#L123) | Map of NAME=> {REGION, KMS_KEY} for additional read replicas. Set to null to disable replica creation. | map(object({…})) | | {} | +| [users](variables.tf#L137) | Map of users to create in the primary instance (and replicated to other replicas) in the format USER=>PASSWORD. For MySQL, anything afterr the first `@` (if persent) will be used as the user's host. Set PASSWORD to null if you want to get an autogenerated password. | map(string) | | null | ## Outputs @@ -182,8 +177,10 @@ module "db" { | [instances](outputs.tf#L50) | Cloud SQL instance resources. | ✓ | | [ip](outputs.tf#L56) | IP address of the primary instance. | | | [ips](outputs.tf#L61) | IP addresses of all instances. | | -| [self_link](outputs.tf#L69) | Self link of the primary instance. | | -| [self_links](outputs.tf#L74) | Self links of all instances. | | -| [user_passwords](outputs.tf#L82) | Map of containing the password of all users created through terraform. | ✓ | +| [name](outputs.tf#L69) | Name of the primary instance. | | +| [names](outputs.tf#L74) | Names of all instances. | | +| [self_link](outputs.tf#L82) | Self link of the primary instance. | | +| [self_links](outputs.tf#L87) | Self links of all instances. | | +| [user_passwords](outputs.tf#L95) | Map of containing the password of all users created through terraform. | ✓ | diff --git a/modules/cloudsql-instance/main.tf b/modules/cloudsql-instance/main.tf index d9c86378..be46b59c 100644 --- a/modules/cloudsql-instance/main.tf +++ b/modules/cloudsql-instance/main.tf @@ -102,7 +102,7 @@ resource "google_sql_database_instance" "primary" { resource "google_sql_database_instance" "replicas" { provider = google-beta - for_each = length(var.replicas) > 0 ? var.replicas : {} + for_each = local.has_replicas ? var.replicas : {} project = var.project_id name = "${local.prefix}${each.key}" region = each.value.region diff --git a/modules/cloudsql-instance/variables.tf b/modules/cloudsql-instance/variables.tf index 00130045..30acd915 100644 --- a/modules/cloudsql-instance/variables.tf +++ b/modules/cloudsql-instance/variables.tf @@ -77,7 +77,7 @@ variable "disk_type" { } variable "encryption_key_name" { - description = "The full path to the encryption key used for the CMEK disk encryption." + description = "The full path to the encryption key used for the CMEK disk encryption of the primary instance." type = string default = null } @@ -95,7 +95,7 @@ variable "labels" { } variable "name" { - description = "Name of primary replica." + description = "Name of primary instance." type = string } @@ -116,7 +116,7 @@ variable "project_id" { } variable "region" { - description = "Region of the primary replica." + description = "Region of the primary instance." type = string } diff --git a/modules/project/service-accounts.tf b/modules/project/service-accounts.tf index f5cffed4..5ba40fc4 100644 --- a/modules/project/service-accounts.tf +++ b/modules/project/service-accounts.tf @@ -44,6 +44,7 @@ locals { secretmanager = "service-%s@gcp-sa-secretmanager" sql = "service-%s@gcp-sa-cloud-sql" storage = "service-%s@gs-project-accounts" + sqladmin = "service-%s@gcp-sa-cloud-sql" } service_accounts_default = { compute = "${local.project.number}-compute@developer.gserviceaccount.com" diff --git a/tests/modules/cloudsql_instance/fixture/variables.tf b/tests/modules/cloudsql_instance/fixture/variables.tf index 62ff27a3..d6cc7d83 100644 --- a/tests/modules/cloudsql_instance/fixture/variables.tf +++ b/tests/modules/cloudsql_instance/fixture/variables.tf @@ -94,7 +94,7 @@ variable "region" { } variable "replicas" { - type = map(any) + type = any default = null } diff --git a/tests/modules/cloudsql_instance/test_plan.py b/tests/modules/cloudsql_instance/test_plan.py index 3c45f5c6..c4e8ba0a 100644 --- a/tests/modules/cloudsql_instance/test_plan.py +++ b/tests/modules/cloudsql_instance/test_plan.py @@ -35,8 +35,8 @@ def test_prefix(plan_runner): assert r['values']['name'] == 'prefix-db' replicas = """{ - replica1 = "europe-west3" - replica2 = "us-central1" + replica1 = { region = "europe-west3", encryption_key_name = null } + replica2 = { region = "us-central1", encryption_key_name = null } }""" _, resources = plan_runner(prefix="prefix") @@ -49,8 +49,8 @@ def test_replicas(plan_runner): "Test replicated instance." replicas = """{ - replica1 = "europe-west3" - replica2 = "us-central1" + replica1 = { region = "europe-west3", encryption_key_name = null } + replica2 = { region = "us-central1", encryption_key_name = null } }""" _, resources = plan_runner(replicas=replicas, prefix="prefix") @@ -80,10 +80,9 @@ def test_mysql_replicas_enables_backup(plan_runner): "Test MySQL backup setup with replicas." replicas = """{ - replica1 = "europe-west3" + replica1 = { region = "europe-west3", encryption_key_name = null } }""" - _, resources = plan_runner(replicas=replicas, - database_version="MYSQL_8_0") + _, resources = plan_runner(replicas=replicas, database_version="MYSQL_8_0") assert len(resources) == 2 primary = [r for r in resources if r['name'] == 'primary'][0] backup_config = primary['values']['settings'][0]['backup_configuration'][0] From b415d824c962ac0c86ae36de3109813dac970109 Mon Sep 17 00:00:00 2001 From: Lorenzo Caggioni Date: Wed, 13 Apr 2022 08:59:14 +0200 Subject: [PATCH 3/3] Fix README, bye bye pippo :-) --- modules/cloudsql-instance/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/cloudsql-instance/README.md b/modules/cloudsql-instance/README.md index 75a7af0b..73b2e3f2 100644 --- a/modules/cloudsql-instance/README.md +++ b/modules/cloudsql-instance/README.md @@ -94,7 +94,7 @@ module "db" { # tftest modules=1 resources=6 ``` -### CMEK encryption pippo +### CMEK encryption ```hcl module "project" {