diff --git a/blueprints/data-solutions/shielded-folder/kms.tf b/blueprints/data-solutions/shielded-folder/kms.tf new file mode 100644 index 00000000..94ff244d --- /dev/null +++ b/blueprints/data-solutions/shielded-folder/kms.tf @@ -0,0 +1,100 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + kms_locations = distinct(flatten([ + for k, v in var.kms_keys : v.locations + ])) + kms_locations_keys = { + for loc in local.kms_locations : loc => { + for k, v in var.kms_keys : k => v if contains(v.locations, loc) + } + } + + kms_log_locations = distinct(flatten([ + for k, v in local.kms_log_sink_keys : compact(v.locations) + ])) + + # Log sink keys + kms_log_sink_keys = { + "log-gcs" = { + labels = {} + locations = [var.log_locations.gcs] + rotation_period = "7776000s" + } + "log-bq" = { + labels = {} + locations = [var.log_locations.bq] + rotation_period = "7776000s" + } + "log-pubsub" = { + labels = {} + locations = [var.log_locations.pubsub] + rotation_period = "7776000s" + } + } + kms_log_locations_keys = { + for loc in local.kms_log_locations : loc => { + for k, v in local.kms_log_sink_keys : k => v if contains(v.locations, loc) + } + } +} + +module "sec-project" { + source = "../../../modules/project" + name = "sec-core" + parent = module.folder.id + billing_account = try(var.projects_create.billing_account_id, null) + project_create = var.projects_create != null + prefix = var.projects_create == null ? null : var.prefix + group_iam = { + (local.groups.data-engineers) = [ + "roles/cloudkms.admin", + "roles/viewer", + ] + } + services = [ + "cloudkms.googleapis.com", + "secretmanager.googleapis.com", + "stackdriver.googleapis.com" + ] +} + +module "sec-kms" { + for_each = toset(local.kms_locations) + source = "../../../modules/kms" + project_id = module.sec-project.project_id + keyring = { + location = each.key + name = "${each.key}" + } + # rename to `key_iam` to switch to authoritative bindings + key_iam_additive = { + for k, v in local.kms_locations_keys[each.key] : k => v.iam + } + keys = local.kms_locations_keys[each.key] +} + +module "log-kms" { + for_each = toset(local.kms_log_locations) + source = "../../../modules/kms" + project_id = module.sec-project.project_id + keyring = { + location = each.key + name = "log-${each.key}" + } + keys = local.kms_log_locations_keys[each.key] +} diff --git a/blueprints/data-solutions/shielded-folder/log-export.tf b/blueprints/data-solutions/shielded-folder/log-export.tf new file mode 100644 index 00000000..430b9f33 --- /dev/null +++ b/blueprints/data-solutions/shielded-folder/log-export.tf @@ -0,0 +1,95 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Audit log project and sink. + +locals { + gcs_storage_class = ( + length(split("-", var.log_locations.gcs)) < 2 + ? "MULTI_REGIONAL" + : "REGIONAL" + ) + log_types = toset([for k, v in var.log_sinks : v.type]) + _log_keys = { + bq = [module.log-kms[var.log_locations.bq].keys["log-bq"].id] + pubsub = try([module.log-kms[var.log_locations.pubsub].keys["log-pubsub"].id], null) + storage = [module.log-kms[var.log_locations.gcs].keys["log-gcs"].id] + } + + log_keys = { + for service, key in local._log_keys : service => key if key != null + } +} + +module "log-export-project" { + source = "../../../modules/project" + name = "audit-logs" + parent = module.folder.id + billing_account = try(var.projects_create.billing_account_id, null) + project_create = var.projects_create != null + prefix = var.projects_create == null ? null : var.prefix + iam = { + # "roles/owner" = [module.automation-tf-bootstrap-sa.iam_email] + } + services = [ + "bigquery.googleapis.com", + "storage.googleapis.com", + "stackdriver.googleapis.com" + ] + service_encryption_key_ids = local.log_keys +} + +# one log export per type, with conditionals to skip those not needed + +module "log-export-dataset" { + source = "../../../modules/bigquery-dataset" + count = contains(local.log_types, "bigquery") ? 1 : 0 + project_id = module.log-export-project.project_id + id = "${var.prefix}_audit_export" + friendly_name = "Audit logs export." + location = replace(var.log_locations.bq, "europe", "EU") + encryption_key = module.log-kms[var.log_locations.bq].keys["log-bq"].id +} + +module "log-export-gcs" { + source = "../../../modules/gcs" + count = contains(local.log_types, "storage") ? 1 : 0 + project_id = module.log-export-project.project_id + name = "audit-logs" + prefix = var.prefix + location = replace(var.log_locations.gcs, "europe", "EU") + storage_class = local.gcs_storage_class + encryption_key = module.log-kms[var.log_locations.gcs].keys["log-gcs"].id +} + +module "log-export-logbucket" { + source = "../../../modules/logging-bucket" + for_each = toset([for k, v in var.log_sinks : k if v.type == "logging"]) + parent_type = "project" + parent = module.log-export-project.project_id + id = "audit-logs-${each.key}" + location = var.log_locations.logging + #TODO check if logging bucket support encryption. +} + +module "log-export-pubsub" { + source = "../../../modules/pubsub" + for_each = toset([for k, v in var.log_sinks : k if v.type == "pubsub"]) + project_id = module.log-export-project.project_id + name = "audit-logs-${each.key}" + regions = [var.log_locations.pubsub] + kms_key = module.log-kms[var.log_locations.pubsub].keys["log-pubsub"].id +} diff --git a/blueprints/data-solutions/shielded-folder/main.tf b/blueprints/data-solutions/shielded-folder/main.tf index 410931e8..33a2b50b 100644 --- a/blueprints/data-solutions/shielded-folder/main.tf +++ b/blueprints/data-solutions/shielded-folder/main.tf @@ -23,7 +23,7 @@ locals { ) groups = { - for k, v in var.groups : k => "${v}@${var.organization_domain}" + for k, v in var.groups : k => "${v}@${var.organization.domain}" } groups_iam = { for k, v in local.groups : k => "group:${v}" @@ -38,14 +38,28 @@ locals { for k, v in data.google_projects.folder-projects.projects : format("projects/%s", v.number) ] + log_sink_destinations = merge( + # use the same dataset for all sinks with `bigquery` as destination + { for k, v in var.log_sinks : k => module.log-export-dataset.0 if v.type == "bigquery" }, + # use the same gcs bucket for all sinks with `storage` as destination + { for k, v in var.log_sinks : k => module.log-export-gcs.0 if v.type == "storage" }, + # use separate pubsub topics and logging buckets for sinks with + # destination `pubsub` and `logging` + module.log-export-pubsub, + module.log-export-logbucket + ) } module "folder" { - source = "../../../modules/folder" - folder_create = var.folder_create != null - parent = try(var.folder_create.parent, null) - name = try(var.folder_create.display_name, null) - id = var.folder_id + source = "../../../modules/folder" + folder_create = var.folder_create != null + parent = try(var.folder_create.parent, null) + name = try(var.folder_create.display_name, null) + id = var.folder_id + iam = { + "roles/owner" = ["serviceAccount:${var.bootstrap_service_account}"] + "roles/resourcemanager.projectCreator" = ["serviceAccount:${var.bootstrap_service_account}"] + } group_iam = local.group_iam org_policies_data_path = "${var.data_dir}/org-policies" firewall_policy_factory = { @@ -53,7 +67,14 @@ module "folder" { policy_name = "hierarchical-policy" rules_file = "${var.data_dir}/firewall-policies/hierarchical-policy-rules.yaml" } - #TODO logsink + logging_sinks = { + for name, attrs in var.log_sinks : name => { + bq_partitioned_table = attrs.type == "bigquery" + destination = local.log_sink_destinations[name].id + filter = attrs.filter + type = attrs.type + } + } } #TODO VPCSC @@ -72,7 +93,7 @@ module "vpc-sc" { shielded = { status = { access_levels = keys(var.vpc_sc_access_levels) - resources = local.vpc_sc_resources + resources = null #TODO local.vpc_sc_resources restricted_services = local._vpc_sc_restricted_services egress_policies = keys(var.vpc_sc_egress_policies) ingress_policies = keys(var.vpc_sc_ingress_policies) diff --git a/blueprints/data-solutions/shielded-folder/variables.tf b/blueprints/data-solutions/shielded-folder/variables.tf index 77b68a4e..548d6fd2 100644 --- a/blueprints/data-solutions/shielded-folder/variables.tf +++ b/blueprints/data-solutions/shielded-folder/variables.tf @@ -30,6 +30,11 @@ variable "access_policy_create" { default = null } +variable "bootstrap_service_account" { + description = "Folder bootstrap service account: owner of the folder." + type = string +} + variable "data_dir" { description = "Relative path for the folder storing configuration data." type = string @@ -57,13 +62,92 @@ variable "groups" { default = { #TODO data-analysts = "gcp-data-analysts" data-engineers = "gcp-data-engineers" - #TODO data-security = "gcp-data-security" + data-security = "gcp-data-security" } } -variable "organization_domain" { - description = "Organization domain." +variable "kms_keys" { + description = "KMS keys to create, keyed by name." + type = map(object({ + iam = optional(map(list(string)), {}) + labels = optional(map(string), {}) + locations = optional(list(string), ["global", "europe", "europe-west1"]) + rotation_period = optional(string, "7776000s") + })) + default = {} +} + +variable "log_locations" { + description = "Optional locations for GCS, BigQuery, and logging buckets created here." + type = object({ + bq = optional(string, "europe") + gcs = optional(string, "europe") + logging = optional(string, "global") + pubsub = optional(string, null) + }) + default = { + bq = "europe" + gcs = "europe" + logging = "global" + pubsub = null + } + nullable = false +} + +variable "log_sinks" { + description = "Org-level log sinks, in name => {type, filter} format." + type = map(object({ + filter = string + type = string + })) + default = { + audit-logs = { + filter = "logName:\"/logs/cloudaudit.googleapis.com%2Factivity\" OR logName:\"/logs/cloudaudit.googleapis.com%2Fsystem_event\"" + type = "bigquery" + } + vpc-sc = { + filter = "protoPayload.metadata.@type=\"type.googleapis.com/google.cloud.audit.VpcServiceControlAuditMetadata\"" + type = "bigquery" + } + } + validation { + condition = alltrue([ + for k, v in var.log_sinks : + contains(["bigquery", "logging", "pubsub", "storage"], v.type) + ]) + error_message = "Type must be one of 'bigquery', 'logging', 'pubsub', 'storage'." + } +} + +variable "organization" { + description = "Organization details." + type = object({ + domain = string + }) +} + +variable "prefix" { + description = "Prefix used for resources that need unique names. Use 9 characters or less." type = string + + validation { + condition = try(length(var.prefix), 0) < 10 + error_message = "Use a maximum of 9 characters for prefix." + } +} + +variable "projects_create" { + description = "Provide values if projects creation is needed, uses existing project if null. Projects will be created in the shielded folder." + type = object({ + billing_account_id = string + }) + default = null +} + +variable "projects_id" { + description = "Project id, references existing project if `project_create` is null. Projects will be moved into the shielded folder." + type = map(string) + default = null } variable "vpc_sc_access_levels" {