zecfoundation
/
cloud-foundation-fabric
mirror of https://github.com/ZcashFoundation/cloud-foundation-fabric.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add KMS and Log.

This commit is contained in:
lcaggio 2023-01-21 01:08:51 +01:00
parent dcbfdd9c91
commit 4007d42705
4 changed files with 311 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
blueprints/data-solutions/shielded-folder/kms.tf Normal file
View File

@ -0,0 +1,100 @@
/**
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
locals {
kms_locations = distinct(flatten([
for k, v in var.kms_keys : v.locations
]))
kms_locations_keys = {
for loc in local.kms_locations : loc => {
for k, v in var.kms_keys : k => v if contains(v.locations, loc)
}
}
kms_log_locations = distinct(flatten([
for k, v in local.kms_log_sink_keys : compact(v.locations)
]))
# Log sink keys
kms_log_sink_keys = {
"log-gcs" = {
labels = {}
locations = [var.log_locations.gcs]
rotation_period = "7776000s"
}
"log-bq" = {
labels = {}
locations = [var.log_locations.bq]
rotation_period = "7776000s"
}
"log-pubsub" = {
labels = {}
locations = [var.log_locations.pubsub]
rotation_period = "7776000s"
}
}
kms_log_locations_keys = {
for loc in local.kms_log_locations : loc => {
for k, v in local.kms_log_sink_keys : k => v if contains(v.locations, loc)
}
}
}
module "sec-project" {
source = "../../../modules/project"
name = "sec-core"
parent = module.folder.id
billing_account = try(var.projects_create.billing_account_id, null)
project_create = var.projects_create != null
prefix = var.projects_create == null ? null : var.prefix
group_iam = {
(local.groups.data-engineers) = [
"roles/cloudkms.admin",
"roles/viewer",
]
}
services = [
"cloudkms.googleapis.com",
"secretmanager.googleapis.com",
"stackdriver.googleapis.com"
]
}
module "sec-kms" {
for_each = toset(local.kms_locations)
source = "../../../modules/kms"
project_id = module.sec-project.project_id
keyring = {
location = each.key
name = "${each.key}"
}
# rename to `key_iam` to switch to authoritative bindings
key_iam_additive = {
for k, v in local.kms_locations_keys[each.key] : k => v.iam
}
keys = local.kms_locations_keys[each.key]
}
module "log-kms" {
for_each = toset(local.kms_log_locations)
source = "../../../modules/kms"
project_id = module.sec-project.project_id
keyring = {
location = each.key
name = "log-${each.key}"
}
keys = local.kms_log_locations_keys[each.key]
}

95
blueprints/data-solutions/shielded-folder/log-export.tf Normal file
View File

@ -0,0 +1,95 @@
/**
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
# tfdoc:file:description Audit log project and sink.
locals {
gcs_storage_class = (
length(split("-", var.log_locations.gcs)) < 2
? "MULTI_REGIONAL"
: "REGIONAL"
)
log_types = toset([for k, v in var.log_sinks : v.type])
_log_keys = {
bq = [module.log-kms[var.log_locations.bq].keys["log-bq"].id]
pubsub = try([module.log-kms[var.log_locations.pubsub].keys["log-pubsub"].id], null)
storage = [module.log-kms[var.log_locations.gcs].keys["log-gcs"].id]
}
log_keys = {
for service, key in local._log_keys : service => key if key != null
}
}
module "log-export-project" {
source = "../../../modules/project"
name = "audit-logs"
parent = module.folder.id
billing_account = try(var.projects_create.billing_account_id, null)
project_create = var.projects_create != null
prefix = var.projects_create == null ? null : var.prefix
iam = {
# "roles/owner" = [module.automation-tf-bootstrap-sa.iam_email]
}
services = [
"bigquery.googleapis.com",
"storage.googleapis.com",
"stackdriver.googleapis.com"
]
service_encryption_key_ids = local.log_keys
}
# one log export per type, with conditionals to skip those not needed
module "log-export-dataset" {
source = "../../../modules/bigquery-dataset"
count = contains(local.log_types, "bigquery") ? 1 : 0
project_id = module.log-export-project.project_id
id = "${var.prefix}_audit_export"
friendly_name = "Audit logs export."
location = replace(var.log_locations.bq, "europe", "EU")
encryption_key = module.log-kms[var.log_locations.bq].keys["log-bq"].id
}
module "log-export-gcs" {
source = "../../../modules/gcs"
count = contains(local.log_types, "storage") ? 1 : 0
project_id = module.log-export-project.project_id
name = "audit-logs"
prefix = var.prefix
location = replace(var.log_locations.gcs, "europe", "EU")
storage_class = local.gcs_storage_class
encryption_key = module.log-kms[var.log_locations.gcs].keys["log-gcs"].id
}
module "log-export-logbucket" {
source = "../../../modules/logging-bucket"
for_each = toset([for k, v in var.log_sinks : k if v.type == "logging"])
parent_type = "project"
parent = module.log-export-project.project_id
id = "audit-logs-${each.key}"
location = var.log_locations.logging
#TODO check if logging bucket support encryption.
}
module "log-export-pubsub" {
source = "../../../modules/pubsub"
for_each = toset([for k, v in var.log_sinks : k if v.type == "pubsub"])
project_id = module.log-export-project.project_id
name = "audit-logs-${each.key}"
regions = [var.log_locations.pubsub]
kms_key = module.log-kms[var.log_locations.pubsub].keys["log-pubsub"].id
}

37
blueprints/data-solutions/shielded-folder/main.tf
View File

@ -23,7 +23,7 @@ locals {
)
groups = {
for k, v in var.groups : k => "${v}@${var.organization_domain}"
for k, v in var.groups : k => "${v}@${var.organization.domain}"
}
groups_iam = {
for k, v in local.groups : k => "group:${v}"
@ -38,14 +38,28 @@ locals {
for k, v in data.google_projects.folder-projects.projects : format("projects/%s", v.number)
]
log_sink_destinations = merge(
# use the same dataset for all sinks with `bigquery` as destination
{ for k, v in var.log_sinks : k => module.log-export-dataset.0 if v.type == "bigquery" },
# use the same gcs bucket for all sinks with `storage` as destination
{ for k, v in var.log_sinks : k => module.log-export-gcs.0 if v.type == "storage" },
# use separate pubsub topics and logging buckets for sinks with
# destination `pubsub` and `logging`
module.log-export-pubsub,
module.log-export-logbucket
)
}
module "folder" {
source = "../../../modules/folder"
folder_create = var.folder_create != null
parent = try(var.folder_create.parent, null)
name = try(var.folder_create.display_name, null)
id = var.folder_id
source = "../../../modules/folder"
folder_create = var.folder_create != null
parent = try(var.folder_create.parent, null)
name = try(var.folder_create.display_name, null)
id = var.folder_id
iam = {
"roles/owner" = ["serviceAccount:${var.bootstrap_service_account}"]
"roles/resourcemanager.projectCreator" = ["serviceAccount:${var.bootstrap_service_account}"]
}
group_iam = local.group_iam
org_policies_data_path = "${var.data_dir}/org-policies"
firewall_policy_factory = {
@ -53,7 +67,14 @@ module "folder" {
policy_name = "hierarchical-policy"
rules_file = "${var.data_dir}/firewall-policies/hierarchical-policy-rules.yaml"
}
#TODO logsink
logging_sinks = {
for name, attrs in var.log_sinks : name => {
bq_partitioned_table = attrs.type == "bigquery"
destination = local.log_sink_destinations[name].id
filter = attrs.filter
type = attrs.type
}
}
}
#TODO VPCSC
@ -72,7 +93,7 @@ module "vpc-sc" {
shielded = {
status = {
access_levels = keys(var.vpc_sc_access_levels)
resources = local.vpc_sc_resources
resources = null #TODO local.vpc_sc_resources
restricted_services = local._vpc_sc_restricted_services
egress_policies = keys(var.vpc_sc_egress_policies)
ingress_policies = keys(var.vpc_sc_ingress_policies)

90
blueprints/data-solutions/shielded-folder/variables.tf
View File

@ -30,6 +30,11 @@ variable "access_policy_create" {
default = null
}
variable "bootstrap_service_account" {
description = "Folder bootstrap service account: owner of the folder."
type = string
}
variable "data_dir" {
description = "Relative path for the folder storing configuration data."
type = string
@ -57,13 +62,92 @@ variable "groups" {
default = {
#TODO data-analysts = "gcp-data-analysts"
data-engineers = "gcp-data-engineers"
#TODO data-security = "gcp-data-security"
data-security = "gcp-data-security"
}
}
variable "organization_domain" {
description = "Organization domain."
variable "kms_keys" {
description = "KMS keys to create, keyed by name."
type = map(object({
iam = optional(map(list(string)), {})
labels = optional(map(string), {})
locations = optional(list(string), ["global", "europe", "europe-west1"])
rotation_period = optional(string, "7776000s")
}))
default = {}
}
variable "log_locations" {
description = "Optional locations for GCS, BigQuery, and logging buckets created here."
type = object({
bq = optional(string, "europe")
gcs = optional(string, "europe")
logging = optional(string, "global")
pubsub = optional(string, null)
})
default = {
bq = "europe"
gcs = "europe"
logging = "global"
pubsub = null
}
nullable = false
}
variable "log_sinks" {
description = "Org-level log sinks, in name => {type, filter} format."
type = map(object({
filter = string
type = string
}))
default = {
audit-logs = {
filter = "logName:\"/logs/cloudaudit.googleapis.com%2Factivity\" OR logName:\"/logs/cloudaudit.googleapis.com%2Fsystem_event\""
type = "bigquery"
}
vpc-sc = {
filter = "protoPayload.metadata.@type=\"type.googleapis.com/google.cloud.audit.VpcServiceControlAuditMetadata\""
type = "bigquery"
}
}
validation {
condition = alltrue([
for k, v in var.log_sinks :
contains(["bigquery", "logging", "pubsub", "storage"], v.type)
])
error_message = "Type must be one of 'bigquery', 'logging', 'pubsub', 'storage'."
}
}
variable "organization" {
description = "Organization details."
type = object({
domain = string
})
}
variable "prefix" {
description = "Prefix used for resources that need unique names. Use 9 characters or less."
type = string
validation {
condition = try(length(var.prefix), 0) < 10
error_message = "Use a maximum of 9 characters for prefix."
}
}
variable "projects_create" {
description = "Provide values if projects creation is needed, uses existing project if null. Projects will be created in the shielded folder."
type = object({
billing_account_id = string
})
default = null
}
variable "projects_id" {
description = "Project id, references existing project if `project_create` is null. Projects will be moved into the shielded folder."
type = map(string)
default = null
}
variable "vpc_sc_access_levels" {