Merge branch 'master' into fast-dev-dp

This commit is contained in:
Ludovico Magnocavallo 2022-02-09 14:37:28 +01:00 committed by GitHub
commit a64e7a8e41
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
34 changed files with 799 additions and 117 deletions

View File

@ -8,6 +8,10 @@ All notable changes to this project will be documented in this file.
- remove GCS to BQ with Dataflow example, replace by GCS to BQ with least privileges
- the `net-vpc` and `project` modules now use the beta provider for shared VPC-related resources
- new `iot-core` module
- **incompatible change** the variables for host and service Shared VPCs have changed in the project module
- **incompatible change** the variable for service identities IAM has changed in the project factory
- the `net-vpc` and `project` modules now use the beta provider for shared VPC-related resources
- add `data-catalog-policy-tag` module
## [13.0.0] - 2022-01-27

View File

@ -29,7 +29,7 @@ Currently available modules:
- **foundational** - [folder](./modules/folder), [organization](./modules/organization), [project](./modules/project), [service accounts](./modules/iam-service-account), [logging bucket](./modules/logging-bucket), [billing budget](./modules/billing-budget), [naming convention](./modules/naming-convention)
- **networking** - [VPC](./modules/net-vpc), [VPC firewall](./modules/net-vpc-firewall), [VPC peering](./modules/net-vpc-peering), [VPN static](./modules/net-vpn-static), [VPN dynamic](./modules/net-vpn-dynamic), [VPN HA](./modules/net-vpn-ha), [NAT](./modules/net-cloudnat), [address reservation](./modules/net-address), [DNS](./modules/dns), [L4 ILB](./modules/net-ilb), [Service Directory](./modules/service-directory), [Cloud Endpoints](./modules/endpoints)
- **compute** - [VM/VM group](./modules/compute-vm), [MIG](./modules/compute-mig), [GKE cluster](./modules/gke-cluster), [GKE nodepool](./modules/gke-nodepool), [COS container](./modules/cloud-config-container/cos-generic-metadata/) (coredns, mysql, onprem, squid)
- **data** - [GCS](./modules/gcs), [BigQuery dataset](./modules/bigquery-dataset), [Pub/Sub](./modules/pubsub), [Datafusion](./modules/datafusion), [Bigtable instance](./modules/bigtable-instance), [Cloud SQL instance](./modules/cloudsql-instance)
- **data** - [GCS](./modules/gcs), [BigQuery dataset](./modules/bigquery-dataset), [Pub/Sub](./modules/pubsub), [Datafusion](./modules/datafusion), [Bigtable instance](./modules/bigtable-instance), [Cloud SQL instance](./modules/cloudsql-instance), [Data Catalog Policy Tag](./modules/data-catalog-policy-tag)
- **development** - [Cloud Source Repository](./modules/source-repository), [Container Registry](./modules/container-registry), [Artifact Registry](./modules/artifact-registry), [Apigee Organization](./modules/apigee-organization), [Apigee X Instance](./modules/apigee-x-instance)
- **security** - [KMS](./modules/kms), [SecretManager](./modules/secret-manager), [VPC Service Control](./modules/vpc-sc)
- **serverless** - [Cloud Function](./modules/cloud-function), [Cloud Run](./modules/cloud-run)

View File

@ -231,15 +231,15 @@ vpc:
| [org_policies](variables.tf#L98) | Org-policy overrides at project level. | <code title="object&#40;&#123;&#10; policy_boolean &#61; map&#40;bool&#41;&#10; policy_list &#61; map&#40;object&#40;&#123;&#10; inherit_from_parent &#61; bool&#10; suggested_value &#61; string&#10; status &#61; bool&#10; values &#61; list&#40;string&#41;&#10; &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> |
| [prefix](variables.tf#L112) | Prefix used for the project id. | <code>string</code> | | <code>null</code> |
| [service_accounts](variables.tf#L123) | Service accounts to be created, and roles to assign them. | <code>map&#40;list&#40;string&#41;&#41;</code> | | <code>&#123;&#125;</code> |
| [service_identities_iam](variables.tf#L136) | Custom IAM settings for service identities in service => [role] format. | <code>map&#40;list&#40;string&#41;&#41;</code> | | <code>&#123;&#125;</code> |
| [services](variables.tf#L129) | Services to be enabled for the project. | <code>list&#40;string&#41;</code> | | <code>&#91;&#93;</code> |
| [services_iam](variables.tf#L135) | Custom IAM settings for robot ServiceAccounts in service => [role] format. | <code>map&#40;list&#40;string&#41;&#41;</code> | | <code>&#123;&#125;</code> |
| [vpc](variables.tf#L141) | VPC configuration for the project. | <code title="object&#40;&#123;&#10; host_project &#61; string&#10; gke_setup &#61; object&#40;&#123;&#10; enable_security_admin &#61; bool&#10; enable_host_service_agent &#61; bool&#10; &#125;&#41;&#10; subnets_iam &#61; map&#40;list&#40;string&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> |
| [vpc](variables.tf#L143) | VPC configuration for the project. | <code title="object&#40;&#123;&#10; host_project &#61; string&#10; gke_setup &#61; object&#40;&#123;&#10; enable_security_admin &#61; bool&#10; enable_host_service_agent &#61; bool&#10; &#125;&#41;&#10; subnets_iam &#61; map&#40;list&#40;string&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> |
## Outputs
| name | description | sensitive |
|---|---|:---:|
| [project](outputs.tf#L19) | The project resource as return by the `project` module | |
| [project_id](outputs.tf#L30) | Project ID. | |
| [project_id](outputs.tf#L29) | Project ID. | |
<!-- END TFDOC -->

View File

@ -15,24 +15,26 @@
*/
locals {
_gke_iam_hsau = try(var.vpc.gke_setup.enable_host_service_agent, false) ? {
"roles/container.hostServiceAgentUser" = "serviceAccount:${module.project.service_accounts.robots.container-engine}"
} : {}
_gke_iam_securityadmin = try(var.vpc.gke_setup.enable_security_admin, false) ? {
"roles/compute.securityAdmin" = "serviceAccount:${module.project.service_accounts.robots.container-engine}"
} : {}
# internal structures for group IAM bindings
_group_iam = {
for r in local._group_iam_roles : r => [
for k, v in var.group_iam : "group:${k}" if try(index(v, r), null) != null
for r in local._group_iam_bindings : r => [
for k, v in var.group_iam :
"group:${k}" if try(index(v, r), null) != null
]
}
_group_iam_roles = distinct(flatten(values(var.group_iam)))
_group_iam_bindings = distinct(flatten(values(var.group_iam)))
# internal structures for project service accounts IAM bindings
_service_accounts_iam = {
for r in local._service_accounts_iam_roles : r => [
for k, v in var.service_accounts : "serviceAccount:${k}@${var.project_id}.iam.gserviceaccount.com" if try(index(v, r), null) != null
for r in local._service_accounts_iam_bindings : r => [
for k, v in var.service_accounts :
"serviceAccount:${k}@${var.project_id}.iam.gserviceaccount.com"
if try(index(v, r), null) != null
]
}
_service_accounts_iam_roles = distinct(flatten(values(var.service_accounts)))
_service_accounts_iam_bindings = distinct(flatten(
values(var.service_accounts)
))
# internal structures for project services
_services = concat([
"billingbudgets.googleapis.com",
"essentialcontacts.googleapis.com"
@ -41,46 +43,81 @@ locals {
try(var.vpc.gke_setup, null) != null ? ["container.googleapis.com"] : [],
var.vpc != null ? ["compute.googleapis.com"] : [],
)
_services_iam_roles = distinct(flatten(values(var.services_iam)))
_services_iam = {
for r in local._services_iam_roles : r => [
for k, v in var.services_iam : "serviceAccount:${module.project.service_accounts.robots[k]}" if try(index(v, r), null) != null
# internal structures for service identity IAM bindings
_service_identities_roles = distinct(flatten(values(var.service_identities_iam)))
_service_identities_iam = {
for role in local._service_identities_roles : role => [
for service, roles in var.service_identities_iam :
"serviceAccount:${module.project.service_accounts.robots[service]}"
if contains(roles, role)
]
}
billing_account_id = coalesce(var.billing_account_id, try(var.defaults.billing_account_id, ""))
billing_alert = var.billing_alert == null ? try(var.defaults.billing_alert, null) : var.billing_alert
essential_contacts = concat(try(var.defaults.essential_contacts, []), var.essential_contacts)
host_project_bindings = merge(
local._gke_iam_hsau,
local._gke_iam_securityadmin
# internal structure for Shared VPC service project IAM bindings
_vpc_subnet_bindings = (
local.vpc.subnets_iam == null || local.vpc.host_project == null
? []
: flatten([
for subnet, members in local.vpc.subnets_iam : [
for member in members : {
region = split("/", subnet)[0]
subnet = split("/", subnet)[1]
member = member
}
]
])
)
# structures for billing id
billing_account_id = coalesce(
var.billing_account_id, try(var.defaults.billing_account_id, "")
)
billing_alert = (
var.billing_alert == null
? try(var.defaults.billing_alert, null)
: var.billing_alert
)
# structure for essential contacts
essential_contacts = concat(
try(var.defaults.essential_contacts, []), var.essential_contacts
)
# structure that combines all authoritative IAM bindings
iam = {
for role in distinct(concat(
keys(var.iam),
keys(local._group_iam),
keys(local._service_accounts_iam),
keys(local._services_iam),
keys(local._service_identities_iam),
)) :
role => concat(
try(var.iam[role], []),
try(local._group_iam[role], []),
try(local._service_accounts_iam[role], []),
try(local._services_iam[role], []),
try(local._service_identities_iam[role], []),
)
}
labels = merge(coalesce(var.labels, {}), coalesce(try(var.defaults.labels, {}), {}))
network_user_service_accounts = concat(
contains(local.services, "compute.googleapis.com") ? [
"serviceAccount:${module.project.service_accounts.robots.compute}"
] : [],
contains(local.services, "container.googleapis.com") ? [
"serviceAccount:${module.project.service_accounts.robots.container-engine}",
"serviceAccount:${module.project.service_accounts.cloud_services}"
] : [],
[])
services = distinct(concat(var.services, local._services))
vpc_host_project = try(var.vpc.host_project, var.defaults.vpc_host_project)
vpc_setup = var.vpc != null
# merge labels with defaults
labels = merge(
coalesce(var.labels, {}), coalesce(try(var.defaults.labels, {}), {})
)
# deduplicate services
services = distinct(concat(var.services, local._services))
# structures for Shared VPC resources in host project
vpc = coalesce(var.vpc, {
host_project = null, gke_setup = null, subnets_iam = null
})
vpc_cloudservices = (
local.vpc_gke_service_agent ||
contains(var.services, "compute.googleapis.com")
)
vpc_gke_security_admin = coalesce(
try(local.vpc.gke_setup.enable_security_admin, null), false
)
vpc_gke_service_agent = coalesce(
try(local.vpc.gke_setup.enable_host_service_agent, null), false
)
vpc_subnet_bindings = {
for binding in local._vpc_subnet_bindings :
"${binding.subnet}:${binding.member}" => binding
}
}
module "billing-alert" {
@ -122,9 +159,21 @@ module "project" {
policy_list = try(var.org_policies.policy_list, {})
service_encryption_key_ids = var.kms_service_agents
services = local.services
shared_vpc_service_config = {
attach = local.vpc_setup
host_project = local.vpc_host_project
shared_vpc_service_config = var.vpc == null ? null : {
host_project = local.vpc.host_project
# these are non-authoritative
service_identity_iam = {
"roles/compute.networkUser" = compact([
local.vpc_gke_service_agent ? "container-engine" : null,
local.vpc_cloudservices ? "cloudservices" : null
])
"roles/compute.securityAdmin" = compact([
local.vpc_gke_security_admin ? "container-engine" : null,
])
"roles/container.hostServiceAgentUser" = compact([
local.vpc_gke_service_agent ? "container-engine" : null
])
}
}
}
@ -135,19 +184,11 @@ module "service-accounts" {
project_id = module.project.project_id
}
# TODO(jccb): we should probably change this to non-authoritative bindings
resource "google_compute_subnetwork_iam_binding" "binding" {
for_each = local.vpc_setup ? coalesce(var.vpc.subnets_iam, {}) : {}
project = local.vpc_host_project
subnetwork = "projects/${local.vpc_host_project}/regions/${split("/", each.key)[0]}/subnetworks/${split("/", each.key)[1]}"
region = split("/", each.key)[0]
resource "google_compute_subnetwork_iam_member" "default" {
for_each = local.vpc_subnet_bindings
project = local.vpc.host_project
subnetwork = "projects/${local.vpc.host_project}/regions/${each.value.region}/subnetworks/${each.value.subnet}"
region = each.value.region
role = "roles/compute.networkUser"
members = concat(each.value, local.network_user_service_accounts)
}
resource "google_project_iam_member" "host_project_bindings" {
for_each = local.host_project_bindings
project = local.vpc_host_project
role = each.key
member = each.value
member = each.value.member
}

View File

@ -21,8 +21,7 @@ output "project" {
value = module.project
depends_on = [
google_compute_subnetwork_iam_binding.binding,
google_project_iam_member.host_project_bindings,
google_compute_subnetwork_iam_member.default,
module.dns
]
}
@ -31,8 +30,7 @@ output "project_id" {
description = "Project ID."
value = module.project.project_id
depends_on = [
google_compute_subnetwork_iam_binding.binding,
google_project_iam_member.host_project_bindings,
google_compute_subnetwork_iam_member.default,
module.dns
]
}

View File

@ -130,12 +130,14 @@ variable "services" {
description = "Services to be enabled for the project."
type = list(string)
default = []
nullable = false
}
variable "services_iam" {
description = "Custom IAM settings for robot ServiceAccounts in service => [role] format."
variable "service_identities_iam" {
description = "Custom IAM settings for service identities in service => [role] format."
type = map(list(string))
default = {}
nullable = false
}
variable "vpc" {

View File

@ -252,8 +252,10 @@ module "project-app" {
prefix = var.prefix
services = ["compute.googleapis.com"]
shared_vpc_service_config = {
attach = true
host_project = module.project-host.project_id
service_identity_iam = {
"roles/compute.networkUser" = ["cloudservices"]
}
}
}

View File

@ -31,9 +31,6 @@ module "project-host" {
service_projects = [] # defined later
}
iam = {
"roles/container.hostServiceAgentUser" = [
"serviceAccount:${module.project-svc-gke.service_accounts.robots.container-engine}"
]
"roles/owner" = var.owners_host
}
}
@ -48,8 +45,10 @@ module "project-svc-gce" {
oslogin = true
oslogin_admins = var.owners_gce
shared_vpc_service_config = {
attach = true
host_project = module.project-host.project_id
service_identity_iam = {
"roles/compute.networkUser" = ["cloudservices"]
}
}
iam = {
"roles/owner" = var.owners_gce
@ -67,8 +66,11 @@ module "project-svc-gke" {
name = "gke"
services = var.project_services
shared_vpc_service_config = {
attach = true
host_project = module.project-host.project_id
service_identity_iam = {
"roles/container.hostServiceAgentUser" = ["container-engine"]
"roles/compute.networkUser" = ["container-engine"]
}
}
iam = merge(
{

View File

@ -25,7 +25,7 @@ org_policies: include('org_policies', required=False)
secrets: map(list(str()), key=str(), required=False)
service_accounts: map(list(str()), required=False)
services: list(str(matches='^[a-z-]*\.googleapis\.com$'), required=False)
services_iam: map(list(str()), key=str(), required=False)
service_identities_iam: map(list(str()), key=str(), required=False)
vpc: include('vpc', required=False)
---
billing_alert:

View File

@ -87,6 +87,7 @@ module "organization" {
"constraints/compute.requireOsLogin" = true
"constraints/compute.restrictXpnProjectLienRemoval" = true
"constraints/compute.skipDefaultNetworkCreation" = true
"constraints/compute.setNewProjectDefaultToZonalDNSOnly" = true
"constraints/iam.automaticIamGrantsForDefaultServiceAccounts" = true
"constraints/iam.disableServiceAccountKeyCreation" = true
"constraints/iam.disableServiceAccountKeyUpload" = true

View File

@ -56,7 +56,7 @@ It's of course possible to run this stage in isolation, by making sure the archi
If you're running this on top of Fast, you should run the following commands to create the providers file, and populate the required variables from the previous stage.
```bash
# Variable `outputs_location` is set to `../../config` in stage 01-resman
# Variable `outputs_location` is set to `../../../config` in stage 01-resman
$ cd fabric-fast/stages/03-project-factory/prod
ln -s ../../../config/03-project-factory-prod/providers.tf
```
@ -73,7 +73,7 @@ To avoid the tedious job of filling in the first group of variables with values
If you configured a valid path for `outputs_location` in the bootstrap and networking stage, simply link the relevant `terraform-*.auto.tfvars.json` files from this stage's outputs folder (under the path you specified), where the `*` above is set to the name of the stage that produced it. For this stage, a single `.tfvars` file is available:
```bash
# Variable `outputs_location` is set to `../../config` in stages 01-bootstrap and the 02-networking stage in use
# Variable `outputs_location` is set to `../../../config` in stages 01-bootstrap and the 02-networking stage in use
ln -s ../../../config/03-project-factory-prod/terraform-bootstrap.auto.tfvars.json
ln -s ../../../config/03-project-factory-prod/terraform-networking.auto.tfvars.json
```

View File

@ -72,8 +72,8 @@ services:
- stackdriver.googleapis.com
- compute.googleapis.com
# [opt] Roles to assign to the robots service accounts in robot => [roles] format
services_iam:
# [opt] Roles to assign to the service identities in service => [roles] format
service_identities_iam:
compute:
- roles/storage.objectViewer

View File

@ -69,6 +69,7 @@ These modules are used in the examples included in this repository. If you are u
- [Pub/Sub](./pubsub)
- [Bigtable instance](./bigtable-instance)
- [Cloud SQL instance](./cloudsql-instance)
- [Data Catalog Policy Tag](./data-catalog-policy-tag)
## Development

View File

@ -0,0 +1,62 @@
# Data Catalog Module
This module simplifies the creation of [Data Catalog](https://cloud.google.com/data-catalog) Policy Tags. Policy Tags can be used to configure [Bigquery column-level access](https://cloud.google.com/bigquery/docs/best-practices-policy-tags).
Note: Data Catalog is still in beta, hence this module currently uses the beta provider.
## Examples
### Simple Taxonomy with policy tags
```hcl
module "cmn-dc" {
source = "./modules/data-catalog-policy-tag"
name = "my-datacatalog-policy-tags"
project_id = "my-project"
tags = ["low", "medium", "high"]
}
# tftest modules=1 resources=4
```
### Simple Taxonomy with IAM binding
```hcl
module "cmn-dc" {
source = "./modules/data-catalog-policy-tag"
name = "my-datacatalog-policy-tags"
project_id = "my-project"
tags = ["low", "medium", "high"]
iam = {
"roles/datacatalog.categoryAdmin" = ["group:GROUP_NAME@example.com"]
}
}
# tftest modules=1 resources=5
```
<!-- BEGIN TFDOC -->
## Variables
| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
| [name](variables.tf#L59) | Name of this taxonomy. | <code>string</code> | ✓ | |
| [project_id](variables.tf#L70) | GCP project id. | <code></code> | ✓ | |
| [activated_policy_types](variables.tf#L17) | A list of policy types that are activated for this taxonomy. | <code>list&#40;string&#41;</code> | | <code>&#91;&#34;FINE_GRAINED_ACCESS_CONTROL&#34;&#93;</code> |
| [description](variables.tf#L23) | Description of this taxonomy. | <code>string</code> | | <code>&#34;Taxonomy - Terraform managed&#34;</code> |
| [group_iam](variables.tf#L29) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | <code>map&#40;list&#40;string&#41;&#41;</code> | | <code>&#123;&#125;</code> |
| [iam](variables.tf#L35) | IAM bindings in {ROLE => [MEMBERS]} format. | <code>map&#40;list&#40;string&#41;&#41;</code> | | <code>&#123;&#125;</code> |
| [iam_additive](variables.tf#L41) | IAM additive bindings in {ROLE => [MEMBERS]} format. | <code>map&#40;list&#40;string&#41;&#41;</code> | | <code>&#123;&#125;</code> |
| [iam_additive_members](variables.tf#L47) | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | <code>map&#40;list&#40;string&#41;&#41;</code> | | <code>&#123;&#125;</code> |
| [location](variables.tf#L53) | Data Catalog Taxonomy location. | <code>string</code> | | <code>&#34;eu&#34;</code> |
| [prefix](variables.tf#L64) | Prefix used to generate project id and name. | <code>string</code> | | <code>null</code> |
| [tags](variables.tf#L74) | List of Data Catalog Policy tags to be created. | <code>list&#40;string&#41;</code> | | <code>&#91;&#93;</code> |
## Outputs
| name | description | sensitive |
|---|---|:---:|
| [tags](outputs.tf#L17) | Policy Tags. | |
| [taxonomy_id](outputs.tf#L22) | Taxonomy id. | |
<!-- END TFDOC -->
## TODO
- Support IAM at tag level.
- Support Child policy tags

View File

@ -0,0 +1,67 @@
/**
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
# tfdoc:file:description Data Catalog Taxonomy IAM definition.
locals {
_group_iam = {
for r in local._group_iam_roles : r => [
for k, v in var.group_iam : "group:${k}" if try(index(v, r), null) != null
]
}
_group_iam_roles = distinct(flatten(values(var.group_iam)))
_iam_additive_member_pairs = flatten([
for member, roles in var.iam_additive_members : [
for role in roles : { role = role, member = member }
]
])
_iam_additive_pairs = flatten([
for role, members in var.iam_additive : [
for member in members : { role = role, member = member }
]
])
iam = {
for role in distinct(concat(keys(var.iam), keys(local._group_iam))) :
role => concat(
try(var.iam[role], []),
try(local._group_iam[role], [])
)
}
iam_additive = {
for pair in concat(local._iam_additive_pairs, local._iam_additive_member_pairs) :
"${pair.role}-${pair.member}" => pair
}
}
resource "google_data_catalog_taxonomy_iam_binding" "authoritative" {
provider = google-beta
for_each = local.iam
role = each.key
members = each.value
taxonomy = google_data_catalog_taxonomy.default.id
}
resource "google_data_catalog_taxonomy_iam_member" "additive" {
provider = google-beta
for_each = (
length(var.iam_additive) + length(var.iam_additive_members) > 0
? local.iam_additive
: {}
)
role = each.value.role
member = each.value.member
taxonomy = google_data_catalog_taxonomy.default.id
}

View File

@ -0,0 +1,43 @@
/**
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
# tfdoc:file:description Data Catalog Taxonomy definition
locals {
name = (
var.name != null ? var.name : "${local.prefix}taxonomy"
)
prefix = var.prefix == null ? "" : "${var.prefix}-"
}
resource "google_data_catalog_taxonomy" "default" {
provider = google-beta
project = var.project_id
region = var.location
display_name = local.name
description = var.description
activated_policy_types = var.activated_policy_types
}
resource "google_data_catalog_policy_tag" "default" {
for_each = toset(var.tags)
provider = google-beta
taxonomy = google_data_catalog_taxonomy.default.id
display_name = each.key
description = "${each.key} - Terraform managed. "
}
#TODO Add IAM at tag level

View File

@ -0,0 +1,25 @@
/**
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
output "tags" {
description = "Policy Tags."
value = { for k, v in google_data_catalog_policy_tag.default : v.id => v.name }
}
output "taxonomy_id" {
description = "Taxonomy id."
value = google_data_catalog_taxonomy.default.id
}

View File

@ -0,0 +1,78 @@
/**
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
variable "activated_policy_types" {
description = "A list of policy types that are activated for this taxonomy."
type = list(string)
default = ["FINE_GRAINED_ACCESS_CONTROL"]
}
variable "description" {
description = "Description of this taxonomy."
type = string
default = "Taxonomy - Terraform managed"
}
variable "group_iam" {
description = "Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable."
type = map(list(string))
default = {}
}
variable "iam" {
description = "IAM bindings in {ROLE => [MEMBERS]} format."
type = map(list(string))
default = {}
}
variable "iam_additive" {
description = "IAM additive bindings in {ROLE => [MEMBERS]} format."
type = map(list(string))
default = {}
}
variable "iam_additive_members" {
description = "IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values."
type = map(list(string))
default = {}
}
variable "location" {
description = "Data Catalog Taxonomy location."
type = string
default = "eu"
}
variable "name" {
description = "Name of this taxonomy."
type = string
}
variable "prefix" {
description = "Prefix used to generate project id and name."
type = string
default = null
}
variable "project_id" {
description = "GCP project id."
}
variable "tags" {
description = "List of Data Catalog Policy tags to be created."
type = list(string)
default = []
}

View File

@ -0,0 +1,29 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
terraform {
required_version = ">= 1.1.0"
required_providers {
google = {
source = "hashicorp/google"
version = ">= 4.0.0"
}
google-beta = {
source = "hashicorp/google-beta"
version = ">= 4.0.0"
}
}
}

View File

@ -36,14 +36,46 @@ module "project" {
name = "project-example"
iam_additive = {
"roles/viewer" = ["group:one@example.org", "group:two@xample.org"],
"roles/storage.objectAdmin" = ["group:two@example.org"],
"roles/owner" = ["group:three@example.org"],
"roles/viewer" = [
"group:one@example.org", "group:two@xample.org"
],
"roles/storage.objectAdmin" = [
"group:two@example.org"
],
"roles/owner" = [
"group:three@example.org"
],
}
}
# tftest modules=1 resources=5
```
### Shared VPC service
```hcl
module "project" {
source = "./modules/project"
name = "project-example"
shared_vpc_service_config = {
attach = true
host_project = "my-host-project"
service_identity_iam = {
"roles/compute.networkUser" = [
"cloudservices", "container-engine"
]
"roles/vpcaccess.user" = [
"cloudrun"
]
"roles/container.hostServiceAgentUser" = [
"container-engine"
]
}
}
}
# tftest modules=1 resources=6
```
### Organization policies
```hcl
@ -74,6 +106,7 @@ module "project" {
```
## Logging Sinks
```hcl
module "gcs" {
source = "./modules/gcs"
@ -187,7 +220,7 @@ module "project" {
| [organization-policies.tf](./organization-policies.tf) | Project-level organization policies. | <code>google_project_organization_policy</code> |
| [outputs.tf](./outputs.tf) | Module outputs. | |
| [service-accounts.tf](./service-accounts.tf) | Service identities and supporting resources. | <code>google_kms_crypto_key_iam_member</code> · <code>google_project_service_identity</code> |
| [shared-vpc.tf](./shared-vpc.tf) | Shared VPC project-level configuration. | <code>google_compute_shared_vpc_host_project</code> · <code>google_compute_shared_vpc_service_project</code> |
| [shared-vpc.tf](./shared-vpc.tf) | Shared VPC project-level configuration. | <code>google_compute_shared_vpc_host_project</code> · <code>google_compute_shared_vpc_service_project</code> · <code>google_project_iam_member</code> |
| [variables.tf](./variables.tf) | Module variables. | |
| [versions.tf](./versions.tf) | Version pins. | |
| [vpc-sc.tf](./vpc-sc.tf) | VPC-SC project-level perimeter configuration. | <code>google_access_context_manager_service_perimeter_resource</code> |
@ -224,9 +257,9 @@ module "project" {
| [service_perimeter_bridges](variables.tf#L211) | Name of VPC-SC Bridge perimeters to add project into. See comment in the variables file for format. | <code>list&#40;string&#41;</code> | | <code>null</code> |
| [service_perimeter_standard](variables.tf#L218) | Name of VPC-SC Standard perimeter to add project into. See comment in the variables file for format. | <code>string</code> | | <code>null</code> |
| [services](variables.tf#L224) | Service APIs to enable. | <code>list&#40;string&#41;</code> | | <code>&#91;&#93;</code> |
| [shared_vpc_host_config](variables.tf#L230) | Configures this project as a Shared VPC host project (mutually exclusive with shared_vpc_service_project). | <code title="object&#40;&#123;&#10; enabled &#61; bool&#10; service_projects &#61; list&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code title="&#123;&#10; enabled &#61; false&#10; service_projects &#61; &#91;&#93;&#10;&#125;">&#123;&#8230;&#125;</code> |
| [shared_vpc_service_config](variables.tf#L243) | Configures this project as a Shared VPC service project (mutually exclusive with shared_vpc_host_config). | <code title="object&#40;&#123;&#10; attach &#61; bool&#10; host_project &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code title="&#123;&#10; attach &#61; false&#10; host_project &#61; &#34;&#34;&#10;&#125;">&#123;&#8230;&#125;</code> |
| [skip_delete](variables.tf#L256) | Allows the underlying resources to be destroyed without destroying the project itself. | <code>bool</code> | | <code>false</code> |
| [shared_vpc_host_config](variables.tf#L230) | Configures this project as a Shared VPC host project (mutually exclusive with shared_vpc_service_project). | <code title="object&#40;&#123;&#10; enabled &#61; bool&#10; service_projects &#61; list&#40;string&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> |
| [shared_vpc_service_config](variables.tf#L239) | Configures this project as a Shared VPC service project (mutually exclusive with shared_vpc_host_config). | <code title="object&#40;&#123;&#10; host_project &#61; string&#10; service_identity_iam &#61; map&#40;list&#40;string&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>null</code> |
| [skip_delete](variables.tf#L249) | Allows the underlying resources to be destroyed without destroying the project itself. | <code>bool</code> | | <code>false</code> |
## Outputs
@ -234,9 +267,9 @@ module "project" {
|---|---|:---:|
| [custom_roles](outputs.tf#L17) | Ids of the created custom roles. | |
| [name](outputs.tf#L25) | Project name. | |
| [number](outputs.tf#L37) | Project number. | |
| [project_id](outputs.tf#L49) | Project id. | |
| [service_accounts](outputs.tf#L63) | Product robot service accounts in project. | |
| [sink_writer_identities](outputs.tf#L79) | Writer identities created for each sink. | |
| [number](outputs.tf#L38) | Project number. | |
| [project_id](outputs.tf#L51) | Project id. | |
| [service_accounts](outputs.tf#L66) | Product robot service accounts in project. | |
| [sink_writer_identities](outputs.tf#L82) | Writer identities created for each sink. | |
<!-- END TFDOC -->

View File

@ -30,6 +30,7 @@ output "name" {
google_project_organization_policy.list,
google_project_service.project_services,
google_compute_shared_vpc_service_project.service_projects,
google_project_iam_member.shared_vpc_host_robots,
google_kms_crypto_key_iam_member.service_identity_cmek
]
}
@ -42,6 +43,7 @@ output "number" {
google_project_organization_policy.list,
google_project_service.project_services,
google_compute_shared_vpc_service_project.service_projects,
google_project_iam_member.shared_vpc_host_robots,
google_kms_crypto_key_iam_member.service_identity_cmek
]
}
@ -56,6 +58,7 @@ output "project_id" {
google_project_organization_policy.list,
google_project_service.project_services,
google_compute_shared_vpc_service_project.service_projects,
google_project_iam_member.shared_vpc_host_robots,
google_kms_crypto_key_iam_member.service_identity_cmek
]
}

View File

@ -29,6 +29,8 @@ locals {
bq = "bq-%s@bigquery-encryption"
cloudasset = "service-%s@gcp-sa-cloudasset"
cloudbuild = "service-%s@gcp-sa-cloudbuild"
cloudfunctions = "service-%s@gcf-admin-robot"
cloudrun = "service-%s@serverless-robot-prod"
composer = "service-%s@cloudcomposer-accounts"
compute = "service-%s@compute-system"
container-engine = "service-%s@container-engine-robot"
@ -36,10 +38,11 @@ locals {
dataflow = "service-%s@dataflow-service-producer-prod"
dataproc = "service-%s@dataproc-accounts"
gae-flex = "service-%s@gae-api-prod"
gcf = "service-%s@gcf-admin-robot"
pubsub = "service-%s@gcp-sa-pubsub"
secretmanager = "service-%s@gcp-sa-secretmanager"
storage = "service-%s@gs-project-accounts"
# TODO: deprecate gcf
gcf = "service-%s@gcf-admin-robot"
pubsub = "service-%s@gcp-sa-pubsub"
secretmanager = "service-%s@gcp-sa-secretmanager"
storage = "service-%s@gs-project-accounts"
}
service_accounts_default = {
compute = "${local.project.number}-compute@developer.gserviceaccount.com"

View File

@ -16,19 +16,41 @@
# tfdoc:file:description Shared VPC project-level configuration.
locals {
# compute the host project IAM bindings for this project's service identities
_svpc_service_identity_iam = coalesce(
local.svpc_service_config.service_identity_iam, {}
)
_svpc_service_iam = flatten([
for role, services in local._svpc_service_identity_iam : [
for service in services : { role = role, service = service }
]
])
svpc_host_config = {
enabled = coalesce(
try(var.shared_vpc_host_config.enabled, null), false
)
service_projects = coalesce(
try(var.shared_vpc_host_config.service_projects, null), []
)
}
svpc_service_config = coalesce(var.shared_vpc_service_config, {
host_project = null, service_identity_iam = {}
})
svpc_service_iam = {
for b in local._svpc_service_iam : "${b.role}:${b.service}" => b
}
}
resource "google_compute_shared_vpc_host_project" "shared_vpc_host" {
provider = google-beta
count = try(var.shared_vpc_host_config.enabled, false) ? 1 : 0
count = local.svpc_host_config.enabled ? 1 : 0
project = local.project.project_id
}
resource "google_compute_shared_vpc_service_project" "service_projects" {
provider = google-beta
for_each = (
try(var.shared_vpc_host_config.enabled, false)
? toset(coalesce(var.shared_vpc_host_config.service_projects, []))
: toset([])
)
provider = google-beta
for_each = toset(local.svpc_host_config.service_projects)
host_project = local.project.project_id
service_project = each.value
depends_on = [google_compute_shared_vpc_host_project.shared_vpc_host]
@ -36,7 +58,19 @@ resource "google_compute_shared_vpc_service_project" "service_projects" {
resource "google_compute_shared_vpc_service_project" "shared_vpc_service" {
provider = google-beta
count = try(var.shared_vpc_service_config.attach, false) ? 1 : 0
count = local.svpc_service_config.host_project != null ? 1 : 0
host_project = var.shared_vpc_service_config.host_project
service_project = local.project.project_id
}
resource "google_project_iam_member" "shared_vpc_host_robots" {
for_each = local.svpc_service_iam
project = var.shared_vpc_service_config.host_project
role = each.value.role
member = (
each.value.service == "cloudservices"
? local.service_account_cloud_services
: local.service_accounts_robots[each.value.service]
)
}

View File

@ -233,24 +233,17 @@ variable "shared_vpc_host_config" {
enabled = bool
service_projects = list(string)
})
default = {
enabled = false
service_projects = []
}
nullable = false
default = null
}
variable "shared_vpc_service_config" {
description = "Configures this project as a Shared VPC service project (mutually exclusive with shared_vpc_host_config)."
# the list of valid service identities is in service-accounts.tf
type = object({
attach = bool
host_project = string
host_project = string
service_identity_iam = map(list(string))
})
default = {
attach = false
host_project = ""
}
nullable = false
default = null
}
variable "skip_delete" {

View File

@ -88,12 +88,15 @@ def e2e_plan_runner(_plan_runner):
@ pytest.fixture(scope='session')
def example_plan_runner(_plan_runner):
def doc_example_plan_runner(_plan_runner):
"Returns a function to run Terraform plan on documentation examples."
def run_plan(fixture_path=None):
"Runs Terraform plan and returns count of modules and resources."
plan = _plan_runner(fixture_path)
tf = tftest.TerraformTest(fixture_path, BASEDIR,
os.environ.get('TERRAFORM', 'terraform'))
tf.setup(upgrade=True)
plan = tf.plan(output=True, refresh=True)
# the fixture is the example we are testing
modules = plan.modules or {}
return (

View File

@ -20,7 +20,7 @@ BASE_PATH = Path(__file__).parent
EXPECTED_RESOURCES_RE = re.compile(r'# tftest modules=(\d+) resources=(\d+)')
def test_example(example_plan_runner, tmp_path, example):
def test_example(doc_example_plan_runner, tmp_path, example):
(tmp_path / 'modules').symlink_to(
Path(BASE_PATH, '../../modules/').resolve())
(tmp_path / 'variables.tf').symlink_to(
@ -31,6 +31,6 @@ def test_example(example_plan_runner, tmp_path, example):
expected_modules = int(match.group(1)) if match is not None else 1
expected_resources = int(match.group(2)) if match is not None else 1
num_modules, num_resources = example_plan_runner(str(tmp_path))
num_modules, num_resources = doc_example_plan_runner(str(tmp_path))
assert expected_modules == num_modules
assert expected_resources == num_resources

View File

@ -0,0 +1,13 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

View File

@ -0,0 +1,24 @@
# skip boilerplate check
billing_account_id: 012345-67890A-BCDEF0
# [opt] Setup for billing alerts
billing_alert:
amount: 1000
thresholds:
current: [0.5, 0.8]
forecasted: [0.5, 0.8]
credit_treatment: INCLUDE_ALL_CREDITS
# [opt] Contacts for billing alerts and important notifications
essential_contacts: ["team-contacts@example.com"]
# [opt] Labels set for all projects
labels:
environment: prod
department: accounting
application: example-app
foo: bar
# [opt] Additional notification channels for billing
notification_channels: []

View File

@ -0,0 +1,51 @@
/**
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
locals {
_defaults = yamldecode(file(var.defaults_file))
_defaults_net = {
billing_account_id = var.billing_account_id
environment_dns_zone = var.environment_dns_zone
shared_vpc_self_link = var.shared_vpc_self_link
vpc_host_project = var.vpc_host_project
}
defaults = merge(local._defaults, local._defaults_net)
projects = {
for f in fileset("${var.data_dir}", "**/*.yaml") :
trimsuffix(f, ".yaml") => yamldecode(file("${var.data_dir}/${f}"))
}
}
module "projects" {
source = "../../../../../examples/factories/project-factory"
for_each = local.projects
defaults = local.defaults
project_id = each.key
billing_account_id = try(each.value.billing_account_id, null)
billing_alert = try(each.value.billing_alert, null)
dns_zones = try(each.value.dns_zones, [])
essential_contacts = try(each.value.essential_contacts, [])
folder_id = each.value.folder_id
group_iam = try(each.value.group_iam, {})
iam = try(each.value.iam, {})
kms_service_agents = try(each.value.kms, {})
labels = try(each.value.labels, {})
org_policies = try(each.value.org_policies, null)
service_accounts = try(each.value.service_accounts, {})
services = try(each.value.services, [])
service_identities_iam = try(each.value.service_identities_iam, {})
vpc = try(each.value.vpc, null)
}

View File

@ -0,0 +1,100 @@
# skip boilerplate check
# [opt] Billing account id - overrides default if set
billing_account_id: 012345-67890A-BCDEF0
# [opt] Billing alerts config - overrides default if set
billing_alert:
amount: 10
thresholds:
current:
- 0.5
- 0.8
forecasted: []
credit_treatment: INCLUDE_ALL_CREDITS
# [opt] DNS zones to be created as children of the environment_dns_zone defined in defaults
dns_zones:
- lorem
- ipsum
# [opt] Contacts for billing alerts and important notifications
essential_contacts:
- team-a-contacts@example.com
# Folder the project will be created as children of
folder_id: folders/012345678901
# [opt] Authoritative IAM bindings in group => [roles] format
group_iam:
test-team-foobar@fast-lab-0.gcp-pso-italy.net:
- roles/compute.admin
# [opt] Authoritative IAM bindings in role => [principals] format
# Generally used to grant roles to service accounts external to the project
iam:
roles/compute.admin:
- serviceAccount:service-account
# [opt] Service robots and keys they will be assigned as cryptoKeyEncrypterDecrypter
# in service => [keys] format
kms_service_agents:
compute: [key1, key2]
storage: [key1, key2]
# [opt] Labels for the project - merged with the ones defined in defaults
labels:
environment: prod
# [opt] Org policy overrides defined at project level
org_policies:
policy_boolean:
constraints/compute.disableGuestAttributesAccess: true
policy_list:
constraints/compute.trustedImageProjects:
inherit_from_parent: null
status: true
suggested_value: null
values:
- projects/fast-prod-iac-core-0
# [opt] Service account to create for the project and their roles on the project
# in name => [roles] format
service_accounts:
another-service-account:
- roles/compute.admin
my-service-account:
- roles/compute.admin
# [opt] APIs to enable on the project.
services:
- storage.googleapis.com
- stackdriver.googleapis.com
- compute.googleapis.com
# [opt] Roles to assign to the service identities in service => [roles] format
service_identities_iam:
compute:
- roles/storage.objectViewer
# [opt] VPC setup.
# If set enables the `compute.googleapis.com` service and configures
# service project attachment
vpc:
# [opt] If set, enables the container API
gke_setup:
# Grants "roles/container.hostServiceAgentUser" to the container robot if set
enable_host_service_agent: false
# Grants "roles/compute.securityAdmin" to the container robot if set
enable_security_admin: true
# Host project the project will be service project of
host_project: fast-prod-net-spoke-0
# [opt] Subnets in the host project where principals will be granted networkUser
# in region/subnet-name => [principals]
subnets_iam:
europe-west1/prod-default-ew1:
- user:foobar@example.com
- serviceAccount:service-account1

View File

@ -0,0 +1,52 @@
/**
* Copyright 2022 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
variable "billing_account_id" {
description = "Billing account id."
type = string
default = "012345-67890A-BCDEF0"
}
variable "data_dir" {
description = "Relative path for the folder storing configuration data."
type = string
default = "./projects/"
}
variable "environment_dns_zone" {
description = "DNS zone suffix for environment."
type = string
default = "prod.gcp.example.com"
}
variable "defaults_file" {
description = "Relative path for the file storing the project factory configuration."
type = string
default = "./defaults.yaml"
}
variable "shared_vpc_self_link" {
description = "Self link for the shared VPC."
type = string
default = "self-link"
}
variable "vpc_host_project" {
# tfdoc:variable:source 02-networking
description = "Host project for the shared VPC."
type = string
default = "host-project"
}

View File

@ -0,0 +1,18 @@
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
def test_counts(e2e_plan_runner):
"Check for a clean plan"
modules, resources = e2e_plan_runner()
assert len(modules) > 0 and len(resources) > 0

View File

@ -16,8 +16,8 @@ def test_resources(e2e_plan_runner):
"Test that plan works and the numbers of resources is as expected."
modules, resources = e2e_plan_runner()
assert len(modules) == 11
assert len(resources) == 29
assert len(resources) == 30
modules, resources = e2e_plan_runner(mig="true")
assert len(modules) == 13
assert len(resources) == 35
assert len(resources) == 36

View File

@ -16,4 +16,4 @@ def test_resources(e2e_plan_runner):
"Test that plan works and the numbers of resources is as expected."
modules, resources = e2e_plan_runner()
assert len(modules) == 10
assert len(resources) == 41
assert len(resources) == 43